Releases/9.0.1 (#3171)

* Update readthedocs requirements.txt

* Update conf.py

* Update requirements_demo.txt

Add missing pandas requirement for demo

* work around for issue #3154

* Fix for security issue #3168 (#3169)

* Fix for security issue #3168

* handling clean up errors in test

* testing group commands in different test module

* moved group and role test to different module

* moved group and role test to different module

* Added a cache for agent names since platform start

* Fixes process overload from file events

* fixed issue with variable definition.

* Remove PersistentDict from web-user.json file.

* Update admin_endpoints.py

Handle behavior of removing PersistentDict

* Update version to 9.0.1

---------

Co-authored-by: Chandrika Sivaramakrishnan <chandrika@pnnl.gov>
Co-authored-by: Chandrika <schandrika@users.noreply.github.com>
Co-authored-by: Andrew Rodgers <andrew@aceiotsolutions.com>

docs/source/conf.py

@@ -90,9 +90,9 @@ def __getattr__(cls, name):
 author = 'The VOLTTRON Community'
 
 # The short X.Y version
-version = '9.0'
+version = '9.0.1'
 # The full version, including alpha/beta/rc tags
-release = '9.0'
+release = '9.0.1'
 
 # -- General configuration ---------------------------------------------------
 

services/core/IEEE_2030_5/requirements_demo.txt

@@ -2,3 +2,4 @@ nicegui
 requests
 xsdata>=23.8
 blinker
+pandas

volttron/platform/__init__.py

@@ -35,7 +35,7 @@
 from urllib.parse import urlparse
 
 from ..utils.frozendict import FrozenDict
-__version__ = '9.0rc0'
+__version__ = '9.0.1'
 
 _log = logging.getLogger(__name__)
 

volttron/platform/aip.py

@@ -247,6 +247,7 @@ def __init__(self, env, **kwargs):
         if self.message_bus == 'rmq':
             self.rmq_mgmt = RabbitMQMgmt()
         self.instance_name = get_platform_instance_name()
+        self.agent_uuid_name_map = {}
 
     def add_agent_user_group(self):
         user = pwd.getpwuid(os.getuid())
@@ -682,11 +683,14 @@ def remove_agent(self, agent_uuid, remove_auth=True):
             self.remove_agent_user(volttron_agent_user)
 
     def agent_name(self, agent_uuid):
+        if cached_name := self.agent_uuid_name_map.get(agent_uuid):
+            return cached_name
         agent_path = os.path.join(self.install_dir, agent_uuid)
         for agent_name in os.listdir(agent_path):
             dist_info = os.path.join(
                 agent_path, agent_name, agent_name + '.dist-info')
             if os.path.exists(dist_info):
+                self.agent_uuid_name_map[agent_uuid] = agent_name
                 return agent_name
         raise KeyError(agent_uuid)
 

volttron/platform/main.py

@@ -435,9 +435,6 @@ def issue(self, topic, frames, extra=None):
     #    return result
 
     def handle_subsystem(self, frames, user_id):
-        _log.debug(
-            f"Handling subsystem with frames: {frames} user_id: {user_id}")
-
         subsystem = frames[5]
         if subsystem == 'quit':
             sender = frames[0]

volttron/platform/vip/agent/subsystems/auth.py

@@ -265,6 +265,7 @@ def update_rpc_method_capabilities(self):
         """
         rpc_method_authorizations = {}
         rpc_methods = self.get_rpc_exports()
+        updated_rpc_authorizations = None
         for method in rpc_methods:
             if len(method.split(".")) > 1:
                 pass
@@ -295,9 +296,7 @@ def update_rpc_method_capabilities(self):
                 _log.info(
                     f"Skipping updating rpc auth capabilities for agent "
                     f"{self._core().identity} connecting to remote address: {self._core().address} ")
-                updated_rpc_authorizations = None
         except gevent.timeout.Timeout:
-            updated_rpc_authorizations = None
             _log.warning(f"update_id_rpc_authorization rpc call timed out for {self._core().identity}   {rpc_method_authorizations}")
         except MethodNotFound:
             _log.warning("update_id_rpc_authorization method is missing from "
@@ -306,7 +305,6 @@ def update_rpc_method_capabilities(self):
                          "dynamic RPC authorizations.")
             return
         except Exception as e:
-            updated_rpc_authorizations = None
             _log.exception(f"Exception when calling rpc method update_id_rpc_authorizations for identity: "
                            f"{self._core().identity}  Exception:{e}")
         if updated_rpc_authorizations is None:
@@ -318,7 +316,7 @@ def update_rpc_method_capabilities(self):
                 f"the identity of the agent"
             )
             return
-        if rpc_method_authorizations != updated_rpc_authorizations:
+        if rpc_method_authorizations != updated_rpc_authorizations and updated_rpc_authorizations is not None:
             for method in updated_rpc_authorizations:
                 self.set_rpc_authorizations(
                     method, updated_rpc_authorizations[method]

volttron/platform/vip/agent/subsystems/rpc.py

@@ -278,8 +278,8 @@ def _iterate_exports(self):
         for method_name in self._exports:
             method = self._exports[method_name]
             caps = annotations(method, set, "rpc.allow_capabilities")
-            # if caps:
-            #     self._exports[method_name] = self._add_auth_check(method, caps)
+            if caps:
+                self._exports[method_name] = self._add_auth_check(method, caps)
 
     def _add_auth_check(self, method, required_caps):
         """

volttron/platform/web/admin_endpoints.py

@@ -46,7 +46,6 @@
 from volttron.platform import get_home
 from volttron.platform import jsonapi
 from volttron.utils import VolttronHomeFileReloader
-from volttron.utils.persistance import PersistentDict
 
 
 _log = logging.getLogger(__name__)
@@ -84,7 +83,7 @@ def __init__(self, rmq_mgmt=None, ssl_public_key: bytes = None, rpc_caller=None)
         else:
             self._ssl_public_key = None
 
-        self._userdict = None
+        self._userdict = {}
         self.reload_userdict()
 
         self._observer = Observer()
@@ -96,7 +95,14 @@ def __init__(self, rmq_mgmt=None, ssl_public_key: bytes = None, rpc_caller=None)
 
     def reload_userdict(self):
         webuserpath = os.path.join(get_home(), 'web-users.json')
-        self._userdict = PersistentDict(webuserpath, format="json")
+        if os.path.exists(webuserpath):
+            with open(webuserpath) as fp:
+                try:
+                    self._userdict = jsonapi.loads(fp.read())
+                except json.decoder.JSONDecodeError:
+                    self._userdict = {}
+                    # Keep same behavior as with PersistentDict
+                    raise ValueError("File not in a supported format")
 
     def get_routes(self):
         """
@@ -339,4 +345,5 @@ def add_user(self, username, unencrypted_pw, groups=None, overwrite=False):
             groups=groups
         )
 
-        self._userdict.sync()
+        with open(os.path.join(get_home(), 'web-users.json'), 'w') as fp:
+            fp.write(jsonapi.dumps(self._userdict, indent=2))

volttron/utils/__init__.py

@@ -108,7 +108,7 @@ def __init__(self, filetowatch, callback):
         _log.debug("patterns is {}".format([get_home() + '/' + filetowatch]))
         self._callback = callback
 
-    def on_any_event(self, event):
+    def on_closed(self, event):
         _log.debug("Calling callback on event {}. Calling {}".format(event, self._callback))
         try:
             self._callback()
@@ -133,7 +133,7 @@ def __init__(self, filetowatch, callback):
     def watchfile(self):
         return self._filetowatch
 
-    def on_any_event(self, event):
+    def on_closed(self, event):
         _log.debug("Calling callback on event {}. Calling {}".format(event, self._callback))
         try:
             self._callback(self._filetowatch)

volttrontesting/platform/auth_tests/test_auth_control.py

@@ -375,20 +375,6 @@ def test_auth_rpc_method_remove(auth_instance):
     assert entries[-1]['rpc_method_authorizations'] != {'test_method': ["test_auth"]}
 
 
-@pytest.mark.control
-def test_group_cmds(auth_instance):
-    """Test add-group, list-groups, update-group, and remove-group"""
-    _run_group_or_role_cmds(auth_instance, _add_group, _list_groups,
-            _update_group, _remove_group)
-
-
-@pytest.mark.control
-def test_role_cmds(auth_instance):
-    """Test add-role, list-roles, update-role, and remove-role"""
-    _run_group_or_role_cmds(auth_instance, _add_role, _list_roles,
-            _update_role, _remove_role)
-
-
 def _run_group_or_role_cmds(platform, add_fn, list_fn, update_fn, remove_fn):
     expected = []
     key = '0'

volttrontesting/platform/auth_tests/test_auth_group_roles.py

@@ -0,0 +1,174 @@
+
+import os
+import re
+import subprocess
+
+import gevent
+import pytest
+from mock import MagicMock
+from volttron.platform.auth.auth_protocols.auth_zmq import ZMQAuthorization, ZMQServerAuthentication
+
+from volttrontesting.platform.auth_tests.conftest import assert_auth_entries_same
+from volttrontesting.utils.platformwrapper import with_os_environ
+from volttrontesting.utils.utils import AgentMock
+from volttron.platform.vip.agent import Agent
+from volttron.platform.auth import AuthService
+from volttron.platform.auth import AuthEntry
+from volttron.platform import jsonapi
+
+@pytest.fixture(autouse=True)
+def auth_instance(volttron_instance):
+    if not volttron_instance.auth_enabled:
+        pytest.skip("AUTH tests are not applicable if auth is disabled")
+    with open(os.path.join(volttron_instance.volttron_home, "auth.json"), 'r') as f:
+        auth_file = jsonapi.load(f)
+    print(auth_file)
+    try:
+        yield volttron_instance
+    finally:
+        with with_os_environ(volttron_instance.env):
+            with open(os.path.join(volttron_instance.volttron_home, "auth.json"), 'w') as f:
+                jsonapi.dump(auth_file, f)
+
+
+def _run_group_or_role_cmds(platform, add_fn, list_fn, update_fn, remove_fn):
+    expected = []
+    key = '0'
+    values = ['0', '1']
+    expected.extend(values)
+
+    add_fn(platform, key, values)
+    gevent.sleep(4)
+    keys = list_fn(platform)
+    assert set(keys[key]) == set(expected)
+
+    # Update add single value
+    values = ['2']
+    expected.extend(values)
+    update_fn(platform, key, values)
+    gevent.sleep(2)
+    keys = list_fn(platform)
+    assert set(keys[key]) == set(expected)
+
+    # Update add multiple values
+    values = ['3', '4']
+    expected.extend(values)
+    update_fn(platform, key, values)
+    gevent.sleep(2)
+    keys = list_fn(platform)
+    assert set(keys[key]) == set(expected)
+
+    # Update remove single value
+    value = '0'
+    expected.remove(value)
+    update_fn(platform, key, [value], remove=True)
+    gevent.sleep(2)
+    keys = list_fn(platform)
+    assert set(keys[key]) == set(expected)
+
+    # Update remove single value
+    values = ['1', '2']
+    for value in values:
+        expected.remove(value)
+    update_fn(platform, key, values, remove=True)
+    gevent.sleep(2)
+    keys = list_fn(platform)
+    assert set(keys[key]) == set(expected)
+
+    # Remove key
+    remove_fn(platform, key)
+    gevent.sleep(2)
+    keys = list_fn(platform)
+    assert key not in keys
+
+
+
+def _add_group_or_role(platform, cmd, name, list_):
+    with with_os_environ(platform.env):
+        args = ['volttron-ctl', 'auth', cmd, name]
+        args.extend(list_)
+        p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True)
+        p.communicate()
+        assert p.returncode == 0
+
+
+def _add_group(platform, group, roles):
+    _add_group_or_role(platform, 'add-group', group, roles)
+
+
+def _add_role(platform, role, capabilities):
+    _add_group_or_role(platform, 'add-role', role, capabilities)
+
+
+def _list_groups_or_roles(platform, cmd):
+    with with_os_environ(platform.env):
+        output = subprocess.check_output(['volttron-ctl', 'auth', cmd],
+                                        env=platform.env, universal_newlines=True)
+        # For these tests don't use names that contain space, [, comma, or '
+        output = output.replace('[', '').replace("'", '').replace(']', '')
+        output = output.replace(',', '')
+        lines = output.split('\n')
+
+        dict_ = {}
+        for line in lines[2:-1]: # skip two header lines and last (empty) line
+            list_ = ' '.join(line.split()).split() # combine multiple spaces
+            dict_[list_[0]] = list_[1:]
+        return dict_
+
+
+def _list_groups(platform):
+    return _list_groups_or_roles(platform, 'list-groups')
+
+
+def _list_roles(platform):
+    return _list_groups_or_roles(platform, 'list-roles')
+
+
+def _update_group_or_role(platform, cmd, key, values, remove):
+    with with_os_environ(platform.env):
+        args = ['volttron-ctl', 'auth', cmd, key]
+        args.extend(values)
+        if remove:
+            args.append('--remove')
+        p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True)
+        p.communicate()
+        assert p.returncode == 0
+
+
+def _update_group(platform, group, roles, remove=False):
+    _update_group_or_role(platform, 'update-group', group, roles, remove)
+
+
+def _update_role(platform, role, caps, remove=False):
+    _update_group_or_role(platform, 'update-role', role, caps, remove)
+
+
+def _remove_group_or_role(platform, cmd, key):
+    with with_os_environ(platform.env):
+        args = ['volttron-ctl', 'auth', cmd, key]
+        p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True)
+        p.communicate()
+        assert p.returncode == 0
+
+
+def _remove_group(platform, group):
+    _remove_group_or_role(platform, 'remove-group', group)
+
+
+def _remove_role(platform, role):
+    _remove_group_or_role(platform, 'remove-role', role)
+
+
+@pytest.mark.control
+def test_group_cmds(auth_instance):
+    """Test add-group, list-groups, update-group, and remove-group"""
+    _run_group_or_role_cmds(auth_instance, _add_group, _list_groups,
+            _update_group, _remove_group)
+
+
+@pytest.mark.control
+def test_role_cmds(auth_instance):
+    """Test add-role, list-roles, update-role, and remove-role"""
+    _run_group_or_role_cmds(auth_instance, _add_role, _list_roles,
+            _update_role, _remove_role)
+