[master] Add websocket transport

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ on:
 
 env:
   COLUMNS: 190
-  CACHE_SEED: SEED-2  # Bump the number to invalidate all caches
+  CACHE_SEED: SEED-4  # Bump the number to invalidate all caches
   RELENV_DATA: "${{ github.workspace }}/.relenv"
 
 permissions:

.github/workflows/nightly.yml

@@ -22,7 +22,7 @@ on:
 
 env:
   COLUMNS: 190
-  CACHE_SEED: SEED-2  # Bump the number to invalidate all caches
+  CACHE_SEED: SEED-4  # Bump the number to invalidate all caches
   RELENV_DATA: "${{ github.workspace }}/.relenv"
 
 permissions:

.github/workflows/release.yml

@@ -21,7 +21,7 @@ on:
 
 env:
   COLUMNS: 190
-  CACHE_SEED: SEED-2  # Bump the number to invalidate all caches
+  CACHE_SEED: SEED-4  # Bump the number to invalidate all caches
   RELENV_DATA: "${{ github.workspace }}/.relenv"
 
 permissions:

.github/workflows/scheduled.yml

@@ -12,7 +12,7 @@ on:
 
 env:
   COLUMNS: 190
-  CACHE_SEED: SEED-2  # Bump the number to invalidate all caches
+  CACHE_SEED: SEED-4  # Bump the number to invalidate all caches
   RELENV_DATA: "${{ github.workspace }}/.relenv"
 
 permissions:

.github/workflows/staging.yml

@@ -37,7 +37,7 @@ on:
 
 env:
   COLUMNS: 190
-  CACHE_SEED: SEED-2  # Bump the number to invalidate all caches
+  CACHE_SEED: SEED-4  # Bump the number to invalidate all caches
   RELENV_DATA: "${{ github.workspace }}/.relenv"
 
 permissions:

.github/workflows/templates/layout.yml.jinja

@@ -34,7 +34,7 @@ on:
 
 env:
   COLUMNS: 190
-  CACHE_SEED: SEED-2  # Bump the number to invalidate all caches
+  CACHE_SEED: SEED-4  # Bump the number to invalidate all caches
   RELENV_DATA: "${{ github.workspace }}/.relenv"
 
 <%- endblock env %>

.pylintrc

@@ -699,7 +699,8 @@ allowed-3rd-party-modules=msgpack,
                           packaging,
                           looseversion,
                           pytestskipmarkers,
-                          cryptography
+                          cryptography,
+                          aiohttp
 
 [EXCEPTIONS]
 

doc/topics/transports/index.rst

@@ -38,3 +38,5 @@ The request client sends requests to a Request Server and receives a reply messa
 
     zeromq
     tcp
+    ws
+    ssl

doc/topics/transports/ssl.rst

@@ -0,0 +1,73 @@
+Transport TLS Support
+=====================
+
+Whenever possible transports should provide TLS Support. Currently the :doc:`tcp` and
+:doc:`ws` transports support encryption and verification using TLS.
+
+.. versionadded:: 2016.11.1
+
+The TCP transport allows for the master/minion communication to be optionally
+wrapped in a TLS connection. Enabling this is simple, the master and minion need
+to be using the tcp connection, then the ``ssl``  option is enabled. The ``ssl``
+option is passed as a dict and roughly corresponds to the options passed to the
+Python `ssl.wrap_socket <https://docs.python.org/3/library/ssl.html#ssl.wrap_socket>`_
+function for backwards compatability.
+
+.. versionadded:: 3007.0
+
+The ``ssl`` option accepts ``verify_locations`` and ``verify_flags``. The
+``verify_locations`` option is a list of strings or dictionaries. Strings are
+passed as a single argument to the SSL context's ``load_verify_locations``
+method. Dictionary keys are expected to be one of ``cafile``, ``capath``,
+``cadata``. For each corresponding key, the key and value will be passed as a
+keyword argument to ``load_verify_locations``. The ``verify_flags`` option is
+a list of string names of verification flags which will be set on the SSL
+context. All paths are assumed to be the full path to the file or directory.
+
+A simple setup looks like this, on the Salt Master add the ``ssl`` option to the
+master configuration file:
+
+.. code-block:: yaml
+
+    ssl:
+      keyfile: <path_to_keyfile>
+      certfile: <path_to_certfile>
+
+A more complex setup looks like this, on the Salt Master add the ``ssl``
+option to the master's configuration file. In this example the Salt Master will
+require valid client side certificates from Minions by setting ``cert_reqs`` to
+``CERT_REQUIRED``. The Salt Master will also check a certificate revocation list
+if one is provided in ``verify_locations``:
+
+.. code-block:: yaml
+
+    ssl:
+      keyfile: <path_to_keyfile>
+      certfile: <path_to_certfile>
+      cert_reqs: CERT_REQUIRED
+      verify_locations:
+        - <path_to_ca_cert>
+        - capath: <directory_of_certs>
+        - cafile: <path_to_crl>
+      verify_flags:
+        - VERIFY_CRL_CHECK_CHAIN
+
+
+The minimal `ssl` option in the minion configuration file looks like this:
+
+.. code-block:: yaml
+
+    ssl: True
+    # Versions below 2016.11.4:
+    ssl: {}
+
+A Minion can be configured to present a client certificate to the master like this:
+
+.. code-block:: yaml
+
+    ssl:
+      keyfile: <path_to_keyfile>
+      certfile: <path_to_certfile>
+
+Specific options can be sent to the minion also, as defined in the Python
+`ssl.wrap_socket` function.

doc/topics/transports/tcp.rst

@@ -19,6 +19,14 @@ to ``tcp`` on each Salt minion and Salt master.
     use the same transport. We're investigating a report of an error when using
     mixed transport types at very heavy loads.
 
+
+TLS Support
+===========
+
+The TLS transport supports full encryption and verification using both server
+and client certificates. See :doc:`ssl` for more details.
+
+
 Wire Protocol
 =============
 This implementation over TCP focuses on flexibility over absolute efficiency.
@@ -37,51 +45,9 @@ actual message that we are sending. With this flexible wire protocol we can
 implement any message semantics that we'd like-- including multiplexed message
 passing on a single socket.
 
-TLS Support
-===========
-
-.. versionadded:: 2016.11.1
-
-The TCP transport allows for the master/minion communication to be optionally
-wrapped in a TLS connection. Enabling this is simple, the master and minion need
-to be using the tcp connection, then the `ssl` option is enabled. The `ssl`
-option is passed as a dict and corresponds to the options passed to the
-Python `ssl.wrap_socket <https://docs.python.org/3/library/ssl.html#ssl.wrap_socket>`_
-function.
-
-A simple setup looks like this, on the Salt Master add the `ssl` option to the
-master configuration file:
-
-.. code-block:: yaml
-
-    ssl:
-      keyfile: <path_to_keyfile>
-      certfile: <path_to_certfile>
-      ssl_version: PROTOCOL_TLSv1_2
-      ciphers: ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384
-
-The minimal `ssl` option in the minion configuration file looks like this:
-
-.. code-block:: yaml
-
-    ssl: True
-    # Versions below 2016.11.4:
-    ssl: {}
-
-Specific options can be sent to the minion also, as defined in the Python
-`ssl.wrap_socket` function.
-
-.. note::
-
-    While setting the ssl_version is not required, we recommend it. Some older
-    versions of python do not support the latest TLS protocol and if this is
-    the case for your version of python we strongly recommend upgrading your
-    version of Python. Ciphers specification might be omitted, but strongly
-    recommended as otherwise all available ciphers will be enabled.
-
-
 Crypto
 ======
+
 The current implementation uses the same crypto as the ``zeromq`` transport.
 
 

doc/topics/transports/ws.rst

@@ -0,0 +1,21 @@
+===================
+Websocket Transport
+===================
+
+The Websocket transport is an implementation of Salt's transport using the websocket protocol.
+The Websocket transport is enabled by changing the :conf_minion:`transport` setting
+to ``ws`` on each Salt minion and Salt master.
+
+TLS Support
+===========
+
+The Websocket transport supports full encryption and verification using both server
+and client certificates. See :doc:`ssl` for more details.
+
+Publish Server and Client
+=========================
+The publish server and client are implemented using aiohttp.
+
+Request Server and Client
+=========================
+The request server and client are implemented using aiohttp.

requirements/base.txt

@@ -11,6 +11,7 @@ psutil>=5.0.0
 packaging>=21.3
 looseversion
 tornado>=6.3.3
+aiohttp>=3.9.0
 
 # We need contextvars for salt-ssh.
 # Even on python versions which ships with contextvars in the standard library!

requirements/static/ci/py3.10/darwin.txt

@@ -4,20 +4,28 @@
 #
 #    pip-compile --no-emit-index-url --output-file=requirements/static/ci/py3.10/darwin.txt requirements/darwin.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/darwin.in requirements/static/pkg/darwin.in
 #
-aiohttp==3.9.0
-    # via etcd3-py
+aiohttp==3.9.1
+    # via
+    #   -c requirements/static/ci/../pkg/py3.10/darwin.txt
+    #   -r requirements/base.txt
+    #   etcd3-py
 aiosignal==1.3.1
-    # via aiohttp
+    # via
+    #   -c requirements/static/ci/../pkg/py3.10/darwin.txt
+    #   aiohttp
 apache-libcloud==3.7.0 ; sys_platform != "win32"
     # via -r requirements/static/ci/common.in
 asn1crypto==1.5.1
     # via
     #   certvalidator
     #   oscrypto
-async-timeout==4.0.2
-    # via aiohttp
+async-timeout==4.0.3
+    # via
+    #   -c requirements/static/ci/../pkg/py3.10/darwin.txt
+    #   aiohttp
 attrs==23.1.0
     # via
+    #   -c requirements/static/ci/../pkg/py3.10/darwin.txt
     #   aiohttp
     #   jsonschema
     #   pytest-salt-factories
@@ -119,8 +127,9 @@ filelock==3.13.1
     # via virtualenv
 flaky==3.7.0
     # via -r requirements/pytest.txt
-frozenlist==1.3.3
+frozenlist==1.4.0
     # via
+    #   -c requirements/static/ci/../pkg/py3.10/darwin.txt
     #   aiohttp
     #   aiosignal
 future==0.18.3
@@ -247,6 +256,7 @@ msgpack==1.0.7
     #   pytest-salt-factories
 multidict==6.0.4
     # via
+    #   -c requirements/static/ci/../pkg/py3.10/darwin.txt
     #   aiohttp
     #   yarl
 napalm==4.1.0 ; sys_platform != "win32"
@@ -546,8 +556,10 @@ yamllint==1.32.0
     # via -r requirements/static/ci/darwin.in
 yamlordereddictloader==0.4.0
     # via junos-eznc
-yarl==1.9.2
-    # via aiohttp
+yarl==1.9.4
+    # via
+    #   -c requirements/static/ci/../pkg/py3.10/darwin.txt
+    #   aiohttp
 zc.lockfile==3.0.post1
     # via
     #   -c requirements/static/ci/../pkg/py3.10/darwin.txt

requirements/static/ci/py3.10/freebsd.txt

@@ -4,20 +4,28 @@
 #
 #    pip-compile --no-emit-index-url --output-file=requirements/static/ci/py3.10/freebsd.txt requirements/base.txt requirements/pytest.txt requirements/static/ci/common.in requirements/static/ci/freebsd.in requirements/static/pkg/freebsd.in requirements/zeromq.txt
 #
-aiohttp==3.9.0
-    # via etcd3-py
+aiohttp==3.9.1
+    # via
+    #   -c requirements/static/ci/../pkg/py3.10/freebsd.txt
+    #   -r requirements/base.txt
+    #   etcd3-py
 aiosignal==1.3.1
-    # via aiohttp
+    # via
+    #   -c requirements/static/ci/../pkg/py3.10/freebsd.txt
+    #   aiohttp
 apache-libcloud==3.7.0 ; sys_platform != "win32"
     # via -r requirements/static/ci/common.in
 asn1crypto==1.5.1
     # via
     #   certvalidator
     #   oscrypto
-async-timeout==4.0.2
-    # via aiohttp
+async-timeout==4.0.3
+    # via
+    #   -c requirements/static/ci/../pkg/py3.10/freebsd.txt
+    #   aiohttp
 attrs==23.1.0
     # via
+    #   -c requirements/static/ci/../pkg/py3.10/freebsd.txt
     #   aiohttp
     #   jsonschema
     #   pytest-salt-factories
@@ -118,8 +126,9 @@ filelock==3.13.1
     # via virtualenv
 flaky==3.7.0
     # via -r requirements/pytest.txt
-frozenlist==1.3.3
+frozenlist==1.4.0
     # via
+    #   -c requirements/static/ci/../pkg/py3.10/freebsd.txt
     #   aiohttp
     #   aiosignal
 future==0.18.3
@@ -250,6 +259,7 @@ msgpack==1.0.7
     #   pytest-salt-factories
 multidict==6.0.4
     # via
+    #   -c requirements/static/ci/../pkg/py3.10/freebsd.txt
     #   aiohttp
     #   yarl
 napalm==4.1.0 ; sys_platform != "win32"
@@ -551,8 +561,10 @@ yamllint==1.32.0
     # via -r requirements/static/ci/freebsd.in
 yamlordereddictloader==0.4.0
     # via junos-eznc
-yarl==1.9.2
-    # via aiohttp
+yarl==1.9.4
+    # via
+    #   -c requirements/static/ci/../pkg/py3.10/freebsd.txt
+    #   aiohttp
 zc.lockfile==3.0.post1
     # via
     #   -c requirements/static/ci/../pkg/py3.10/freebsd.txt