Changes:

1. gunicorn.py - added a post_fork() that calls db.engine.dispose() if preload_app
2. eNMS/controller/base.py - added `atexit` method to call db.engine.dispose()
3. eNMS/models/automation.py - bug fix to log method in enter_remote_device
4. docs/automation/installation.rst - added information on WatchedFileHandler for logging.json and fixed any build warnings.
5. docs/automation/services.rst - in Variables section, added separate Device and Link custom properties files.

docs/automation/custom_device_properties.rst

@@ -0,0 +1,5 @@
+==============================================================
+Custom Device Properties (if defined in setup/properties.json)
+==============================================================
+
+Currently None

docs/automation/custom_link_properties.rst

@@ -0,0 +1,5 @@
+============================================================
+Custom Link Properties (if defined in setup/properties.json)
+============================================================
+
+Currently None

docs/automation/services.rst

@@ -286,7 +286,7 @@ Variables
       :maxdepth: 2
       :titlesonly:
 
-      custom_properties.rst
+      custom_device_properties.rst
 
 - ``link``
 
@@ -307,7 +307,7 @@ Variables
       :maxdepth: 2
       :titlesonly:
 
-      custom_properties.rst
+      custom_link_properties.rst
 
 - ``get_result`` (see :ref:`get_result`)
 

docs/base/installation.rst

@@ -144,9 +144,10 @@ Settings ``app`` section
   not work consistently depending on your environment (nginx configuration, proxy, ...)
 - ``config_mode`` (default: ``"debug"``) Must be set to "debug" or "production".
 - ``startup_migration`` (default: ``"examples"``) Name of the migration to load when eNMS starts for the first time.
-By default, when eNMS loads for the first time, it will create a network topology and a number of services and workflows
-as examples of what you can do. You can set the migration to ``"default"`` instead, in which case eNMS will only load what
-is required for the application to function properly.
+
+  - By default, when eNMS loads for the first time, it will create a network topology and a number of services and workflows as examples of what you can do.
+  - You can set the migration to ``"default"`` instead, in which case eNMS will only load what is required for the application to function properly.
+
 - ``documentation_url`` (default: ``"https://enms.readthedocs.io/en/latest/"``) Can be changed if you want to host your
   own version of the documentation locally. Points to the online documentation by default.
 - ``git_repository`` (default: ``""``) Git is used as a version control system for device configurations: this variable
@@ -326,7 +327,23 @@ By default, the two loggers are configured:
 
 And these can be reconfigured here to forward through syslog to remote collection if desired.
 
-Additionally, the ``external loggers`` section allows for changing the log levels for the various libraries used by eNMS
+Additionally, the ``external loggers`` section allows for changing the log levels for the various libraries used by eNMS.
+
+With multiple gunicorn workers, please consider:
+  - Using ``Python WatchedFileHandler`` instead of the ``RotatingFileHandler``
+  - Configuring the LINUX ``logrotate`` utility to perform the desired log rotation
+
+.. code-block:: JSON
+
+  {
+  "handlers": {
+    "rotation": {
+      "level": "DEBUG",
+      "formatter": "standard",
+      "filename": "logs/enms.log",
+      "class": "logging.handlers.WatchedFileHandler"
+    }
+  }
 
 
 Properties file
@@ -349,11 +366,11 @@ properties.json custom device addition example:
             - "type":"string", *data type of attribute*
             - "default":"None", *default value of attribute*
             - "private": true *optional - is attribute hidden from user*
-            - "configuration": true *optional - creates a custom 'Inventory/Configurations' attribute
-            - "log_change" false *optional - disables logging when a changes is made to attribute
-            - "form": false *optional - disables option to edit attribute in Device User Interface
-            - "migrate": fasle *optional - choose if attribute should be consdered for migration
-            - "serialize": false *optional - whether it is passed to the front-end when the object itself is
+            - "configuration": true *optional* - creates a custom 'Inventory/Configurations' attribute
+            - "log_change" false *optional* - disables logging when a changes is made to attribute
+            - "form": false *optional* - disables option to edit attribute in Device User Interface
+            - "migrate": fasle *optional* - choose if attribute should be consdered for migration
+            - "serialize": false *optional* - whether it is passed to the front-end when the object itself is
     - Keys under ``"tables" : { "device" : [ {  & "tables" : { "configuration" : [ {``
         - Details which attributes to display in these table, add custom attributes here
         - Keys/Value pairs for tables
@@ -384,7 +401,7 @@ Environment variables
   - SLACK_TOKEN=slack_token
 
 Scheduler
----------
+#########
 
 The scheduler used for running tasks at a later time is a web application that is distinct from eNMS.
 It can be installed on the same server as eNMS, or a remote server.

eNMS/controller/base.py

@@ -31,6 +31,7 @@
 from sys import path as sys_path
 from uuid import getnode
 from warnings import warn
+import atexit
 
 try:
     from hvac import Client as VaultClient
@@ -692,3 +693,8 @@ def update_database_configurations_from_git(self):
                 for property in self.configuration_properties
             ):
                 pool.compute_pool()
+
+
+@atexit.register
+def cleanup():
+    db.engine.dispose()  # gracefully close database connections

eNMS/models/automation.py

@@ -1410,7 +1410,12 @@ def enter_remote_device(self, connection, device):
         for (send, expect) in zip(commands[::2], commands[1::2]):
             if not send or not expect:
                 continue
-            self.log("info", f"Sent '{send}', waiting for '{expect}'", device)
+            self.log(
+                "info",
+                f"Sent '{send if send != commands[4] else 'jump on connect password'}'",
+                f", waiting for '{expect}'",
+                device,
+            )
             connection.send_command(
                 send,
                 expect_string=expect,

gunicorn.py

@@ -12,3 +12,10 @@
 raw_env = ["TERM=screen"]
 timeout = 3000
 workers = 2 * cpu_count() + 1 if getenv("REDIS_ADDR") else 1
+
+
+def post_fork(server, worker):
+    if preload_app:
+        from eNMS.database import db
+
+        db.engine.dispose()