tweaks to install.py for opensearch URL stuff

README.md

@@ -438,19 +438,21 @@ Alternately, if you have forked Malcolm on GitHub, [workflow files](./.github/wo
 $ ./scripts/malcolm_appliance_packager.sh 
 You must set a username and password for Malcolm, and self-signed X.509 certificates will be generated
 
-Store administrator username/password for local Malcolm access? (Y/n): 
+Store administrator username/password for local Malcolm access? (Y/n): y
 
 Administrator username: analyst
-analyst password: 
-analyst password (again): 
+analyst password: : 
+analyst password (again): : 
+
+(Re)generate self-signed certificates for HTTPS access (Y/n): y 
 
-(Re)generate self-signed certificates for HTTPS access (Y/n): 
+(Re)generate self-signed certificates for a remote log forwarder (Y/n): y
 
-(Re)generate self-signed certificates for a remote log forwarder (Y/n): 
+Store username/password for primary remote OpenSearch instance? (y/N): n
 
-Store username/password for forwarding Logstash events to a secondary, external OpenSearch instance (y/N): 
+Store username/password for secondary remote OpenSearch instance? (y/N): n
 
-Store username/password for email alert sender account (y/N): 
+Store username/password for email alert sender account? (y/N): n
 
 Packaged Malcolm to "/home/user/tmp/malcolm_20190513_101117_f0d052c.tar.gz"
 
@@ -1567,11 +1569,11 @@ Store administrator username/password for local Malcolm access? (Y/n): n
 
 (Re)generate self-signed certificates for a remote log forwarder (Y/n): n
 
-Store username/password for forwarding Logstash events to a secondary, external OpenSearch instance (y/N): n
+Store username/password for primary remote OpenSearch instance? (y/N): n
 
-Store username/password for email alert sender account (y/N): y
+Store username/password for secondary remote OpenSearch instance? (y/N): n
 
-OpenSearch alerting destination name: destination_alpha
+Store username/password for email alert sender account? (y/N): y
 
 Email account username: analyst@example.org
 analyst@example.org password: 
@@ -3719,6 +3721,10 @@ Now that any necessary system configuration changes have been made, the local Ma
 ```
 Malcolm processes will run as UID 1000 and GID 1000. Is this OK? (Y/n): y
 
+Should Malcolm use and maintain its own OpenSearch instance? (Y/n): y
+
+Forward Logstash logs to a secondary remote OpenSearch instance? (y/N): n
+
 Setting 10g for OpenSearch and 3g for Logstash. Is this OK? (Y/n): y
 
 Setting 3 workers for Logstash pipelines. Is this OK? (Y/n): y
@@ -3760,8 +3766,6 @@ Expose OpenSearch port to external hosts? (y/N): n
 
 Expose Logstash port to external hosts? (y/N): n
 
-Forward Logstash logs to external OpenSearch instance? (y/N): n
-
 Expose Filebeat TCP port to external hosts? (y/N): y
 1: json
 2: raw
@@ -3828,16 +3832,18 @@ user@host:~/Malcolm$ ./scripts/auth_setup
 Store administrator username/password for local Malcolm access? (Y/n): y
 
 Administrator username: analyst
-analyst password:
-analyst password (again):
+analyst password: : 
+analyst password (again): : 
 
-(Re)generate self-signed certificates for HTTPS access (Y/n): y
+(Re)generate self-signed certificates for HTTPS access (Y/n): y 
 
 (Re)generate self-signed certificates for a remote log forwarder (Y/n): y
 
-Store username/password for forwarding Logstash events to a secondary, external OpenSearch instance (y/N): n
+Store username/password for primary remote OpenSearch instance? (y/N): n
+
+Store username/password for secondary remote OpenSearch instance? (y/N): n
 
-Store username/password for email alert sender account (see https://opensearch.org/docs/latest/monitoring-plugins/alerting/monitors/#authenticate-sender-account) (y/N): n
+Store username/password for email alert sender account? (y/N): n
 ```
 
 For now, rather than [build Malcolm from scratch](#Build), we'll pull images from [Docker Hub](https://hub.docker.com/u/malcolmnetsec):

docker-compose-standalone.yml

@@ -50,24 +50,35 @@ x-opensearch-variables: &opensearch-variables
   OPENSEARCH_LOCAL : 'true'
   # URL for connecting to OpenSearch instance. When using Malcolm's internal instance
   #   of OpenSearch (i.e., OPENSEARCH_LOCAL is 'true') this should be
-  #   'http://opensearch:9200', otherwise specify the remote instance URL in the format
-  #   'protocol://host:port'.
+  #   'http://opensearch:9200', otherwise specify the primary remote instance URL
+  #   in the format 'protocol://host:port'.
   OPENSEARCH_URL : 'http://opensearch:9200'
   # Used when OPENSEARCH_LOCAL is 'false', the cURL-formatted config file contains login
-  #   credentials for the OpenSearch instance. See cURL config file format at
-  #   https://everything.curl.dev/cmdline/configfile. The notable parameters expected
-  #   from this file would be user (with a "user:password" value) and "insecure"
-  #   (if OPENSEARCH_SSL_CERTIFICATE_VERIFICATION below is 'false'). This file is
-  #   Bind mounted locally from .opensearch.primary.curlrc as /var/local/opensearch.primary.curlrc
+  #   credentials for the primary OpenSearch instance. It can be generated for you by the
+  #   ./scripts/auth_setup script. The notable parameters expected from this file would be
+  #   user (with a "user:password" value) and "insecure" (if the certificate verification
+  #   setting below is 'false'). See cURL config file format at
+  #   https://everything.curl.dev/cmdline/configfile. This file is bind mounted locally
+  #   from .opensearch.primary.curlrc as /var/local/opensearch.primary.curlrc
   OPENSEARCH_CREDS_CONFIG_FILE : '/var/local/opensearch.primary.curlrc'
-  # Whether or not connections to the remote OpenSearch instance requrie full
+  # Whether or not connections to the primary remote OpenSearch instance require full
   #  TLS certificate validation for the connection (this may fail if using self-signed
   #  certificates).
   OPENSEARCH_SSL_CERTIFICATE_VERIFICATION : 'false'
-  # TODO:
+  # Whether or not Malcolm's Logstash instance will forward logs to a secondary remote
+  #   OpenSearch instance in addition to the (local or remote) primary instance.
   OPENSEARCH_SECONDARY : 'false'
+  # URL for connecting to the secondary remote OpenSearch instance, specified
+  #   in the format 'protocol://host:port'.
   OPENSEARCH_SECONDARY_URL : ''
+  # Used when OPENSEARCH_SECONDARY is 'true', the cURL-formatted config file contains login
+  #   credentials for the secondary OpenSearch instance. The comments describing
+  #   OPENSEARCH_CREDS_CONFIG_FILE above also apply here. This file is bind mounted locally
+  #   from .opensearch.secondary.curlrc as /var/local/opensearch.secondary.curlrc
   OPENSEARCH_SECONDARY_CREDS_CONFIG_FILE : '/var/local/opensearch.secondary.curlrc'
+  # Whether or not connections to the secondary remote OpenSearch instance require full
+  #  TLS certificate validation for the connection (this may fail if using self-signed
+  #  certificates).
   OPENSEARCH_SECONDARY_SSL_CERTIFICATE_VERIFICATION : 'false'
 
 x-arkime-variables: &arkime-variables

docker-compose.yml

@@ -50,24 +50,35 @@ x-opensearch-variables: &opensearch-variables
   OPENSEARCH_LOCAL : 'true'
   # URL for connecting to OpenSearch instance. When using Malcolm's internal instance
   #   of OpenSearch (i.e., OPENSEARCH_LOCAL is 'true') this should be
-  #   'http://opensearch:9200', otherwise specify the remote instance URL in the format
-  #   'protocol://host:port'.
+  #   'http://opensearch:9200', otherwise specify the primary remote instance URL
+  #   in the format 'protocol://host:port'.
   OPENSEARCH_URL : 'http://opensearch:9200'
   # Used when OPENSEARCH_LOCAL is 'false', the cURL-formatted config file contains login
-  #   credentials for the OpenSearch instance. See cURL config file format at
-  #   https://everything.curl.dev/cmdline/configfile. The notable parameters expected
-  #   from this file would be user (with a "user:password" value) and "insecure"
-  #   (if OPENSEARCH_SSL_CERTIFICATE_VERIFICATION below is 'false'). This file is
-  #   Bind mounted locally from .opensearch.primary.curlrc as /var/local/opensearch.primary.curlrc
+  #   credentials for the primary OpenSearch instance. It can be generated for you by the
+  #   ./scripts/auth_setup script. The notable parameters expected from this file would be
+  #   user (with a "user:password" value) and "insecure" (if the certificate verification
+  #   setting below is 'false'). See cURL config file format at
+  #   https://everything.curl.dev/cmdline/configfile. This file is bind mounted locally
+  #   from .opensearch.primary.curlrc as /var/local/opensearch.primary.curlrc
   OPENSEARCH_CREDS_CONFIG_FILE : '/var/local/opensearch.primary.curlrc'
-  # Whether or not connections to the remote OpenSearch instance requrie full
+  # Whether or not connections to the primary remote OpenSearch instance require full
   #  TLS certificate validation for the connection (this may fail if using self-signed
   #  certificates).
   OPENSEARCH_SSL_CERTIFICATE_VERIFICATION : 'false'
-  # TODO:
+  # Whether or not Malcolm's Logstash instance will forward logs to a secondary remote
+  #   OpenSearch instance in addition to the (local or remote) primary instance.
   OPENSEARCH_SECONDARY : 'false'
+  # URL for connecting to the secondary remote OpenSearch instance, specified
+  #   in the format 'protocol://host:port'.
   OPENSEARCH_SECONDARY_URL : ''
+  # Used when OPENSEARCH_SECONDARY is 'true', the cURL-formatted config file contains login
+  #   credentials for the secondary OpenSearch instance. The comments describing
+  #   OPENSEARCH_CREDS_CONFIG_FILE above also apply here. This file is bind mounted locally
+  #   from .opensearch.secondary.curlrc as /var/local/opensearch.secondary.curlrc
   OPENSEARCH_SECONDARY_CREDS_CONFIG_FILE : '/var/local/opensearch.secondary.curlrc'
+  # Whether or not connections to the secondary remote OpenSearch instance require full
+  #  TLS certificate validation for the connection (this may fail if using self-signed
+  #  certificates).
   OPENSEARCH_SECONDARY_SSL_CERTIFICATE_VERIFICATION : 'false'
 
 x-arkime-variables: &arkime-variables

scripts/control.py

@@ -141,7 +141,7 @@ def keystore_op(service, dropPriv=False, *keystore_args, **run_process_kwargs):
                     '-T' if ('stdin' in run_process_kwargs and run_process_kwargs['stdin']) else '',
                     # execute as UID:GID in docker-compose.yml file
                     '-u',
-                    f'{uidGidDict["PUID"]}:{uidGidDict["PGID"]}'
+                    f'{uidGidDict["PUID"]}:{uidGidDict["PGID"]}',
                     # the work directory in the container is the directory to contain the keystore file
                     '-w',
                     volumeKeystoreDir,
@@ -1030,7 +1030,7 @@ def authSetup(wipe=False):
     # OpenSearch authenticate sender account credentials
     # https://opensearch.org/docs/latest/monitoring-plugins/alerting/monitors/#authenticate-sender-account
     if YesOrNo(
-        'Store username/password for email alert sender account (see https://opensearch.org/docs/latest/monitoring-plugins/alerting/monitors/#authenticate-sender-account)',
+        'Store username/password for email alert sender account? (see https://opensearch.org/docs/latest/monitoring-plugins/alerting/monitors/#authenticate-sender-account)',
         default=False,
     ):