diff --git a/README.md b/README.md index ac155a67..0b23eceb 100644 --- a/README.md +++ b/README.md @@ -67,8 +67,10 @@ _In a different terminal window `cd` into root of **common-workflow-service** fo For development we tend to create our own separate build script `` (firstinitial-lastname.sh), i.e.:`jsmith.sh`, that calls `dev.sh`. Here's an template for your personal build script that will work for development on a local machine: -* Correctly set the Elasticsearch configuration within your personal script by including the proper protocol `http://` or `https://` in the Elasticsearch endpoint. - * Example: `ES_HOST="http://localhost"` +* Correctly set the Elasticsearch configuration within your personal script by assigning the proper protocol, `HTTP` or `HTTPS`, to `ES_PROTOCOL` with Elasticsearch hostname assigned to `ES_HOST`. + * Example: + * `ES_PROTOCOL="HTTP"` + * `ES_HOST="locahost"` ``` #File: jsmith.sh @@ -103,7 +105,8 @@ ADMIN_LAST="{last}" ADMIN_EMAIL="{email}" # ES config -ES_HOST="http://localhost" +ES_PROTOCOL="HTTP" # options: 'HTTP' or 'HTTPS' +ES_HOST="localhost" ES_PORT=9200 ES_USE_AUTH=n ES_USERNAME="na" @@ -117,7 +120,7 @@ NUM_WORKERS=1 WORKER_ABANDONED_DAYS=1 # Run the dev script -./dev.sh `pwd` ${USER} ${DB_TYPE} ${DB_HOST} ${DB_PORT} ${DB_NAME} ${DB_USER} ${DB_PASS} ${ES_HOST} ${ES_PORT} ${ES_USE_AUTH} ${ES_USERNAME} ${ES_PASSWORD} ${CLOUD} ${SECURITY} ${HOSTNAME} ${EMAIL_LIST} ${ADMIN_FIRST} ${ADMIN_LAST} ${ADMIN_EMAIL} ${NUM_WORKERS} ${WORKER_ABANDONED_DAYS} +./dev.sh `pwd` ${USER} ${DB_TYPE} ${DB_HOST} ${DB_PORT} ${DB_NAME} ${DB_USER} ${DB_PASS} ${ES_PROTOCOL} ${ES_HOST} ${ES_PORT} ${ES_USE_AUTH} ${ES_USERNAME} ${ES_PASSWORD} ${CLOUD} ${SECURITY} ${HOSTNAME} ${EMAIL_LIST} ${ADMIN_FIRST} ${ADMIN_LAST} ${ADMIN_EMAIL} ${NUM_WORKERS} ${WORKER_ABANDONED_DAYS} ``` ###### Run Personal Dev Script diff --git a/create_server_dist.sh b/create_server_dist.sh index 86b7ab59..6fb20f73 100755 --- a/create_server_dist.sh +++ b/create_server_dist.sh @@ -218,7 +218,6 @@ cp ${INSTALL_DIR}/launch_ls.sh ${CWS} print 'Copying Modeller scripts and libraries...' cp -R ${INSTALL_DIR}/modeler ${CWS} -cp ${ROOT}/cws-modeler/install/modeler-config/* ${CWS}/modeler/ print 'Installing context.xml to Tomcat...' cp ${INSTALL_DIR}/context.xml ${CWS_TOMCAT_ROOT}/conf/context.xml diff --git a/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java b/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java index 98e76b8b..b0799ee0 100644 --- a/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java +++ b/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java @@ -169,7 +169,10 @@ public class CwsInstaller { private static String cws_security_filter_class; private static String startup_autoregister_process_defs; private static String cws_token_expiration_hours; + private static String elasticsearch_protocol; + private static String elasticsearch_protocol_init; private static String elasticsearch_host; + private static String elasticsearch_host_init; private static String elasticsearch_port; private static String elasticsearch_use_auth; private static String elasticsearch_username; @@ -1064,6 +1067,50 @@ private static void setupLogstash() { private static void setupElasticsearch() { + // PROMPT USER FOR ELASTICSEARCH PROTOCOL + elasticsearch_protocol = getPreset("elasticsearch_protocol"); + + if (cws_installer_mode.equals("interactive")) { + if (elasticsearch_protocol == null) { + + String read_elasticsearch_protocol = ""; + while (!read_elasticsearch_protocol.toLowerCase().startsWith("https") && + !read_elasticsearch_protocol.toLowerCase().startsWith("http")) { + read_elasticsearch_protocol = readRequiredLine("Enter the Elasticsearch protocol (be sure to use HTTP or HTTPS): ", + "You must enter a protocol"); + } + + elasticsearch_protocol_init = read_elasticsearch_protocol; + elasticsearch_protocol = read_elasticsearch_protocol.toLowerCase(); + if (elasticsearch_protocol.startsWith("https")) { + elasticsearch_protocol = "https"; + } + if (elasticsearch_protocol.startsWith("http")) { + elasticsearch_protocol = "http"; + } + } else { + elasticsearch_protocol = readLine("Enter the Elasticsearch protocol. " + "Default is " + elasticsearch_protocol + ": ", elasticsearch_protocol); + } + } else { + if (elasticsearch_protocol == null) { + bailOutMissingOption("elasticsearch_protocol"); + } + + elasticsearch_protocol_init = elasticsearch_protocol; + elasticsearch_protocol = elasticsearch_protocol.toLowerCase(); + if (elasticsearch_protocol.startsWith("https")) { + elasticsearch_protocol = "https"; + } else if (elasticsearch_protocol.startsWith("http")) { + elasticsearch_protocol = "http"; + } else { + bailOutWithMessage("ERROR: elasticsearch_protocol config input is '" + elasticsearch_protocol + + "' ... Be sure to use 'HTTP' or 'HTTPS' for elasticsearch_protocol configuration."); + } + } + + log.debug("elasticsearch_protocol: " + elasticsearch_protocol); + + // PROMPT USER FOR ELASTICSEARCH HOST elasticsearch_host = getPreset("elasticsearch_host"); @@ -1071,26 +1118,34 @@ private static void setupElasticsearch() { if (elasticsearch_host == null) { String read_elasticsearch_host = ""; - - while (!read_elasticsearch_host.startsWith("https://") && - !read_elasticsearch_host.startsWith("http://")) { - read_elasticsearch_host = readRequiredLine("Enter the Elasticsearch host (be sure to include protocol in URL: http:// or https://): ", + read_elasticsearch_host = readRequiredLine("Enter the Elasticsearch host: ", "You must enter a hostname"); - } + elasticsearch_host_init = read_elasticsearch_host; elasticsearch_host = read_elasticsearch_host.toLowerCase(); + if (elasticsearch_host.startsWith("http:/") || elasticsearch_host.startsWith("http://") || + elasticsearch_host.startsWith("https:/") || elasticsearch_host.startsWith("https://")) { + elasticsearch_host = elasticsearch_host.replaceAll("http://", "").replaceAll("http:/","").replaceAll("https://","").replaceAll("https:/",""); + } } else { - elasticsearch_host = readLine("Enter the Elasticsearch host. " + - "Default is " + elasticsearch_host + ": ", elasticsearch_host); + elasticsearch_host = readLine("Enter the Elasticsearch host. " + "Default is " + elasticsearch_host + ": ", elasticsearch_host); } } else { if (elasticsearch_host == null) { bailOutMissingOption("elasticsearch_host"); } + + elasticsearch_host_init = elasticsearch_host; + elasticsearch_host = elasticsearch_host.toLowerCase(); + if (elasticsearch_host.startsWith("http:/") || elasticsearch_host.startsWith("http://") || + elasticsearch_host.startsWith("https:/") || elasticsearch_host.startsWith("https://")) { + elasticsearch_host = elasticsearch_host.replaceAll("http://", "").replaceAll("http:/","").replaceAll("https://","").replaceAll("https:/",""); + } } log.debug("elasticsearch_host: " + elasticsearch_host); + // PROMPT USER FOR ELASTICSEARCH PORT elasticsearch_port = getPreset("elasticsearch_port"); @@ -1489,7 +1544,8 @@ private static void showInstallationInfo() { print("SMTP host = " + cws_smtp_hostname); print("SMTP port = " + cws_smtp_port); print("...................................................................................."); - print("Elasticsearch URL = " + elasticsearch_host); + print("Elasticsearch Protocol = " + elasticsearch_protocol); + print("Elasticsearch Host = " + elasticsearch_host); print("Elasticsearch Port = " + elasticsearch_port); if (elasticsearch_use_auth.equalsIgnoreCase("Y")) { print("Elasticsearch User = " + elasticsearch_username); @@ -1854,14 +1910,32 @@ else if (Integer.valueOf(cws_amq_port) < 1024) { * */ private static int validateElasticsearch() { - print("checking that user provided Elasticsearch (" + elasticsearch_host + ":" + elasticsearch_port + ") is running..."); + print("checking that user provided Elasticsearch (" + elasticsearch_protocol + "://" + elasticsearch_host + ":" + elasticsearch_port + ") is running..."); try { - String[] cmdArray = new String[] {"curl", "--fail", elasticsearch_host + ":" + elasticsearch_port + "/_cluster/health"}; + if (!(elasticsearch_protocol.startsWith("http") || elasticsearch_protocol.startsWith("https")) ) { + print(" [WARNING]"); + print(" It was determined that the user provided Elasticsearch endpoint protocol '" + elasticsearch_protocol + "' did not properly set or protocol to 'HTTP' OR 'HTTPS'"); + print(""); + return 1; + } + + if (elasticsearch_protocol == "http" && elasticsearch_host_init.toLowerCase().startsWith("https") || + elasticsearch_protocol == "https" && elasticsearch_host_init.toLowerCase().startsWith("http")) { + print(" [SETUP RESOLUTION]"); + print(" It was determined that the user provided elasticsearch_protocol and elasticsearch_host have mismatched protocol identifiers."); + print(" elasticsearch_protocol=" + elasticsearch_protocol_init + " "); + print(" elasticsearch_host=" + elasticsearch_host_init + " "); + print(""); + print(" CWS Installation will default to using given elasticsearch_protocol value: " + elasticsearch_protocol_init + " "); + print(""); + } + + String[] cmdArray = new String[] {"curl", "--fail", elasticsearch_protocol + "://" + elasticsearch_host + ":" + elasticsearch_port + "/_cluster/health"}; if (elasticsearch_use_auth.equalsIgnoreCase("Y")) { // Add auth to curl - cmdArray = new String[] {"curl", "--fail", "-u", elasticsearch_username + ":" + elasticsearch_password, elasticsearch_host + ":" + elasticsearch_port + "/_cluster/health"}; + cmdArray = new String[] {"curl", "--fail", "-u", elasticsearch_username + ":" + elasticsearch_password, elasticsearch_protocol + "://" + elasticsearch_host + ":" + elasticsearch_port + "/_cluster/health"}; } Process p = Runtime.getRuntime().exec(cmdArray); @@ -1873,19 +1947,20 @@ private static int validateElasticsearch() { if (p.exitValue() != 0) { print(" [WARNING]"); print(" It was determined that the user provided Elasticsearch is not running or is inaccessible."); + print(" ........................................................................................."); + print(" [ELASTICSEARCH]: Configuration Details"); + print(" elasticsearch_protocol=" + elasticsearch_protocol_init + " "); + print(" elasticsearch_host=" + elasticsearch_host_init + " "); + print(" elasticsearch_port=" + elasticsearch_port + " "); + print(" ........................................................................................."); print(""); - return 1; - } - if (!(elasticsearch_host.startsWith("https://") || elasticsearch_host.startsWith("http://")) ) { - print(" [WARNING]"); - print(" It was determined that the user provided Elasticsearch endpoint '" + elasticsearch_host + "' did not properly set or include protocol 'http://' OR 'https://'"); - print(""); return 1; } print(" [OK]"); print(""); + return 0; // no warnings } catch (Exception e) { @@ -2277,6 +2352,7 @@ private static void updateCwsUiProperties() throws IOException { content = content.replace("__CWS_DB_USERNAME__", cws_db_username); content = content.replace("__CWS_DB_PASSWORD__", cws_db_password); content = content.replace("__CWS_CONSOLE_SSL_PORT__", cws_console_ssl_port); + content = content.replace("__CWS_ES_PROTOCOL__", elasticsearch_protocol); content = content.replace("__CWS_ES_HOST__", elasticsearch_host); content = content.replace("__CWS_ES_PORT__", elasticsearch_port); content = content.replace("__CWS_ES_USE_AUTH__", elasticsearch_use_auth); @@ -2365,6 +2441,7 @@ private static void updateCwsUiConfig() throws IOException { // Update clean_es_history.sh file path = Paths.get(config_work_dir + SEP + "clean_es_history.sh"); content = getFileContents(path); + content = content.replace("__ES_PROTOCOL__", elasticsearch_protocol); content = content.replace("__ES_HOST__", elasticsearch_host); content = content.replace("__ES_PORT__", elasticsearch_port); content = content.replace("__ES_USE_AUTH__", elasticsearch_use_auth); @@ -2507,6 +2584,7 @@ private static void installLogstash() throws IOException { catalinaLogPath = catalinaLogPath.replace("\\", "/"); logstashContent = logstashContent.replace("__CWS_CATALINA_OUT_PATH__", catalinaLogPath); + logstashContent = logstashContent.replace("__CWS_ES_PROTOCOL__", elasticsearch_protocol); logstashContent = logstashContent.replace("__CWS_ES_HOST__", elasticsearch_host); logstashContent = logstashContent.replace("__CWS_ES_PORT__", elasticsearch_port); if (elasticsearch_use_auth.equalsIgnoreCase(("Y"))) { @@ -2575,6 +2653,7 @@ private static void writeOutConfigurationFile() { setPreset("metrics_publishing_interval", metrics_publishing_interval); setPreset("cws_notification_emails", cws_notification_emails); setPreset("cws_token_expiration_hours", cws_token_expiration_hours); + setPreset("elasticsearch_protocol", elasticsearch_protocol); setPreset("elasticsearch_host", elasticsearch_host); setPreset("elasticsearch_port", elasticsearch_port); setPreset("elasticsearch_use_auth", elasticsearch_use_auth); diff --git a/cws-service/src/main/java/jpl/cws/controller/RestService.java b/cws-service/src/main/java/jpl/cws/controller/RestService.java index 48c8dda6..9c2474bf 100644 --- a/cws-service/src/main/java/jpl/cws/controller/RestService.java +++ b/cws-service/src/main/java/jpl/cws/controller/RestService.java @@ -108,7 +108,8 @@ public class RestService extends MvcCore { @Value("${cws.console.app.root}") private String appRoot; @Value("${cws.install.hostname}") private String hostName; - + + @Value("${cws.elasticsearch.protocol}") private String elasticsearchProtocolName; @Value("${cws.elasticsearch.hostname}") private String elasticsearchHostname; @Value("${cws.elasticsearch.port}") private String elasticsearchPort; @@ -405,7 +406,7 @@ private String doDeployProcessDefinition(MultipartFile file) { * @return fully constructed elasticsearch URL string */ private String constructElasticsearchUrl(String subPath) { - String urlString = elasticsearchHostname + ":" + elasticsearchPort + subPath; + String urlString = elasticsearchProtocolName + "://" + elasticsearchHostname + ":" + elasticsearchPort + subPath; return urlString; } diff --git a/dev.sh b/dev.sh index 15b2b9fe..e8ea1af1 100755 --- a/dev.sh +++ b/dev.sh @@ -12,20 +12,21 @@ DB_PORT=${5} DB_NAME=${6} DB_USER=${7} DB_PASS=${8} -ES_HOST=${9} -ES_PORT=${10} -ES_USE_AUTH=${11} -ES_USERNAME=${12} -ES_PASSWORD=${13} -ENABLE_CLOUD_AS=${14} -SECURITY_SCHEME=${15} -THIS_HOSTNAME=${16} -NOTIFICATION_EMAILS=${17} -ADMIN_FIRSTNAME=${18} -ADMIN_LASTNAME=${19} -ADMIN_EMAIL=${20} -NUM_WORKERS=${21} -WORKER_ABANDONED_DAYS=${22} +ES_PROTOCOL=${9} +ES_HOST=${10} +ES_PORT=${11} +ES_USE_AUTH=${12} +ES_USERNAME=${13} +ES_PASSWORD=${14} +ENABLE_CLOUD_AS=${15} +SECURITY_SCHEME=${16} +THIS_HOSTNAME=${17} +NOTIFICATION_EMAILS=${18} +ADMIN_FIRSTNAME=${19} +ADMIN_LASTNAME=${20} +ADMIN_EMAIL=${21} +NUM_WORKERS=${22} +WORKER_ABANDONED_DAYS=${23} source ${ROOT}/utils.sh diff --git a/install/clean_es_history.sh b/install/clean_es_history.sh index 31704860..dbd6ecc6 100755 --- a/install/clean_es_history.sh +++ b/install/clean_es_history.sh @@ -9,6 +9,7 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source ${ROOT}/utils.sh +ES_PROTOCOL='__ES_PROTOCOL__' ES_HOST='__ES_HOST__' ES_PORT='__ES_PORT__' ES_USE_AUTH='__ES_USE_AUTH__' diff --git a/install/cws-ui/cws-ui.properties b/install/cws-ui/cws-ui.properties index f1d14e0c..309b0f17 100644 --- a/install/cws-ui/cws-ui.properties +++ b/install/cws-ui/cws-ui.properties @@ -16,6 +16,7 @@ cws.db.port=__CWS_DB_PORT__ cws.db.username=__CWS_DB_USERNAME__ cws.db.password=__CWS_DB_PASSWORD__ +cws.elasticsearch.protocol=__CWS_ES_PROTOCOL__ cws.elasticsearch.hostname=__CWS_ES_HOST__ cws.elasticsearch.port=__CWS_ES_PORT__ cws.elasticsearch.use.auth=__CWS_ES_USE_AUTH__ diff --git a/install/docker/worker-ls/cws-logstash.conf b/install/docker/worker-ls/cws-logstash.conf index 8f588e57..07dfda04 100644 --- a/install/docker/worker-ls/cws-logstash.conf +++ b/install/docker/worker-ls/cws-logstash.conf @@ -53,5 +53,5 @@ filter { } output { - elasticsearch { hosts => ["__ES_HOST__:9200"] } + elasticsearch { hosts => ["__ES_PROTOCOL__://__ES_HOST__:9200"] } } diff --git a/install/docker/worker-ls/docker-compose.yml b/install/docker/worker-ls/docker-compose.yml index 7254817b..2527dea0 100644 --- a/install/docker/worker-ls/docker-compose.yml +++ b/install/docker/worker-ls/docker-compose.yml @@ -39,6 +39,7 @@ services: - DB_HOST=__DB_HOST__ - DB_USER=root - DB_PW=test + - ES_PROTOCOL=__ES_PROTOCOL__ - ES_HOST=__ES_HOST__ - ES_PORT=9200 volumes: diff --git a/install/example-cws-configuration.properties b/install/example-cws-configuration.properties index fe9c8901..688d5efe 100644 --- a/install/example-cws-configuration.properties +++ b/install/example-cws-configuration.properties @@ -53,7 +53,7 @@ database_password=[YourPassword] # The LDAP username for the user that will be the CWS Administrator. # This User will have the initial permissions in CWS, and will have the ability # to add users and modify permissions. -# Example: ghollins +# Example: jsmith admin_user=[YourJPLUsername] # The first and last name, and email address of the CWS Administrator. @@ -116,6 +116,13 @@ cws_notification_emails=[AdminEmailAddresses(comma separated)] # This allows CWS to use a user provided Elasticsearch service. # If you wish to use an unsecured Elasticsearch host, you may do so # by specifying elasticsearch_use_auth=n below. +# Also specify if your provided Elasticsearch is encrypted(HTTPS) or non-encrypted(HTTP) +# by setting elasticsearch_protocol to HTTPS or HTTP. +# Examples: elasticsearch_protocol=http +# or elasticsearch_protocol=https +# or elasticsearch_protocol=HTTP +# elasticsearch_protocol=HTTPS +elasticsearch_protocol=[YourElasticsearchProtocol] elasticsearch_host=[YourElasticsearchHost] elasticsearch_port=9200 elasticsearch_use_auth=n diff --git a/install/logging/cws-logstash.conf b/install/logging/cws-logstash.conf index d759ac5c..55c37d8c 100644 --- a/install/logging/cws-logstash.conf +++ b/install/logging/cws-logstash.conf @@ -55,7 +55,7 @@ filter { output { elasticsearch { - hosts => ["__CWS_ES_HOST__:__CWS_ES_PORT__"] + hosts => ["__CWS_ES_PROTOCOL____CWS_ES_HOST__:__CWS_ES_PORT__"] ilm_enabled => false ssl => __LOGSTASH_ES_USE_SSL__ __LOGSTASH_ES_USERNAME__ diff --git a/utils.sh b/utils.sh index dbb74d27..7d053e0e 100644 --- a/utils.sh +++ b/utils.sh @@ -189,22 +189,23 @@ function auto_conf_data () { DB_NAME=${7} DB_USER=${8} DB_PASS=${9} - ES_HOST=${10} - ES_PORT=${11} - ES_USE_AUTH=${12} - ES_USERNAME=${13} - ES_PASSWORD=${14} - ENABLE_CLOUD_AS=${15} - SECURITY_SCHEME=${16} - THIS_HOSTNAME=${17} - NOTIFICATION_EMAILS=${18} - ADMIN_FIRSTNAME=${19} - ADMIN_LASTNAME=${20} - ADMIN_EMAIL=${21} - NUM_WORKERS=${22} - WORKER_ABANDONED_DAYS=${23} - - OUTPUT_FILE=${24} + ES_PROTOCOL=${10} + ES_HOST=${11} + ES_PORT=${12} + ES_USE_AUTH=${13} + ES_USERNAME=${14} + ES_PASSWORD=${15} + ENABLE_CLOUD_AS=${16} + SECURITY_SCHEME=${17} + THIS_HOSTNAME=${18} + NOTIFICATION_EMAILS=${19} + ADMIN_FIRSTNAME=${20} + ADMIN_LASTNAME=${21} + ADMIN_EMAIL=${22} + NUM_WORKERS=${23} + WORKER_ABANDONED_DAYS=${24} + + OUTPUT_FILE=${25} source ${ROOT}/utils.sh @@ -275,6 +276,7 @@ function auto_conf_data () { database_name=${DB_NAME} database_username=${DB_USER} database_password=${DB_PASS} + elasticsearch_protocol=${ES_PROTOCOL} elasticsearch_host=${ES_HOST} elasticsearch_port=${ES_PORT} elasticsearch_use_auth=${ES_USE_AUTH}