From 9787251f75782a5840bf00c0723af962842d66b8 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Thu, 4 Jan 2024 18:19:28 -0500 Subject: [PATCH 1/2] fix issue where elastic doesn't parse username correctly --- elastic_datashader/elastic.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/elastic_datashader/elastic.py b/elastic_datashader/elastic.py index 43d1a1c..7033332 100644 --- a/elastic_datashader/elastic.py +++ b/elastic_datashader/elastic.py @@ -3,12 +3,14 @@ import copy import struct import time +import urllib from dateutil.relativedelta import relativedelta from datashader.utils import lnglat_to_meters from elasticsearch import Elasticsearch from elasticsearch_dsl import AttrDict, Search +import elastic_transport import pynumeral import yaml @@ -45,6 +47,20 @@ def scan(search, use_scroll=False, size=10000): else: _search = None +def hosts_url_to_nodeconfig(elasticsearch_hosts: str): + node_configs = [] + for host in elasticsearch_hosts.split(","): + nodeconfig = elastic_transport.client_utils.url_to_node_config(host) + nodeconfig.verify_certs = False + # check if host has username and password and override the basic auth due to elastic bug + # https://github.com/elastic/elastic-transport-python/issues/141 + parsed_url = urllib.parse.urlparse(host) + if parsed_url.username and parsed_url.password: + nodeconfig.headers = nodeconfig.headers.copy() + nodeconfig.headers["authorization"] = elastic_transport.client_utils.basic_auth_to_header((parsed_url.username, parsed_url.password)) + node_configs.append(nodeconfig) + print(node_configs) + return node_configs def verify_datashader_indices(elasticsearch_hosts: str): """Verify the ES indices exist @@ -52,8 +68,7 @@ def verify_datashader_indices(elasticsearch_hosts: str): :param elasticsearch_hosts: """ es = Elasticsearch( - elasticsearch_hosts.split(","), - verify_certs=False, + hosts_url_to_nodeconfig(elasticsearch_hosts), timeout=120 ) @@ -182,8 +197,7 @@ def get_search_base( x_opaque_id = params.get("x-opaque-id") # Connect to Elasticsearch es = Elasticsearch( - elastic_hosts.split(","), - verify_certs=False, + hosts_url_to_nodeconfig(elastic_hosts), timeout=900, headers=get_es_headers(headers, user, x_opaque_id), ) From baf0c0de5ca0cfa10528c5f172cdaa4a5e74af51 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Thu, 4 Jan 2024 18:26:26 -0500 Subject: [PATCH 2/2] remove print --- elastic_datashader/elastic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elastic_datashader/elastic.py b/elastic_datashader/elastic.py index 7033332..cf0dbfc 100644 --- a/elastic_datashader/elastic.py +++ b/elastic_datashader/elastic.py @@ -59,7 +59,6 @@ def hosts_url_to_nodeconfig(elasticsearch_hosts: str): nodeconfig.headers = nodeconfig.headers.copy() nodeconfig.headers["authorization"] = elastic_transport.client_utils.basic_auth_to_header((parsed_url.username, parsed_url.password)) node_configs.append(nodeconfig) - print(node_configs) return node_configs def verify_datashader_indices(elasticsearch_hosts: str):