Skip to content

Commit

Permalink
work in progress on generalizing opensearch connection params
Browse files Browse the repository at this point in the history
  • Loading branch information
@mmguero
mmguero committed Aug 18, 2022
1 parent e7f2227 commit 83ff378
Show file tree
Hide file tree
Showing 5 changed files with 196 additions and 68 deletions.
4 changes: 3 additions & 1 deletion api/project/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,9 @@
opensearchLocal = (app.config["OPENSEARCH_LOCAL"] == "true") or (opensearchUrl == 'http://opensearch:9200')
opensearchSslVerify = app.config["OPENSEARCH_SSL_CERTIFICATE_VERIFICATION"] == "true"
opensearchCreds = (
malcolm_common.ParseCurlFile(app.config["OPENSEARCH_URL"]) if (not opensearchLocal) else defaultdict(lambda: None)
malcolm_common.ParseCurlFile(app.config["OPENSEARCH_CREDS_CONFIG_FILE"])
if (not opensearchLocal)
else defaultdict(lambda: None)
)
if opensearchCreds['user'] is not None:
opensearchDslHttpAuth = f"{opensearchCreds['user']}:{opensearchCreds['password']}"
Expand Down
68 changes: 59 additions & 9 deletions dashboards/scripts/index-refresh.py
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# TODO: handle OPENSEARCH_URL, OPENSEARCH_LOCAL, OPENSEARCH_SSL_CERTIFICATE_VERIFICATION, OPENSEARCH_CREDS_CONFIG_FILE

import argparse
import json
import malcolm_common
import re
import requests
import os
import sys
import urllib3

from collections import defaultdict

GET_STATUS_API = 'api/status'
GET_INDEX_PATTERN_INFO_URI = 'api/saved_objects/_find'
GET_FIELDS_URI = 'api/index_patterns/_fields_for_wildcard'
Expand Down Expand Up @@ -77,9 +78,36 @@ def main():
dest='opensearchUrl',
metavar='<protocol://host:port>',
type=str,
default=os.getenv('OPENSEARCH_URL', 'http://opensearch:9200'),
default=os.getenv('OPENSEARCH_URL', None),
help='OpenSearch URL',
)
parser.add_argument(
'-c',
'--opensearch-curlrc',
dest='opensearchCurlRcFile',
metavar='<filename>',
type=str,
default=os.getenv('OPENSEARCH_CREDS_CONFIG_FILE', '/var/local/opensearch.curlrc'),
help='cURL.rc formatted file containing OpenSearch connection parameters',
)
parser.add_argument(
'--opensearch-ssl-verify',
dest='opensearchSslVerify',
type=str2bool,
nargs='?',
const=True,
default=str2bool(os.getenv('OPENSEARCH_SSL_CERTIFICATE_VERIFICATION', default='False')),
help="Verify SSL certificates for OpenSearch",
)
parser.add_argument(
'--opensearch-local',
dest='opensearchIsLocal',
type=str2bool,
nargs='?',
const=True,
default=str2bool(os.getenv('OPENSEARCH_LOCAL', default='True')),
help="Malcolm is using its local OpenSearch instance",
)
parser.add_argument(
'-t',
'--template',
Expand Down Expand Up @@ -117,6 +145,23 @@ def main():
else:
sys.tracebacklimit = 0

args.opensearchIsLocal = args.opensearchIsLocal or (args.opensearchUrl == 'http://opensearch:9200')
opensearchCreds = (
malcolm_common.ParseCurlFile(args.opensearchCurlRcFile)
if (not args.opensearchIsLocal)
else defaultdict(lambda: None)
)
if not args.opensearchUrl:
if args.opensearchIsLocal:
args.opensearchUrl = 'http://opensearch:9200'
elif 'url' in opensearchCreds:
args.opensearchUrl = opensearchCreds['url']
opensearchReqHttpAuth = (
HTTPBasicAuth(opensearchCreds['user'], opensearchCreds['password'])
if opensearchCreds['user'] is not None
else None
)

# get version number so Dashboards doesn't think we're doing a XSRF when we do the PUT
statusInfoResponse = requests.get(
'{}/{}'.format(args.dashboardsUrl, GET_STATUS_API),
Expand All @@ -130,7 +175,8 @@ def main():

opensearchInfoResponse = requests.get(
args.opensearchUrl,
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
opensearchInfo = opensearchInfoResponse.json()
opensearchVersion = opensearchInfo['version']['number']
Expand Down Expand Up @@ -168,7 +214,8 @@ def main():
# request template from OpenSearch and pull the mappings/properties (field list) out
getTemplateResponse = requests.get(
'{}/{}/{}'.format(args.opensearchUrl, OS_GET_INDEX_TEMPLATE_URI, args.template),
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
getTemplateResponse.raise_for_status()
getTemplateResponseJson = getTemplateResponse.json()
Expand All @@ -187,7 +234,8 @@ def main():
for componentName in composedOfList:
getComponentResponse = requests.get(
'{}/{}/{}'.format(args.opensearchUrl, OS_GET_COMPONENT_TEMPLATE_URI, componentName),
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
getComponentResponse.raise_for_status()
getComponentResponseJson = getComponentResponse.json()
Expand Down Expand Up @@ -405,12 +453,13 @@ def main():
else:
print("failure (could not find Index ID for {})".format(args.index))

if args.fixUnassigned and not args.dryrun:
if args.opensearchIsLocal and args.fixUnassigned and not args.dryrun:
# set some configuration-related indexes (opensearch/opendistro) replica count to 0
# so we don't have yellow index state on those
shardsResponse = requests.get(
'{}/{}'.format(args.opensearchUrl, GET_SHARDS_URL),
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
for shardLine in shardsResponse.iter_lines():
shardInfo = shardLine.decode('utf-8').split()
Expand All @@ -422,7 +471,8 @@ def main():
'osd-xsrf': 'true',
},
data=json.dumps({'index': {'number_of_replicas': 0}}),
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
putResponse.raise_for_status()

Expand Down
66 changes: 58 additions & 8 deletions shared/bin/opensearch_index_size_prune.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# TODO: handle OPENSEARCH_URL, OPENSEARCH_LOCAL, OPENSEARCH_SSL_CERTIFICATE_VERIFICATION, OPENSEARCH_CREDS_CONFIG_FILE

import argparse
import humanfriendly
import json
import malcolm_common
import re
import requests
import os
import sys
import urllib3

from collections import defaultdict

###################################################################################################
debug = False
scriptName = os.path.basename(__file__)
Expand Down Expand Up @@ -66,9 +67,36 @@ def main():
dest='opensearchUrl',
metavar='<protocol://host:port>',
type=str,
default=os.getenv('OPENSEARCH_URL', 'http://opensearch:9200'),
default=os.getenv('OPENSEARCH_URL', None),
help='OpenSearch URL',
)
parser.add_argument(
'-c',
'--opensearch-curlrc',
dest='opensearchCurlRcFile',
metavar='<filename>',
type=str,
default=os.getenv('OPENSEARCH_CREDS_CONFIG_FILE', '/var/local/opensearch.curlrc'),
help='cURL.rc formatted file containing OpenSearch connection parameters',
)
parser.add_argument(
'--opensearch-ssl-verify',
dest='opensearchSslVerify',
type=str2bool,
nargs='?',
const=True,
default=str2bool(os.getenv('OPENSEARCH_SSL_CERTIFICATE_VERIFICATION', default='False')),
help="Verify SSL certificates for OpenSearch",
)
parser.add_argument(
'--opensearch-local',
dest='opensearchIsLocal',
type=str2bool,
nargs='?',
const=True,
default=str2bool(os.getenv('OPENSEARCH_LOCAL', default='True')),
help="Malcolm is using its local OpenSearch instance",
)
parser.add_argument(
'--node',
dest='node',
Expand Down Expand Up @@ -134,9 +162,27 @@ def main():
if args.limit == '0':
return

args.opensearchIsLocal = args.opensearchIsLocal or (args.opensearchUrl == 'http://opensearch:9200')
opensearchCreds = (
malcolm_common.ParseCurlFile(args.opensearchCurlRcFile)
if (not args.opensearchIsLocal)
else defaultdict(lambda: None)
)
if not args.opensearchUrl:
if args.opensearchIsLocal:
args.opensearchUrl = 'http://opensearch:9200'
elif 'url' in opensearchCreds:
args.opensearchUrl = opensearchCreds['url']
opensearchReqHttpAuth = (
HTTPBasicAuth(opensearchCreds['user'], opensearchCreds['password'])
if opensearchCreds['user'] is not None
else None
)

osInfoResponse = requests.get(
args.opensearchUrl,
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
osInfo = osInfoResponse.json()
opensearchVersion = osInfo['version']['number']
Expand Down Expand Up @@ -167,7 +213,8 @@ def main():
esDiskUsageStats = []
osInfoResponse = requests.get(
f'{args.opensearchUrl}/_cat/allocation{f"/{args.node}" if args.node else ""}?format=json',
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
osInfo = osInfoResponse.json()

Expand Down Expand Up @@ -223,7 +270,8 @@ def main():
# now determine the total size of the indices from the index pattern
osInfoResponse = requests.get(
f'{args.opensearchUrl}/{args.index}/_stats/store',
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
osInfo = osInfoResponse.json()
try:
Expand All @@ -250,7 +298,8 @@ def main():
osInfoResponse = requests.get(
f'{args.opensearchUrl}/_cat/indices/{args.index}',
params={'format': 'json', 'h': 'i,id,status,health,rep,creation.date,pri.store.size,store.size'},
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
osInfo = sorted(osInfoResponse.json(), key=lambda k: k['i' if args.nameSorted else 'creation.date'])

Expand All @@ -277,7 +326,8 @@ def main():
for index in indicesToDelete:
esDeleteResponse = requests.delete(
f'{args.opensearchUrl}/{index["i"]}',
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
print(
f'DELETE {index["i"]} ({humanfriendly.format_size(humanfriendly.parse_size(index[sizeKey]))}): {requests.status_codes._codes[esDeleteResponse.status_code][0]}'
Expand Down
62 changes: 56 additions & 6 deletions shared/bin/opensearch_read_only.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# TODO: handle OPENSEARCH_URL, OPENSEARCH_LOCAL, OPENSEARCH_SSL_CERTIFICATE_VERIFICATION, OPENSEARCH_CREDS_CONFIG_FILE

# Copyright (c) 2022 Battelle Energy Alliance, LLC. All rights reserved.

import argparse
import json
import requests
import malcolm_common
import os
import sys
import urllib3

from collections import defaultdict

###################################################################################################
debug = False
scriptName = os.path.basename(__file__)
Expand Down Expand Up @@ -66,9 +67,38 @@ def main():
dest='opensearchUrl',
metavar='<protocol://host:port>',
type=str,
default=os.getenv('OPENSEARCH_URL', 'http://opensearch:9200'),
default=os.getenv('OPENSEARCH_URL', None),
help='OpenSearch URL',
)
parser.add_argument(
'-c',
'--opensearch-curlrc',
dest='opensearchCurlRcFile',
metavar='<filename>',
type=str,
default=os.getenv('OPENSEARCH_CREDS_CONFIG_FILE', '/var/local/opensearch.curlrc'),
help='cURL.rc formatted file containing OpenSearch connection parameters',
)
parser.add_argument(
'-s',
'--opensearch-ssl-verify',
dest='opensearchSslVerify',
type=str2bool,
nargs='?',
const=True,
default=str2bool(os.getenv('OPENSEARCH_SSL_CERTIFICATE_VERIFICATION', default='False')),
help="Verify SSL certificates for OpenSearch",
)
parser.add_argument(
'-l',
'--opensearch-local',
dest='opensearchIsLocal',
type=str2bool,
nargs='?',
const=True,
default=str2bool(os.getenv('OPENSEARCH_LOCAL', default='True')),
help="Malcolm is using its local OpenSearch instance",
)
parser.add_argument(
'-r',
'--read-only',
Expand Down Expand Up @@ -114,9 +144,27 @@ def main():
else:
sys.tracebacklimit = 0

args.opensearchIsLocal = args.opensearchIsLocal or (args.opensearchUrl == 'http://opensearch:9200')
opensearchCreds = (
malcolm_common.ParseCurlFile(args.opensearchCurlRcFile)
if (not args.opensearchIsLocal)
else defaultdict(lambda: None)
)
if not args.opensearchUrl:
if args.opensearchIsLocal:
args.opensearchUrl = 'http://opensearch:9200'
elif 'url' in opensearchCreds:
args.opensearchUrl = opensearchCreds['url']
opensearchReqHttpAuth = (
HTTPBasicAuth(opensearchCreds['user'], opensearchCreds['password'])
if opensearchCreds['user'] is not None
else None
)

osInfoResponse = requests.get(
args.opensearchUrl,
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
osInfo = osInfoResponse.json()
opensearchVersion = osInfo['version']['number']
Expand Down Expand Up @@ -157,9 +205,10 @@ def main():
# make the PUT request to change the index/cluster setting and raise an exception if it fails
putResponse = requests.put(
settingsUrl,
auth=opensearchReqHttpAuth,
headers={'Content-Type': 'application/json'},
data=json.dumps(settingsInfo),
verify=False,
verify=args.opensearchSslVerify,
)
putResponse.raise_for_status()
if debug:
Expand All @@ -169,7 +218,8 @@ def main():
# request settings to verify change(s)
checkResponse = requests.get(
settingsUrl,
verify=False,
auth=opensearchReqHttpAuth,
verify=args.opensearchSslVerify,
)
if args.index == '_cluster':
eprint(json.dumps(checkResponse.json()))
Expand Down

0 comments on commit 83ff378

Please sign in to comment.