Implement ping command

Why these changes are being introduced:
Pinging the OpenSearch instance and printing information about it is the
first step towards implementing real functionality in this app.

How this addresses that need:
* Adds an opensearch module that configures an OpenSearch client for
  either local or AWS hosted OpenSearch.
* Updates the main cli command to take a url option, pulled from env if
  not provided and with a localhost default, to set the OpenSearch
  instance url.
* Updates the main cli command to configure the OpenSearch client using
  the provided or default url.
* Updates the ping command to get and log basic info about the
  OpenSearch cluster.
* Adds tests for all new functionality.
* Updates README with information on running and connecting to a local
  OpenSearch instance with Docker or connecting to an instance on AWS.
* Updates mypy and isort configurations.
* Updates Python version and dependencies.

Relevant ticket(s):
* https://mitlibraries.atlassian.net/browse/TIMX-72

.python-version

@@ -1 +1 @@
-3.10.3
+3.10.6

Pipfile

@@ -4,7 +4,9 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
+boto3 = "*"
 click = "*"
+opensearch-py = "*"
 sentry-sdk = "*"
 
 [dev-packages]
@@ -15,6 +17,7 @@ coveralls = "*"
 mypy = "*"
 pylama = {extras = ["all"], version = "*"}
 pytest = "*"
+vcrpy = "*"
 
 [requires]
 python_version = "3.10"

README.md

@@ -1,6 +1,12 @@
 # timdex-index-manager (tim)
 
-TIMDEX! Index Manager (TIM) is a Python cli application for managing TIMDEX indexes in Opensearch.
+TIMDEX! Index Manager (TIM) is a Python cli application for managing TIMDEX indexes in OpenSearch.
+
+## Required ENV
+
+- `OPENSEARCH_ENDPOINT` = Optional (can also be passed directly to the CLI via the `--url` option). If using a local Docker OpenSearch instance, this isn't needed. Otherwise set to OpenSearch instance endpoint _without_ the http scheme, e.g. `search-timdex-env-1234567890.us-east-1.es.amazonaws.com`
+- `SENTRY_DSN` = If set to a valid Sentry DSN, enables Sentry exception monitoring. This is not needed for local development.
+- `WORKSPACE` = Set to `dev` for local development, this will be set to `stage` and `prod` in those environments by Terraform.
 
 ## Development
 
@@ -10,7 +16,20 @@ TIMDEX! Index Manager (TIM) is a Python cli application for managing TIMDEX inde
 - To lint the repo: `make lint`
 - To run the app: `pipenv run tim --help`
 
-## Required ENV
+### Local OpenSearch with Docker
 
-- `SENTRY_DSN` = If set to a valid Sentry DSN, enables Sentry exception monitoring. This is not needed for local development.
-- `WORKSPACE` = Set to `dev` for local development, this will be set to `stage` and `prod` in those environments by Terraform.
+A local OpenSearch instance can be started for development purposes by running:
+
+``` bash
+$ docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" \
+  -e "plugins.security.disabled=true" \
+  opensearchproject/opensearch:1.3.3
+```
+
+To confirm the instance is up, run `pipenv run tim -u localhost ping`.
+
+### OpenSearch on AWS
+
+1. Ensure that you have the correct AWS credentials set for the Dev1 (or desired) account.
+2. Set the `OPENSEARCH_ENDPOINT` variable in your .env to match the Dev1 (or desired) TIMDEX OpenSearch endpoint (note: do not include the http scheme prefix).
+3. Run `pipenv run tim ping` to confirm the client is connected to the expected TIMDEX OpenSearch instance.

setup.cfg

@@ -1,8 +1,10 @@
 [mypy]
 disallow_incomplete_defs = True
-disallow_untyped_calls = True
 disallow_untyped_defs = True
 
+[mypy-boto3.*]
+ignore_missing_imports = True
+
 [mypy-sentry_sdk.*]
 ignore_missing_imports = True
 
@@ -11,6 +13,9 @@ ignore = C0114,C0116,D100,D103,W0012
 linters = eradicate,isort,mccabe,pycodestyle,pydocstyle,pyflakes,pylint
 max_line_length = 90
 
+[pylama:isort]
+profile = black
+
 [pylama:pydocstyle]
 convention = pep257
 

tests/conftest.py

@@ -6,7 +6,14 @@
 
 @pytest.fixture(autouse=True)
 def test_env():
-    os.environ = {"SENTRY_DSN": None, "WORKSPACE": "test"}
+    os.environ = {
+        "AWS_ACCESS_KEY_ID": "test",
+        "AWS_SECRET_ACCESS_KEY": "test",
+        "AWS_SESSION_TOKEN": "test",
+        "OPENSEARCH_ENDPOINT": "localhost",
+        "SENTRY_DSN": None,
+        "WORKSPACE": "test",
+    }
     yield
 
 

tests/fixtures/cassettes/ping_localhost.yaml

@@ -0,0 +1,29 @@
+interactions:
+  - request:
+      body: null
+      headers:
+        content-type:
+          - application/json
+        user-agent:
+          - opensearch-py/2.0.0 (Python 3.10.3)
+      method: GET
+      uri: http://localhost:9200/
+    response:
+      body:
+        string:
+          "{\n  \"name\" : \"6370d6be26b2\",\n  \"cluster_name\" : \"docker-cluster\",\n
+          \ \"cluster_uuid\" : \"j7tpRLtKTsSRlyng3RELug\",\n  \"version\" : {\n    \"distribution\"
+          : \"opensearch\",\n    \"number\" : \"1.3.3\",\n    \"build_type\" : \"tar\",\n
+          \   \"build_hash\" : \"6d63fc707f6010c5980ec5cb51324856caf1f03d\",\n    \"build_date\"
+          : \"2022-06-07T15:24:06.598626Z\",\n    \"build_snapshot\" : false,\n    \"lucene_version\"
+          : \"8.10.1\",\n    \"minimum_wire_compatibility_version\" : \"6.8.0\",\n    \"minimum_index_compatibility_version\"
+          : \"6.0.0-beta1\"\n  },\n  \"tagline\" : \"The OpenSearch Project: https://opensearch.org/\"\n}\n"
+      headers:
+        content-length:
+          - "569"
+        content-type:
+          - application/json; charset=UTF-8
+      status:
+        code: 200
+        message: OK
+version: 1

tests/test_cli.py

@@ -1,21 +1,40 @@
+import vcr
+
 from tim.cli import main
 
 
+@vcr.use_cassette("tests/fixtures/cassettes/ping_localhost.yaml")
 def test_main_group_no_options_configures_correctly_and_invokes_result_callback(
-    caplog, runner
+    caplog, monkeypatch, runner
 ):
+    monkeypatch.delenv("OPENSEARCH_ENDPOINT", raising=False)
     result = runner.invoke(main, ["ping"])
     assert result.exit_code == 0
     assert "Logger 'root' configured with level=INFO" in caplog.text
+    assert "OpenSearch client configured for endpoint 'localhost'" in caplog.text
     assert "Total time to complete process" in caplog.text
 
 
+@vcr.use_cassette("tests/fixtures/cassettes/ping_localhost.yaml")
 def test_main_group_all_options_configures_correctly_and_invokes_result_callback(
-    caplog, runner
+    caplog, monkeypatch, runner
 ):
-    result = runner.invoke(main, ["--verbose", "ping"])
+    monkeypatch.delenv("OPENSEARCH_ENDPOINT", raising=False)
+    result = runner.invoke(main, ["--verbose", "--url", "localhost", "ping"])
     assert result.exit_code == 0
     assert "Logger 'root' configured with level=DEBUG" in caplog.text
+    assert "OpenSearch client configured for endpoint 'localhost'" in caplog.text
+    assert "Total time to complete process" in caplog.text
+
+
+@vcr.use_cassette("tests/fixtures/cassettes/ping_localhost.yaml")
+def test_main_group_options_from_env_configures_correctly_and_invokes_result_callback(
+    caplog, runner
+):
+    result = runner.invoke(main, ["ping"])
+    assert result.exit_code == 0
+    assert "Logger 'root' configured with level=INFO" in caplog.text
+    assert "OpenSearch client configured for endpoint 'localhost'" in caplog.text
     assert "Total time to complete process" in caplog.text
 
 
@@ -31,10 +50,11 @@ def test_indexes(caplog, runner):
     assert "'indexes' command not yet implemented" in caplog.text
 
 
+@vcr.use_cassette("tests/fixtures/cassettes/ping_localhost.yaml")
 def test_ping(caplog, runner):
     result = runner.invoke(main, ["ping"])
     assert result.exit_code == 0
-    assert "'ping' command not yet implemented" in caplog.text
+    assert "'cluster_name': 'docker-cluster'" in caplog.text
 
 
 def test_ingest_no_options(caplog, runner):

tests/test_opensearch.py

@@ -0,0 +1,17 @@
+from unittest import mock
+
+from tim.opensearch import configure_opensearch_client
+
+
+def test_configure_opensearch_client_for_localhost():
+    result = configure_opensearch_client("localhost")
+    assert str(result) == "<OpenSearch([{'host': 'localhost', 'port': '9200'}])>"
+
+
+@mock.patch("boto3.session.Session")
+def test_configure_opensearch_client_for_aws(mocked_boto3_session):  # noqa
+    result = configure_opensearch_client("fake-dev.us-east-1.es.amazonaws.com")
+    assert (
+        str(result) == "<OpenSearch([{'host': 'fake-dev.us-east-1.es.amazonaws.com', "
+        "'port': '443'}])>"
+    )

tim/cli.py

@@ -1,26 +1,40 @@
 import logging
+import pprint
 from datetime import timedelta
 from time import perf_counter
 from typing import Optional
 
 import click
 
 from tim.config import configure_logger, configure_sentry
+from tim.opensearch import configure_opensearch_client
 
 logger = logging.getLogger(__name__)
 
 
 @click.group()
+@click.option(
+    "-u",
+    "--url",
+    envvar="OPENSEARCH_ENDPOINT",
+    default="localhost",
+    help="The OpenSearch instance endpoint minus the http scheme, e.g. "
+    "'search-timdex-env-1234567890.us-east-1.es.amazonaws.com'. If not provided, will "
+    "attempt to get from the OPENSEARCH_ENDPOINT environment variable. Defaults to "
+    "'localhost'.",
+)
 @click.option(
     "-v", "--verbose", is_flag=True, help="Pass to log at debug level instead of info"
 )
 @click.pass_context
-def main(ctx: click.Context, verbose: bool) -> None:
+def main(ctx: click.Context, url: str, verbose: bool) -> None:
     ctx.ensure_object(dict)
     ctx.obj["START_TIME"] = perf_counter()
     root_logger = logging.getLogger()
     logger.info(configure_logger(root_logger, verbose))
     logger.info(configure_sentry())
+    ctx.obj["CLIENT"] = configure_opensearch_client(url)
+    logger.info("OpenSearch client configured for endpoint '%s'", url)
 
 
 @main.result_callback()
@@ -60,13 +74,12 @@ def indexes() -> None:
 
 
 @main.command()
-def ping() -> None:
-    """
-    Ping OpenSearch.
-
-    Display the following: name, cluster, version, Lucene version.
-    """
-    logger.info("'ping' command not yet implemented")
+@click.pass_context
+def ping(ctx: click.Context) -> None:
+    """Ping OpenSearch and display information about the cluster."""
+    client = ctx.obj["CLIENT"]
+    output = pprint.pformat(client.info(), indent=2)
+    logger.info(output)
 
 
 # Index commands

tim/opensearch.py

@@ -0,0 +1,24 @@
+import os
+
+import boto3
+from opensearchpy import AWSV4SignerAuth, OpenSearch, RequestsHttpConnection
+
+
+def configure_opensearch_client(url: str) -> OpenSearch:
+    if url == "localhost":
+        return OpenSearch(
+            hosts=[{"host": url, "port": "9200"}],
+            http_auth=("admin", "admin"),
+            connection_class=RequestsHttpConnection,
+        )
+
+    credentials = boto3.Session().get_credentials()
+    region = os.getenv("AWS_REGION", "us-east-1")
+    auth = AWSV4SignerAuth(credentials, region)
+    return OpenSearch(
+        hosts=[{"host": url, "port": "443"}],
+        http_auth=auth,
+        use_ssl=True,
+        verify_certs=True,
+        connection_class=RequestsHttpConnection,
+    )