From 8e801b8b68e1081d7430089a0cb66ccde1a43da2 Mon Sep 17 00:00:00 2001 From: Zoltan Miskolci Date: Wed, 13 Jul 2022 15:35:05 -0700 Subject: [PATCH 1/5] Jupyter notebook --- tests/jupyter/notebook_test.py | 117 +++++++++ .../notebooks/harmony_concise_api_test.ipynb | 241 ++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100644 tests/jupyter/notebook_test.py create mode 100644 tests/jupyter/notebooks/harmony_concise_api_test.ipynb diff --git a/tests/jupyter/notebook_test.py b/tests/jupyter/notebook_test.py new file mode 100644 index 00000000..b4c35dd8 --- /dev/null +++ b/tests/jupyter/notebook_test.py @@ -0,0 +1,117 @@ +import cmr +import papermill as pm +import json +import os +import argparse + +def parse_args(): + """ + Parses the program arguments + Returns + ------- + args + """ + + parser = argparse.ArgumentParser( + description='Update CMR with latest profile', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument('-c', '--collections', + help='List of collection test', + required=False, + metavar='', + type=str) + + parser.add_argument('-e', '--env', + help='CMR environment used to pull results from.', + required=True, + choices=["uat", "ops"], + metavar='uat or ops') + + parser.add_argument('-n', '--notebook', + help='Notebook to run', + required=True, + metavar='') + + parser.add_argument('-i', '--input_file', + help='File of json collections', + required=False, + metavar='') + + parser.add_argument('-o', '--output_path', + help='output path for success and fails', + required=False, + metavar='') + + args = parser.parse_args() + return args + +def run(): + """ + Run from command line. + + Returns + ------- + """ + + _args = parse_args() + + collection_json = _args.collections + environment = _args.env + notebook = _args.notebook + input_file = _args.input_file + + if collection_json: + collections = json.loads(collection_json) + if input_file: + with open(_args.input_file) as json_data: + try: + collections = json.load(json_data) + except ValueError: + collections = [] + json_data.seek(0) + lines = json_data.readlines() + for line in lines: + collections.append(line.strip()) + + + notebook = "./notebooks/harmony_concise_api_test.ipynb" + notebook_path = os.path.dirname(notebook) + notebook_name = os.path.basename(notebook) + + success = [] + fails = [] + + venue = "prod" + if environment == "uat": + venue = "uat" + + for collection in collections: + + try: + print(collection) + pm.execute_notebook( + notebook, + "{}/output/{}_{}_output_{}".format(notebook_path, collection, environment, notebook_name), + parameters=dict(collection=collection, venue=venue) + ) + success.append(collection) + except Exception as ex: + print(ex) + fails.append(collection) + + if _args.output_path: + success_outfile = f'{_args.output_path}/{_args.env}_success.txt' + fail_outfile = f'{_args.output_path}/{_args.env}_fail.txt' + + if success: + with open(success_outfile, 'w') as the_file: + the_file.writelines(x + '\n' for x in success) + + if fails: + with open(fail_outfile, 'w') as the_file: + the_file.writelines(x + '\n' for x in fails) + +if __name__ == '__main__': + run() diff --git a/tests/jupyter/notebooks/harmony_concise_api_test.ipynb b/tests/jupyter/notebooks/harmony_concise_api_test.ipynb new file mode 100644 index 00000000..9a454806 --- /dev/null +++ b/tests/jupyter/notebooks/harmony_concise_api_test.ipynb @@ -0,0 +1,241 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Harmony EOSS Concise API Tutorial" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Before you start\n", + "Before you beginning this tutorial, make sure you have an account in the Earthdata Login UAT or Production environment, which \n", + "will be used for this notebook by visiting [https://uat.urs.earthdata.nasa.gov](https://uat.urs.earthdata.nasa.gov).\n", + "These accounts, as all Earthdata Login accounts, are free to create and only take a moment to set up." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### Set Up Authentication\n", + "\n", + "We need some boilerplate up front to log in to Earthdata Login. The function below will allow Python\n", + "scripts to log into any Earthdata Login application programmatically. To avoid being prompted for\n", + "credentials every time you run and also allow clients such as curl to log in, you can add the following\n", + "to a `.netrc` (`_netrc` on Windows) file in your home directory:\n", + "\n", + "```\n", + "machine uat.urs.earthdata.nasa.gov\n", + " login \n", + " password \n", + " \n", + "machine urs.earthdata.nasa.gov\n", + " login \n", + " password \n", + "```\n", + "\n", + "Make sure that this file is only readable by the current user or you will receive an error stating\n", + "\"netrc access too permissive.\"\n", + "\n", + "`$ chmod 0600 ~/.netrc` \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from urllib import request\n", + "from http.cookiejar import CookieJar\n", + "import getpass\n", + "import netrc\n", + "import json\n", + "import requests\n", + "import sys\n", + "import shutil\n", + "import xarray as xr\n", + "import cmr\n", + "import numpy as np\n", + "from podaac.subsetter import subset\n", + "\n", + "def setup_earthdata_login_auth(endpoint):\n", + " \"\"\"\n", + " Set up the request library so that it authenticates against the given Earthdata Login\n", + " endpoint and is able to track cookies between requests. This looks in the .netrc file \n", + " first and if no credentials are found, it prompts for them.\n", + "\n", + " Valid endpoints include:\n", + " uat.urs.earthdata.nasa.gov - Earthdata Login UAT (Harmony's current default)\n", + " urs.earthdata.nasa.gov - Earthdata Login production\n", + " \"\"\"\n", + " try:\n", + " username, _, password = netrc.netrc().authenticators(endpoint)\n", + " except (FileNotFoundError, TypeError):\n", + " # FileNotFound = There's no .netrc file\n", + " # TypeError = The endpoint isn't in the netrc file, causing the above to try unpacking None\n", + " print('Please provide your Earthdata Login credentials to allow data access')\n", + " print('Your credentials will only be passed to %s and will not be exposed in Jupyter' % (endpoint))\n", + " username = input('Username:')\n", + " password = getpass.getpass()\n", + "\n", + " manager = request.HTTPPasswordMgrWithDefaultRealm()\n", + " manager.add_password(None, endpoint, username, password)\n", + " auth = request.HTTPBasicAuthHandler(manager)\n", + "\n", + " jar = CookieJar()\n", + " processor = request.HTTPCookieProcessor(jar)\n", + " opener = request.build_opener(auth, processor)\n", + " request.install_opener(opener)\n", + "\n", + "\n", + "# GET TOKEN FROM CMR \n", + "def get_token( url: str,client_id: str, user_ip: str,endpoint: str) -> str:\n", + " try:\n", + " token: str = ''\n", + " username, _, password = netrc.netrc().authenticators(endpoint)\n", + " xml: str = \"\"\"\n", + " {}{}{}\n", + " {}\"\"\".format(username, password, client_id, user_ip)\n", + " headers: Dict = {'Content-Type': 'application/xml','Accept': 'application/json'}\n", + " resp = requests.post(url, headers=headers, data=xml)\n", + " \n", + " response_content: Dict = json.loads(resp.content)\n", + " token = response_content['token']['id']\n", + " except:\n", + " print(\"Error getting the token - check user name and password\", sys.exc_info()[0])\n", + " return token" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Set up Environment and Collection data\n", + "\n", + "Below we set a default collection (C1940473819-POCLOUD) and environment (prod) which we will use for the podaac Concise test. As the field is tagged with parameters tag, when executing the notebook, they can be provided as inputs from command line, which will overwrite these default values.\n", + "Finding this information would complicate the tutorial- but po.daac has a tutorial available for using the CMR API to find collections and granules of interest. Please see the following tutorial for that information:\n", + "\n", + "PODAAC_CMR.ipynb\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "# This cell is tagged with parameters\n", + "\n", + "collection = 'C1940473819-POCLOUD'\n", + "venue = 'prod'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Prod Defaults\n", + "cmr_root = 'cmr.earthdata.nasa.gov'\n", + "harmony_root = 'https://harmony.earthdata.nasa.gov'\n", + "edl_root = 'urs.earthdata.nasa.gov'\n", + "mode = cmr.queries.CMR_OPS\n", + "\n", + "# UAT Defaults\n", + "if venue == 'uat':\n", + " cmr_root = 'cmr.uat.earthdata.nasa.gov'\n", + " harmony_root = 'https://harmony.uat.earthdata.nasa.gov'\n", + " edl_root = 'uat.urs.earthdata.nasa.gov'\n", + " mode = cmr.queries.CMR_UAT\n", + "\n", + "print (\"Environments: \")\n", + "print (\"\\t\" + cmr_root)\n", + "print (\"\\t\" + harmony_root)\n", + "print (\"\\t\" + edl_root)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "## Tets execution\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "Now call the above function to set up Earthdata Login for subsequent requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "setup_earthdata_login_auth(edl_root)\n", + "token_url=\"https://\"+cmr_root+\"/legacy-services/rest/tokens\"\n", + "token=get_token(token_url,'jupyter', '127.0.0.1',edl_root)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Verify test results\n" + ] + } + ], + "metadata": { + "celltoolbar": "Tags", + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + }, + "toc-autonumbering": true + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 2711ed0362e8aba62c09f04bd59a48b15d31674b Mon Sep 17 00:00:00 2001 From: Zoltan Miskolci Date: Wed, 13 Jul 2022 18:50:43 -0700 Subject: [PATCH 2/5] - github action - updated notebook location --- .github/workflows/jupyter_test.yml | 29 +++++++++++++++++++++++++++++ tests/jupyter/notebook_test.py | 3 +-- 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/jupyter_test.yml diff --git a/.github/workflows/jupyter_test.yml b/.github/workflows/jupyter_test.yml new file mode 100644 index 00000000..18d49e34 --- /dev/null +++ b/.github/workflows/jupyter_test.yml @@ -0,0 +1,29 @@ +# This is the main build pipeline that verifies and publishes the software +name: Jupyter Test +# Controls when the workflow will run +on: + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + # First job in the workflow installs and verifies the software + build: + name: Test Execution + # The type of runner that the job will run on + runs-on: ubuntu-latest + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Create prerequisites + run: | + mkdir current_project + touch current_project/test_in.txt + echo "ASCATC-L2-Coastal" > current_project/test_in.txt + mkdir tests/jupyter/notebooks/output + - name: Run Jupyter notebook + run: | + python3 "./tests/jupyter/notebook_test.py" -n "./tests/jupyter/notebooks/harmony_concise_api_test.ipynb" -e uat -i ./current_project/test_in.txt -o ./tests/jupyter/notebooks//output + diff --git a/tests/jupyter/notebook_test.py b/tests/jupyter/notebook_test.py index b4c35dd8..5b54712b 100644 --- a/tests/jupyter/notebook_test.py +++ b/tests/jupyter/notebook_test.py @@ -1,4 +1,3 @@ -import cmr import papermill as pm import json import os @@ -76,7 +75,7 @@ def run(): collections.append(line.strip()) - notebook = "./notebooks/harmony_concise_api_test.ipynb" + notebook = "./tests/jupyter/notebooks/harmony_concise_api_test.ipynb" notebook_path = os.path.dirname(notebook) notebook_name = os.path.basename(notebook) From 2515c92757753849571279a99a61162edd995fc2 Mon Sep 17 00:00:00 2001 From: Zoltan Miskolci Date: Wed, 13 Jul 2022 19:00:53 -0700 Subject: [PATCH 3/5] added dependencies to build --- .github/workflows/build-pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml index 37fd96b8..c1928df9 100644 --- a/.github/workflows/build-pipeline.yml +++ b/.github/workflows/build-pipeline.yml @@ -79,6 +79,9 @@ jobs: run: | poetry run pylint podaac poetry run flake8 podaac + - name: Install dependencies + run: | + pip install papermill - name: Test and coverage run: | poetry run pytest --junitxml=build/reports/pytest.xml --cov=podaac/ --cov-report=xml:build/reports/coverage.xml -m "not aws and not integration" tests/ From 2675ad316554a280bbf101ef33cb19ba11dcec78 Mon Sep 17 00:00:00 2001 From: Zoltan Miskolci Date: Thu, 14 Jul 2022 13:13:45 -0700 Subject: [PATCH 4/5] updated build pipeline --- .github/workflows/build-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml index c1928df9..eb801fa5 100644 --- a/.github/workflows/build-pipeline.yml +++ b/.github/workflows/build-pipeline.yml @@ -81,7 +81,7 @@ jobs: poetry run flake8 podaac - name: Install dependencies run: | - pip install papermill + poetry add --dev papermill - name: Test and coverage run: | poetry run pytest --junitxml=build/reports/pytest.xml --cov=podaac/ --cov-report=xml:build/reports/coverage.xml -m "not aws and not integration" tests/ From 34efb41015527f55bea89cb8384f26d382dd7000 Mon Sep 17 00:00:00 2001 From: Zoltan Miskolci Date: Tue, 19 Jul 2022 10:53:20 -0700 Subject: [PATCH 5/5] added install dependencies step --- .github/workflows/jupyter_test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/jupyter_test.yml b/.github/workflows/jupyter_test.yml index 18d49e34..a0c3d247 100644 --- a/.github/workflows/jupyter_test.yml +++ b/.github/workflows/jupyter_test.yml @@ -23,6 +23,9 @@ jobs: touch current_project/test_in.txt echo "ASCATC-L2-Coastal" > current_project/test_in.txt mkdir tests/jupyter/notebooks/output + - name: Install dependencies + run: | + pip install papermill - name: Run Jupyter notebook run: | python3 "./tests/jupyter/notebook_test.py" -n "./tests/jupyter/notebooks/harmony_concise_api_test.ipynb" -e uat -i ./current_project/test_in.txt -o ./tests/jupyter/notebooks//output