Container example in Python

Dockerfile

@@ -0,0 +1,29 @@
+#Dockerfile is used to create an image
+
+#Set the base image.
+FROM python:3.9-slim
+# This is for interacting with a PostgreSQL database; we aren't doing that but one of my dependencies listed in
+# requirements.txt requires libpq-dev, so it must be included.
+RUN apt-get update
+RUN apt-get install -y gcc libpq-dev
+
+#Set the work directory in the container
+WORKDIR /home
+
+# copy the dependencies file to the working directory
+COPY requirements.txt .
+
+# Ensure pip is up to date
+RUN pip install --upgrade pip
+
+# install dependencies
+RUN pip install -r requirements.txt
+
+# copy the content of the local src directory to the working directory
+COPY src/ .
+
+# Define the entrypoint of the container. Passing arguments (known as the COMMAND) when running the
+# container will be passed as arguments to the function. Ultimately what is executed is the ENTRYPOINT concatenated
+# with the COMMAND. You may add custom options to ENTRYPOINT but note that those options will always be set when
+# running in AWS.
+ENTRYPOINT ["python","algorithm_example.py"]

README.md

@@ -1 +1,43 @@
-# imap_python_processing_example
+ # Container Example Code
+
+This the basic code for creating a Docker container. 
+For questions contact: laura.sandoval@lasp.colorado.edu.
+
+## Python Setup
+
+It is recommended that a virtual environment is created within the project:
+1. Make a virtual environment using `python -m venv venv` and activate it 
+with `source venv/bin/activate`. 
+2. Generate requirements.txt only if additional libraries are required. Otherwise, use the default project requirements.txt.
+3. pip install -r requirements.txt
+
+Note: If an error is thrown when installing psycopg2, you may need to read requirements for installing psycopg2: 
+https://www.psycopg.org/docs/install.html#build-prerequisites. On Mac simply install Postgres.app, 
+which comes with a Postgres server, client, and command line utilities (https://postgresapp.com/)
+
+
+## Example Algorithm Usage
+
+To run
+`python src/algorithm_example.py --help`
+
+
+## Building and Running a Docker Image Locally
+
+To build the image run the following command from the Dockerfile directory. You might add -t option to tag your image 
+and --rm to remove intermediate containers after the build is done.
+
+`docker build -t my-image --rm .`
+
+Now we can run our image using bind mounting. In our example, we will name the container ‘my_app’. 
+Adding --rm option will remove the container automatically after the container exits.
+
+`docker run --rm -it \
+  -e PROCESSING_DROPBOX=/opt/data/dropbox \
+  --volume="$(pwd)/container_example_data:/opt/data" \
+  my-image:latest /opt/data/dropbox/input_manifest_20220923t000000.json`
+
+The script `run_container_example.sh` should contain this code as well. You can build and run the example container
+with `./run_container_example.sh`. Docker must be running, you must have permissions to execute the script, and
+you must be in the same directory as the script. If you get a permission error, 
+run `chmod 755 run_container_example.sh`

container_example_data/dropbox/input_manifest_20220923t000000.json

@@ -0,0 +1,8 @@
+{
+    "manifest_type": "INPUT",
+    "files": [
+        {"filename": "/opt/data/source_data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"},
+        {"filename": "/opt/data/source_data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}
+    ],
+    "configuration": null
+}

container_example_data/dropbox/input_manifest_20220923t000000_aws.json

@@ -0,0 +1,8 @@
+{
+    "manifest_type": "INPUT",
+    "files": [
+        {"filename": "s3://bucket-path-dev/source-data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"},
+        {"filename": "s3://bucket-path-dev/source-data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}
+    ],
+    "configuration": null
+}

container_example_data/dropbox/input_manifest_20220923t000000_local.json

@@ -0,0 +1,8 @@
+{
+    "manifest_type": "INPUT",
+    "files": [
+        {"filename": "../container_example_data/source_data/filename_example_1.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"},
+        {"filename": "../container_example_data/source_data/filename_example_2.h5", "checksum": "b4c00c2b7c3a15f7c78343d77ee5f572"}
+    ],
+    "configuration": null
+}

requirements.txt

@@ -0,0 +1,5 @@
+libera-utils
+numpy
+h5py
+pandas
+libera-utils[db,spice] == 2.0.0

run_container_example.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+echo "Building docker container"
+docker build -t algorithm-example:latest .
+echo "Finished."
+echo
+echo "Running container example."
+docker run --rm -it \
+  -e PROCESSING_DROPBOX=/opt/data/dropbox \
+  --volume="$(pwd)/container_example_data:/opt/data" \
+  algorithm-example:latest /opt/data/dropbox/input_manifest_20220923t000000.json
+echo "Algorithm complete"

src/algorithm_example.py

@@ -0,0 +1,164 @@
+"""
+Container example
+
+Authors: Laura Sandoval, Gavin Medley, Matt Watwood
+"""
+# Standard
+import argparse
+from hashlib import md5
+import logging
+import os
+import sys
+# Installed
+import h5py as h5
+import numpy as np
+import pandas as pd
+# Local
+from libera_utils.io.smart_open import smart_open
+from libera_utils.io.manifest import Manifest, ManifestType
+from static.l2_quality_flags import L2QualityFlag
+
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    # Initialize logging. You can configure logging however you like
+    logging.basicConfig(level="DEBUG")
+
+    # Get logger instance, named for the current function for traceability
+    logger.debug(f"Command executed in container: {sys.argv}")
+    logger.info("Parsing CLI arguments.")
+    args = parse_cli_args(sys.argv[1:])
+
+    manifest = args.manifest
+    logger.info(f"Manifest file to read: {manifest}")
+
+    logger.info("Additional options passed are "
+                f"exampleint={args.exampleint}, examplefloat={args.examplefloat}, examplebool={args.examplebool}")
+
+    processing_dropbox = os.environ['PROCESSING_DROPBOX']
+
+    # read json information
+    manifest = Manifest.from_file(manifest)
+    if not manifest.manifest_type == ManifestType.INPUT:
+        raise ValueError("Incorrect manifest type received as input.")
+
+    logger.debug(f"Manifest type is {manifest.manifest_type}")
+    logger.debug(f"Manifest contains files:\n{manifest.files}")
+
+    # read hdf5
+    for record in manifest.files:
+        checksum = record['checksum']
+        filename = record['filename']
+        # Validate checksums
+        with smart_open(filename, 'rb') as fh:
+            if checksum != md5(fh.read()).hexdigest():
+                raise ValueError("Checksums do not match!")
+            logger.debug(f"Checksum matches for {filename}")
+
+        with h5.File(smart_open(filename, 'rb'), 'r') as h5file:
+            # Get the data from each file and put data into some format that will be used in the algorithm
+            data_in = np.array(h5file['HDFEOS/SWATHS/Swath1/DataField/Temperature'])
+            logger.info(f"Found input data in HDF5 file:\n{data_in}")
+
+    df = generate_example_data()
+
+    # write example output data to a new HDF5 file
+    output_files = []
+    output_filepath = os.path.join(processing_dropbox, 'example_output.h5')
+    logger.info(f"Writing output file: {output_filepath}")
+    with h5.File(smart_open(output_filepath, 'xb'), 'x') as hdf:
+        hdf.create_group('new_group')
+        hdf.attrs['someattr'] = "hello, world"
+        hdf.create_dataset('data/array1', data=df.data_out)
+        hdf.create_dataset('quality_out/array1', data=df.quality_int)
+
+    # get the checksum of the written file
+    with smart_open(output_filepath, 'rb') as fh:
+        checksum = md5(fh.read()).hexdigest()
+
+    output_files.append({"filename": output_filepath, "checksum": checksum})
+
+    # Write output manifest file containing a list of the product files that the processing created
+    output_manifest_path = os.path.join(processing_dropbox, "output_manifest_20220923t111111.json")
+    logger.info(f"Writing output manifest: {output_filepath}")
+    output_manifest = Manifest(manifest_type=ManifestType.OUTPUT,
+                               filename=output_manifest_path,
+                               files=output_files,
+                               configuration={})
+    output_manifest.write(output_manifest_path)
+    logger.info("Algorithm complete. Exiting.")
+
+
+def generate_example_data():
+    """
+    Function created to make up data and quality flags
+
+    Returns
+    -------
+    pandas.core.frame.DataFrame
+        Dataframe containing data and quality flag columns
+    """
+
+    # generate fake data product to write as output
+    df = pd.DataFrame()
+    df['data_out'] = [-np.inf, -np.inf, 30, 40, 50, 60]
+    df['quality'] = np.full(len(df.data_out), fill_value=L2QualityFlag.NONE)
+    df['quality_int'] = np.full(len(df.data_out), fill_value=0)
+
+    # flag neg infinite values
+    was_inf = np.equal(df.data_out, -np.inf)
+    df.loc[was_inf, 'quality'] = df.loc[was_inf, 'quality'][0] | L2QualityFlag.INF | L2QualityFlag.NEG
+    df.loc[was_inf, 'quality_int'] = (df.loc[was_inf, 'quality'][0] | L2QualityFlag.INF | L2QualityFlag.NEG).value
+
+    # view decomposed flag and value
+    view_summary = df.quality.values[0].summary
+    view_decomposed = (df.quality.values[0]).decompose()
+    print(view_summary)
+    print(view_decomposed)
+
+    return df
+
+
+def parse_cli_args(cli_args: list):
+    """
+    Function that parses CLI arguments
+
+    Parameters
+    ----------
+    cli_args : list
+        List of string arguments to parse
+
+    Returns
+    -------
+    Namespace
+        A Namespace object containing the parsed arguments as attributes
+    """
+    parser = argparse.ArgumentParser(description='program arguments.')
+
+    parser.add_argument('--exampleint',
+                        type=int,
+                        default=42,
+                        help='An example integer input option'
+                        )
+
+    parser.add_argument('--examplefloat',
+                        type=float,
+                        default=3.14159,
+                        help='An example float input option')
+
+    parser.add_argument('--examplebool',
+                        type=bool,
+                        default=True,
+                        help='An example boolean input option')
+
+    parser.add_argument('manifest',
+                        type=str,
+                        help="Input JSON manifest file containing a list of files to use in the processing algorithm.")
+
+    return parser.parse_args(cli_args)
+
+
+if __name__ == "__main__":
+    main()

src/static/l2_quality_flags.py

@@ -0,0 +1,23 @@
+"""
+Quality flag example
+
+Authors: Laura Sandoval, Gavin Medley, Matt Watwood
+"""
+import libera_utils.quality_flags as qf
+
+
+@qf.with_all_none
+class L2QualityFlag(qf.QualityFlag, metaclass=qf.FrozenFlagMeta):
+    """Quality flag for L2 observation"""
+    INF = qf.FlagBit(
+        2**0,  # bit 0
+        message="Infinite value.")
+    MISSING_TELEM = qf.FlagBit(
+        2**1,  # bit 1
+        message="Missing telemetry.")
+    NEG = qf.FlagBit(
+        2**2,  # bit 2
+        message="Negative value.")
+    UNEXPECTED_TELEM_VALUE_CHANGE = qf.FlagBit(
+        2**3,  # bit 3
+        message="Value changed within the observation that should not have.")

tests/test_quality_flags.py

@@ -0,0 +1,13 @@
+"""Test coverage for the static.l2_quality_flags module"""
+
+import src.static.l2_quality_flags as qf
+
+
+def test_L1QualityFlag():
+    """Test behavior of the L2QualityFlag class"""
+    for f in qf.L2QualityFlag:
+        assert f.value
+        assert f.value.message
+        assert f.summary
+
+    assert len(qf.L2QualityFlag.ALL.summary[1]) == 4