Merge pull request #332 from creare-com/release/1.2.0

Release/1.2.0

.gitignore

@@ -11,12 +11,14 @@ __pycache__
 *.rdb
 *.egg-info
 podpac/cache/
-
+dist/aws/podpac_dist.zip
+dist/aws/podpac_deps.zip
 
 # test
 .cache
 .coverage
 artifacts
+settings.json
 
 # docs
 doc/build

.pylintrc

@@ -52,6 +52,11 @@ confidence=
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
 disable=
+        C0330,
+        E0203,
+        E0266,
+        E0501,
+        W0503,
         C0103,      # snake case enforcement
         W0703,      # too broad except error
         R0201,      # no self-use

.travis.yml

@@ -9,7 +9,7 @@ language: python
 sudo: required
 dist: xenial
 python:
-  - "2.7"
+#  - "2.7"
   - "3.5"
   - "3.6"
   - "3.7"
@@ -28,7 +28,11 @@ install:
 
   # install setup.py and dev extras
   - pip install .[devall]
-
+
+  # Allow Python exec and eval functions for unit tests
+  - mkdir /home/travis/.podpac
+  - touch /home/travis/.podpac/ALLOW_PYTHON_EVAL_EXEC
+
 # cache pip dependencies for faster builds
 cache: pip
 

README.MD

@@ -36,17 +36,13 @@ You can find more information on contributing to PODPAC on the [Contributing pag
 
 ## Stability / Maturity
 
-This is in the early development phase. As such:
+PODPAC is in a *beta* phase of development. As such:
 
-* The code may be refactored drastically
-* Interfaces may change in a backwards-incompatible ways
-* Modules, Functions and classes may change names or disappear altogether
-* The above may happen without notice
-* Documentation and testing may be lacking
+* All development will adhere to a semantic versioning system
+* Major revisions may contain backwards incompatible changes, but these changes will be documented in the [CHANGELOG](CHANGELOG.md)
+* We are working to improve documentation, please contact us or create an issue if documentation is out of date or incorrect
+* We look forward to receive feedback on usability and compatibility
 
-We are working towards stability so that the above would no longer be true, and we have already made significant strides towards a stable, maintainable library. 
-
-As such use PODPAC at your own risk. 
 
 ## Acknowledgments
 

dist/aws/Dockerfile

@@ -31,15 +31,15 @@ RUN cd /tmp/vendored/ && touch pydap/__init__.py && \
 
 RUN cp -r /podpac/ /tmp/vendored/ && \
     mv /tmp/vendored/podpac/dist/aws/handler.py /tmp/vendored/handler.py && \
-    cp tmp/vendored/podpac/dist/aws/mk_dist.py /tmp/vendored/mk_dist.py && \
+    cp tmp/vendored/podpac/dist/aws/_mk_dist.py /tmp/vendored/_mk_dist.py && \
     rm -rf /tmp/vendored/podpac/dist/ && \
     cp -r /tmp/vendored/podpac/podpac/* /tmp/vendored/podpac/ && \
     rm -rf /tmp/vendored/podpac/podpac/*
 
 RUN cd /tmp/vendored && \
-    find * -maxdepth 0 -type f | grep ".zip" -v | grep -v ".pyc" | xargs zip -9 -rqy podpac_dist_latest.zip
+    find * -maxdepth 0 -type f | grep ".zip" -v | grep -v ".pyc" | xargs zip -9 -rqy podpac_dist.zip
 RUN cd /tmp/vendored && \
     find * -maxdepth 0 -type d -exec zip -9 -rqy {}.zip {} \;
 RUN cd /tmp/vendored && du -s *.zip > zip_package_sizes.txt
 RUN cd /tmp/vendored && du -s * | grep .zip -v > package_sizes.txt
-RUN cd /tmp/vendored && python3 mk_dist.py
+RUN cd /tmp/vendored && python3 _mk_dist.py

dist/aws/README.md

@@ -0,0 +1,35 @@
+# AWS
+
+## Public PODPAC distribution
+
+PODPAC is compiled into two distribution .zip archives for building serverless functions.
+PODPAC is compile for each version:
+
+- `podpac_dist.zip`: Archive containing PODPAC core distribution
+- `podpac_deps.zip`: Archive containing PODPAC dependencies 
+
+These archives are posted publically in the S3 bucket `podpac-dist`.
+This bucket has one directory for each podpac version.
+The bucket itself is private, but each directory is made public individually.
+
+### Creating new distribution
+
+The following process is used to create new PODPAC distribution in the `podpac-dist` bucket
+when a new version of PODPAC is released.
+
+- Run `build_lambda.sh`. Note this currently requires `settings.json` to copied to the root of the podpac directory.
+- Run `upload_lambda.sh`
+- Navigate to `podpac-dist` (or input bucket) and make the archives public
+
+## Handler
+
+...document handler.py...
+
+## Using Distribution
+
+...document podpac build process...
+
+## Debugging Lambda Function
+
+Use the script `print_logs.sh` to read cloud watch logs from your built lambda function.
+This is currently the only way to debug your lambda function.

dist/aws/_mk_dist.py

@@ -0,0 +1,65 @@
+#!python
+
+# Make dist and deps zip archives
+# for podpac lambda function
+#
+# Used in the Dockerfile
+
+import subprocess
+
+with open("zip_package_sizes.txt", "r") as fid:
+    zps = fid.read()
+
+with open("package_sizes.txt", "r") as fid:
+    ps = fid.read()
+
+
+def parse_ps(ps):
+    lns = ps.split("\n")
+    pkgs = {}
+    for ln in lns:
+        try:
+            parts = ln.split("\t")
+            pkgs[parts[1]] = int(parts[0])
+        except:
+            pass
+    return pkgs
+
+
+pgz = parse_ps(zps)
+pg = parse_ps(ps)
+
+data = {}
+for p, s in pgz.items():
+    os = pg.get(p[:-4], 0)
+    data[p] = {"zip_size": s, "size": os, "ratio": os * 1.0 / s}
+
+sdata = sorted(data.items(), key=lambda t: t[1]["ratio"])
+
+zipsize = data["podpac_dist.zip"]["zip_size"]
+totsize = sum([pg[k] for k in pg if (k + ".zip") not in pgz.keys()])
+pkgs = []
+for val in sdata[::-1]:
+    if val[0] == "podpac_dist.zip" or "rasterio" in val[0] or "pyproj" in val[0]:
+        continue
+    key = val[0]
+    pkgs.append(key)
+
+    zipsize += data[key]["zip_size"]
+    totsize += data[key]["size"]
+
+    if zipsize > 50000 or totsize > 250000:
+        k = pkgs.pop()
+        zipsize -= data[k]["zip_size"]
+        totsize -= data[k]["size"]
+
+core = [k[:-4] for k in pkgs if k != "podpac_dist.zip"]
+deps = [k[:-4] for k in data if k[:-4] not in core and k != "podpac_dist.zip"]
+dep_size = sum([data[k + ".zip"]["size"] for k in deps])
+dep_size_zip = sum([data[k + ".zip"]["zip_size"] for k in deps])
+
+# add core to podpac_dist.zip
+cmd = ["zip", "-9", "-rq", "podpac_dist.zip"] + core
+subprocess.call(cmd)
+cmd = ["zip", "-9", "-rqy", "podpac_deps.zip"] + deps
+subprocess.call(cmd)

dist/aws/build_lambda.sh

@@ -1,45 +1,39 @@
 #!/bin/sh
+#
+# Build podpac lambda distribution and dependencies
+# 
+# Currently, this builds the function using the local 
+# podpac repository, including any outstanding changes.
+# 
+# Usage:
+# 
+# $ bash build_lambda.sh [s3-bucket] [function-name] 
+# 
+# Requires:
+# - Docker
+# - `settings.json` to be copied to the root directory of the podpac repository
+#   This will not be required in the future
+#   
+# Example usage:
+# 
+# $ bash build_lambda.sh
 
-# Set up some variables
-if [ -z "$1" ]
-  then
-    TAG=""
-    COMMIT_SHA="$(git rev-parse HEAD)"
-    VERSION="develop"
-  else
-    TAG=$1
-    VERSION=$TAG
-    COMMIT_SHA=""
-fi
 
-set -e
-aws s3 ls s3://podpac-s3/podpac
-
-# TODO change this to expect tag from args, exit if there is no tag given.
+# variables
+COMMIT_SHA="$(git rev-parse HEAD)"
+TAG="$(git describe --always)"
 DOCKER_NAME="podpac"
-DOCKER_TAG="latest"
-echo "${COMMIT_SHA}"
-echo $DOCKER_NAME:$DOCKER_TAG
+DOCKER_TAG=$TAG
+
+echo "Creating docker image from podpac version ${TAG}"
+echo "${DOCKER_NAME}:${DOCKER_TAG}"
 
-# Build docker, and extract zips
+# Navigate to root, build docker, and extract zips
+pushd ../../
 docker build -f dist/aws/Dockerfile --no-cache --tag $DOCKER_NAME:$DOCKER_TAG --build-arg COMMIT_SHA="${COMMIT_SHA}" --build-arg TAG="${TAG}" .
 docker run --name "${DOCKER_NAME}" -itd $DOCKER_NAME:$DOCKER_TAG
-docker cp "${DOCKER_NAME}":/tmp/vendored/podpac_dist_latest.zip .
-docker cp "${DOCKER_NAME}":/tmp/vendored/podpac_deps_latest.zip .
+docker cp "${DOCKER_NAME}":/tmp/vendored/podpac_dist.zip ./dist/aws
+docker cp "${DOCKER_NAME}":/tmp/vendored/podpac_deps.zip ./dist/aws
 docker stop "${DOCKER_NAME}"
 docker rm "${DOCKER_NAME}"
-
-# Upload zips to S3, according to naming convention
-if [ -z $TAG ]
-  then
-    echo "tag is empty"
-  else
-    aws s3 cp podpac_deps_latest.zip s3://podpac-s3/podpac/podpac_deps_$TAG.zip
-    aws s3 cp podpac_dist_latest.zip s3://podpac-s3/podpac/podpac_dist_$TAG.zip
-fi
-aws s3 cp podpac_deps_latest.zip s3://podpac-s3/podpac/podpac_deps_ESIP3.zip
-aws s3 cp podpac_dist_latest.zip s3://podpac-s3/podpac/podpac_dist_ESIP3.zip
-rm podpac_deps_latest.zip podpac_dist_latest.zip
-
-# Update lambda function to use the zips from S3 (uploaded above).
-aws lambda update-function-code --function-name podpac_lambda_ESIP3 --s3-bucket podpac-s3 --s3-key podpac/podpac_dist_ESIP3.zip
+popd

dist/aws/configure_lambda.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+#
+# Configure AWS for podpac lambda function
+# 
+# Usage:
+# 
+# $ bash configure_lambda.sh [s3-bucket] [function-name]
+# 
+# Requires:
+# - AWS CLI: https://docs.aws.amazon.com/cli/
+# - AWS credentials must be configured using the `aws` cli.
+#   See https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration
+# - Dist and Dependencies uploaded using `upload_lambda.sh`
+# - Function must be created from the AWS Dashboard
+# - API Gateway must be created from the AWS Dashboard
+#   - Note down the `rest-api-id` and `resource-id` in parentheses in the top bar of the API gatway dashboard
+#   - You must Select the top level resource '/' and select "Create Method"
+# Example usage:
+# 
+# $ bash configure_lambda.sh podpac-s3 podpac_lambda h827as06ji 1ya7h6
+# 
+
+# TODO: remove this in the future when function generation/update is automated elsewhere
+
+BUCKET=$1
+FUNCTION=$2
+API_ID=$3
+API_RESOURCE_ID=$4
+TAG="$(git describe --always)"
+
+if [ -z "$BUCKET" ]
+  then
+    echo "S3 bucket name required as first cli argument"
+    exit 1
+  else
+    echo "Bucket: ${BUCKET}"
+fi
+
+if [ -z "$FUNCTION" ]
+  then
+    echo "Function name required as second cli argument"
+    exit 1
+  else
+    echo "Function: ${FUNCTION}"
+fi
+
+if [ -z "$API_ID" ]
+  then
+    echo "Rest API ID required as third cli argument"
+    exit 1
+  else
+    echo "REST API ID: ${API_ID}"
+fi
+
+if [ -z "$API_RESOURCE_ID" ]
+  then
+    echo "API Resource ID required as fourth cli argument"
+    exit 1
+  else
+    echo "API Resource ID: ${API_RESOURCE_ID}"
+fi
+
+# Update lambda function to use the zips from S3 (uploaded above)
+# aws lambda update-function-code --function-name $FUNCTION --s3-bucket $BUCKET --s3-key podpac/podpac_dist_$TAG.zip
+# aws lambda update-function-configuration --function-name $FUNCTION --handler handler.handler --timeout 300 --memory-size 2048
+# aws apigateway update-rest-api --rest-api-id $API_ID --patch-operations "op=replace,path=/binaryMediaTypes/*~1*,value='*/*'"
+RESOURCE=$(aws apigateway create-resource --rest-api-id $API_ID --parent-id $API_RESOURCE_ID --path-part 'lambda' --output text)
+RESOURCE_ID=$(echo "$(echo $RESOURCE | cut -d " " -f1)")
+aws apigateway put-method --rest-api-id $API_ID --resource-id $RESOURCE_ID --http-method ANY --authorization-type NONE
+
+echo "Log in to AWS and perform the following steps:"
+echo "1. Navigate to your API in the API Gateway and select the resource /lambda HTTP Method (ANY)."
+echo "2. Select Integration Request -> Lambda Function, Check Use Lambda Proxy Integration, Select your lambda function region and function name."
+echo "3. Press the Actions dropdown and select Deploy API. Select [New Stage] and create a stage name (doesn't matter exact name)"
+echo "4. Navigate to your lambda function console and confirm you see API Gateway as a trigger."
+
+# LAMBDA_URI=`$(aws lambda)
+# aws apigateway put-integration --rest-api-id $API_ID --resource-id $API_RESOURCE_ID --http-method ANY --type AWS --integration-http-method POST --uri

dist/aws/example.json

@@ -1,23 +1,15 @@
 {
   "pipeline": {
-    "nodes": {
-      "SinCoords": {
-        "node": "core.algorithm.algorithm.SinCoords",
-        "inputs": {}
-      }
+    "SinCoords": {
+      "node": "core.algorithm.algorithm.SinCoords"
     }
   },
-  "output": [{
-    "mode": "image",
+  "output": {
     "format": "png",
     "vmin": -1.2,
     "vmax": 1.2,
-    "nodes": [
-      "SinCoords"
-    ]
-  }],
-  "coordinates":
-  {
+  },
+  "coordinates": {
     "TIME": "2018-03-03",
     "BBOX": "-180.0,-90.0,180.0,90.0",
     "WIDTH": 256,