Updated actionloop python [work in progress]

.gitignore

@@ -18,7 +18,7 @@ results
 !/ansible/environments/mac
 
 # Eclipse
-bin/
+#bin/
 **/.project
 .settings/
 .classpath

README.md

@@ -113,3 +113,4 @@ Using IntelliJ:
 -Dhttp.proxyHost=localhost
 -Dhttp.proxyPort=3128
 ```
+

core/python2ActionLoop/Dockerfile

@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# build go proxy from source
+FROM golang:1.12 AS builder_source
+#RUN env CGO_ENABLED=0 go get github.com/apache/openwhisk-runtime-go/main && mv /go/bin/main /bin/proxy
+RUN git clone --branch dev \
+   https://github.com/nimbella-corp/openwhisk-runtime-go /src ;\
+   cd /src ; env GO111MODULE=on CGO_ENABLED=0 go build main/proxy.go && \
+   mv proxy /bin/proxy
+
+# or build it from a release
+FROM golang:1.12 AS builder_release
+ARG GO_PROXY_RELEASE_VERSION=1.12@1.15.0
+RUN curl -sL \
+  https://github.com/apache/openwhisk-runtime-go/archive/{$GO_PROXY_RELEASE_VERSION}.tar.gz\
+  | tar xzf -\
+  && cd openwhisk-runtime-go-*/main\
+  && GO111MODULE=on go build -o /bin/proxy
+
+FROM python:2.7-alpine
+# select the builder to use
+ARG GO_PROXY_BUILD_FROM=source
+
+# Upgrade and install basic Python dependencies
+RUN apk add --no-cache \
+        bash \
+        bzip2-dev \
+        gcc \
+        libc-dev \
+        libxslt-dev \
+        libxml2-dev \
+        libffi-dev \
+        linux-headers \
+        openssl-dev \
+        python-dev
+
+# Install common modules for python
+RUN pip install --no-cache-dir --upgrade pip setuptools six \
+ && pip install --no-cache-dir \
+        gevent==1.3.6 \
+        flask==1.0.2 \
+        beautifulsoup4==4.6.3 \
+        httplib2==0.11.3 \
+        kafka_python==1.4.3 \
+        lxml==4.2.5 \
+        python-dateutil==2.7.3 \
+        requests==2.19.1 \
+        scrapy==1.5.1 \
+        simplejson==3.16.0 \
+        virtualenv==16.0.0 \
+        twisted==18.7.0 \
+        signalfx_lambda==0.2.1
+
+RUN mkdir -p /action
+WORKDIR /
+COPY --from=builder_source /bin/proxy /bin/proxy_source
+COPY --from=builder_release /bin/proxy /bin/proxy_release
+RUN mv /bin/proxy_${GO_PROXY_BUILD_FROM} /bin/proxy
+
+ADD bin/compile /bin/compile
+ADD lib/launcher.py /lib/launcher.py
+# the compiler script
+ENV OW_COMPILER=/bin/compile
+# log initialization errors
+ENV OW_LOG_INIT_ERROR=1
+# the launcher must wait for an ack
+ENV OW_WAIT_FOR_ACK=1
+# using the runtime name to identify the execution environment
+ENV OW_EXECUTION_ENV=openwhisk/action-python-v2.7
+ENTRYPOINT ["/bin/proxy"]

core/python2ActionLoop/Makefile

@@ -0,0 +1,15 @@
+IMG=whisk/actionloop-python-v2.7:latest
+
+build:
+	docker build -t $(IMG) -f Dockerfile .
+
+clean:
+	docker rmi -f $(IMG)
+
+debug:
+	docker run -p 8080:8080 \
+	-ti --entrypoint=/bin/bash -v $(PWD):/mnt \
+	-e OW_COMPILER=/mnt/bin/compile \
+	$(IMG)
+
+.PHONY: build clean  debug

core/python2ActionLoop/bin/compile

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+"""Python Action Builder
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+
+from __future__ import print_function
+import os, os.path, sys, imp, ast, shutil, subprocess, traceback
+from os.path import abspath, exists, dirname
+
+# write a file creating intermediate directories
+def write_file(file, body, executable=False):
+    try: os.makedirs(dirname(file), mode=0o755)
+    except: pass
+    with open(file, mode="wb") as f:
+        f.write(body)
+    if executable:
+        os.chmod(file, 0o755)
+
+# copy a file eventually replacing a substring
+def copy_replace(src, dst, match=None, replacement=""):
+    with open(src, 'rb') as s:
+        body = s.read()
+        if match:
+            body = body.replace(match, replacement)
+        write_file(dst, body)
+
+# assemble sources
+def sources(launcher, main, src_dir):
+    # move exec in the right place if exists
+    src_file = "%s/exec" % src_dir
+    if exists(src_file):
+        os.rename(src_file, "%s/__main__.py" % src_dir)
+    if exists("%s/__main__.py" % src_dir):
+        os.rename("%s/__main__.py" % src_dir, "%s/main__.py" % src_dir)
+
+    # write the boilerplate in a temp dir
+    copy_replace(launcher, "%s/exec__.py" % src_dir,
+          "from main__ import main as main",
+          "from main__ import %s as main" % main )
+
+# compile sources
+def build(src_dir, tgt_dir):
+    # in general, compile your program into an executable format
+    # for scripting languages, move sources and create a launcher
+    # move away the action dir and replace with the new
+    shutil.rmtree(tgt_dir)
+    shutil.move(src_dir, tgt_dir)
+    tgt_file = "%s/exec" % tgt_dir
+    write_file(tgt_file, """#!/bin/bash
+if [ "$(cat $0.env)" = "$__OW_EXECUTION_ENV" ]
+then cd "$(dirname $0)"
+     exec /usr/local/bin/python exec__.py "$@"
+else echo "Execution Environment Mismatch"
+     echo "Expected: $(cat $0.env)"
+     echo "Actual: $__OW_EXECUTION_ENV"
+     exit 1
+fi
+""", True)
+    write_file("%s.env"%tgt_file, os.environ['__OW_EXECUTION_ENV'])
+    return tgt_file
+
+#check if a module exists
+def check(tgt_dir, module_name):
+    try:
+
+        # find module
+        mod = imp.find_module(module_name, [tgt_dir])
+        # parse module
+        ast.parse(mod[0].read())
+        # check virtualenv
+        path_to_virtualenv = abspath('%s/virtualenv' % tgt_dir)
+        if os.path.isdir(path_to_virtualenv):
+            activate_this_file = path_to_virtualenv + '/bin/activate_this.py'
+            if not os.path.exists(activate_this_file):
+               sys.stderr.write('Invalid virtualenv. Zip file does not include activate_this.py')
+    except ImportError:
+        sys.stderr.write("Zip file does not include %s" % module_name)
+    except SyntaxError as er:
+        sys.stderr.write(er.msg)
+    except Exception as ex:
+        sys.stderr.write(ex.message)
+    sys.stderr.flush()
+
+if __name__ == '__main__':
+    if len(sys.argv) < 4:
+        sys.stdout.write("usage: <main-function> <source-dir> <target-dir>\n")
+        sys.stdout.flush()
+        sys.exit(1)
+    launcher = "%s/lib/launcher.py" % dirname(dirname(sys.argv[0]))
+    src_dir = abspath(sys.argv[2])
+    tgt_dir = abspath(sys.argv[3])
+    sources(launcher, sys.argv[1], src_dir)
+    tgt = build(abspath(sys.argv[2]), tgt_dir)
+    check(tgt_dir, "main__")
+    sys.stdout.flush()
+    sys.stderr.flush()

core/python2ActionLoop/build.gradle

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ext.dockerImageName = 'actionloop-python-v2.7'
+apply from: '../../gradle/docker.gradle'

core/python2ActionLoop/lib/launcher.py

@@ -0,0 +1,71 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from __future__ import print_function
+from os import fdopen
+import sys, os, codecs, json, traceback, warnings
+
+sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+sys.stderr = codecs.getwriter('utf8')(sys.stderr)
+
+try:
+  # if the directory 'virtualenv' is extracted out of a zip file
+  path_to_virtualenv = os.path.abspath('./virtualenv')
+  if os.path.isdir(path_to_virtualenv):
+    # activate the virtualenv using activate_this.py contained in the virtualenv
+    activate_this_file = path_to_virtualenv + '/bin/activate_this.py'
+    if os.path.exists(activate_this_file):
+      with open(activate_this_file) as f:
+        code = compile(f.read(), activate_this_file, 'exec')
+        exec(code, dict(__file__=activate_this_file))
+    else:
+      sys.stderr.write('Invalid virtualenv. Zip file does not include /virtualenv/bin/' + os.path.basename(activate_this_file) + '\n')
+      sys.exit(1)
+except Exception:
+  traceback.print_exc(file=sys.stderr, limit=0)
+  sys.exit(1)
+
+# now import the action as process input/output
+from main__ import main as main
+
+env = os.environ
+out = fdopen(3, "wb")
+if os.getenv("__OW_WAIT_FOR_ACK", "") != "":
+  out.write(json.dumps({"ok": True}, ensure_ascii=False).encode('utf-8'))
+  out.write(b'\n')
+  out.flush()
+while True:
+  line = sys.stdin.readline().decode('utf-8')
+  if not line: break
+  args = json.loads(line)
+  payload = {}
+  for key in args:
+    akey = key.encode("ascii", "ignore")
+    if akey == "value":
+      payload = args["value"]
+    else:
+      env["__OW_%s" % akey.upper()] = args[key].encode("ascii", "ignore")
+  res = {}
+  try:
+    res = main(payload)
+  except Exception as ex:
+    print(traceback.format_exc(), file=sys.stderr)
+    res = {"error": str(ex)}
+  out.write(json.dumps(res, ensure_ascii=False).encode('utf-8'))
+  out.write(b'\n')
+  sys.stdout.flush()
+  sys.stderr.flush()
+  out.flush()

core/python3ActionLoop/Dockerfile

@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# build go proxy from source
+FROM golang:1.12 AS builder_source
+#RUN env CGO_ENABLED=0 go get github.com/apache/openwhisk-runtime-go/main && mv /go/bin/main /bin/proxy
+RUN git clone --branch dev \
+   https://github.com/nimbella-corp/openwhisk-runtime-go /src ;\
+   cd /src ; env GO111MODULE=on CGO_ENABLED=0 go build main/proxy.go && \
+   mv proxy /bin/proxy
+
+# or build it from a release
+FROM golang:1.12 AS builder_release
+ARG GO_PROXY_RELEASE_VERSION=1.12@1.15.0
+RUN curl -sL \
+  https://github.com/apache/openwhisk-runtime-go/archive/{$GO_PROXY_RELEASE_VERSION}.tar.gz\
+  | tar xzf -\
+  && cd openwhisk-runtime-go-*/main\
+  && GO111MODULE=on go build -o /bin/proxy
+
+FROM python:3.7-stretch
+ARG GO_PROXY_BUILD_FROM=source
+
+# Install common modules for python
+RUN pip install \
+    beautifulsoup4==4.6.3 \
+    httplib2==0.11.3 \
+    kafka_python==1.4.3 \
+    lxml==4.2.5 \
+    python-dateutil==2.7.3 \
+    requests==2.19.1 \
+    scrapy==1.5.1 \
+    simplejson==3.16.0 \
+    virtualenv==16.0.0 \
+    twisted==18.7.0
+
+RUN mkdir -p /action
+WORKDIR /
+COPY --from=builder_source /bin/proxy /bin/proxy_source
+COPY --from=builder_release /bin/proxy /bin/proxy_release
+RUN mv /bin/proxy_${GO_PROXY_BUILD_FROM} /bin/proxy
+
+ADD bin/compile /bin/compile
+ADD lib/launcher.py /lib/launcher.py
+
+# log initialization errors
+ENV OW_LOG_INIT_ERROR=1
+# the launcher must wait for an ack
+ENV OW_WAIT_FOR_ACK=1
+# using the runtime name to identify the execution environment
+ENV OW_EXECUTION_ENV=openwhisk/action-python-v3.7
+# compiler script
+ENV OW_COMPILER=/bin/compile
+
+ENTRYPOINT ["/bin/proxy"]
+