Add Swift 3.1.1 as a kind (apache#2120)

* Experimental branch for 2079, uses ibm swift ubuntu image for 3.1

* Fixes issue apache#2079, add Swift 3.1.1 runtime kind, update Swift dependencies for Watson SDK, KituraNet, SwiftyJson

* add apache license

* Fix swift311 location and catch docker brake

The location of the swift binary for 311 is now in /usr/bin/swift
Need to catch docker errors when building by usig && instead of ;

* new zip for new swift311 runtime

ansible/group_vars/all

@@ -62,6 +62,9 @@ runtimesManifest:
       default: true
       image:
         name: "swift3action"
+    - kind: "swift:3.1.1"
+      image:
+        name: "action-swift-v3.1.1"
     java:
     - kind: "java"
       attached:
@@ -230,4 +233,4 @@ catalog_repos:
     # Set the local location as the same level as openwhisk home, but it can be changed.
     location: "{{ openwhisk_home }}/../openwhisk-catalog"
     version: "HEAD"
-    repo_update: "no"
+    repo_update: "no"

ansible/roles/invoker/tasks/deploy.yml

@@ -12,6 +12,7 @@
     - '{{ docker_image_prefix }}/python2action'
     - '{{ docker_image_prefix }}/python3action'
     - '{{ docker_image_prefix }}/swift3action'
+    - '{{ docker_image_prefix }}/action-swift-v3.1.1'
     - '{{ docker_image_prefix }}/java8action'
   when: docker_registry != ""
   retries: 3

core/swift3.1.1Action/Dockerfile

@@ -0,0 +1,30 @@
+# Dockerfile for swift actions, overrides and extends ActionRunner from actionProxy
+# This Dockerfile is partially based on: https://github.com/IBM-Swift/swift-ubuntu-docker/blob/master/swift-development/Dockerfile
+FROM ibmcom/swift-ubuntu:3.1.1
+
+# Set WORKDIR
+WORKDIR /
+
+# Upgrade and install basic Python dependencies
+RUN apt-get -y update \
+ && apt-get -y install --fix-missing python2.7 python-gevent python-flask zip
+
+# Add the action proxy
+RUN mkdir -p /actionProxy
+ADD actionproxy.py /actionProxy
+
+# Add files needed to build and run action
+RUN mkdir -p /swift3Action
+ADD epilogue.swift /swift3Action
+ADD buildandrecord.py /swift3Action
+ADD swift3runner.py /swift3Action
+ADD spm-build /swift3Action/spm-build
+
+
+# Build kitura net
+RUN touch /swift3Action/spm-build/main.swift
+RUN python /swift3Action/buildandrecord.py && rm /swift3Action/spm-build/.build/release/Action
+#RUN cd /swift3Action/spm-build; swift build -v -c release; rm /swift3Action/spm-build/.build/release/Action
+ENV FLASK_PROXY_PORT 8080
+
+CMD ["/bin/bash", "-c", "cd /swift3Action && PYTHONIOENCODING='utf-8' python -u swift3runner.py"]

core/swift3.1.1Action/build.gradle

@@ -0,0 +1,41 @@
+ext.dockerImageName = 'action-swift-v3.1.1'
+apply from: '../../gradle/docker.gradle'
+distDocker.dependsOn 'copyProxy'
+distDocker.dependsOn 'copyEpilogue'
+distDocker.dependsOn 'copySwiftRunner'
+distDocker.dependsOn 'copyWhisk'
+distDocker.dependsOn 'copyWhiskJsonUtils'
+distDocker.finalizedBy('cleanup')
+
+task copyProxy(type: Copy) {
+    from '../actionProxy/actionproxy.py'
+    into './actionproxy.py'
+}
+
+task copyEpilogue(type: Copy) {
+    from '../swift3Action/epilogue.swift'
+    into '.'
+}
+
+task copySwiftRunner(type: Copy) {
+    from '../swift3Action/swift3runner.py'
+    into '.'
+}
+
+task copyWhisk(type: Copy) {
+    from '../swift3Action/spm-build/_Whisk.swift'
+    into './spm-build'
+}
+
+task copyWhiskJsonUtils(type: Copy) {
+    from '../swift3Action/spm-build/_WhiskJSONUtils.swift'
+    into './spm-build'
+}
+
+task cleanup(type: Delete) {
+    delete 'actionproxy.py'
+    delete 'epilogue.swift'
+    delete 'swift3runner.py'
+    delete 'spm-build/_Whisk.swift'
+    delete 'spm-build/_WhiskJSONUtils.swift'
+}

core/swift3.1.1Action/buildandrecord.py

@@ -0,0 +1,77 @@
+"""Python to generate build script.
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+"""
+from __future__ import print_function
+import os
+import sys
+from subprocess import check_output
+
+# Settings
+COMPILE_PREFIX = "/usr/bin/swiftc -module-name Action "
+LINKER_PREFIX =  "/usr/bin/swiftc -Xlinker '-rpath=$ORIGIN' '-L/swift3Action/spm-build/.build/release' -o '/swift3Action/spm-build/.build/release/Action'"
+GENERATED_BUILD_SCRIPT = "/swift3Action/spm-build/swiftbuildandlink.sh"
+SPM_DIRECTORY = "/swift3Action/spm-build"
+BUILD_COMMAND = ["swift", "build", "-v", "-c", "release"]
+
+# Build Swift package and capture step trace
+print("Building action")
+out = check_output(BUILD_COMMAND, cwd=SPM_DIRECTORY)
+print("action built. Decoding compile and link commands")
+
+# Look for compile and link commands in step trace
+compileCommand = None
+linkCommand = None
+
+buildInstructions = out.decode("utf-8").splitlines()
+
+for instruction in buildInstructions:
+    if instruction.startswith(COMPILE_PREFIX):
+        compileCommand = instruction
+
+        # add flag to quiet warnings
+        compileCommand += " -suppress-warnings"
+
+    elif instruction.startswith(LINKER_PREFIX):
+        linkCommand = instruction
+
+# if found, create build script, otherwise exit with error
+if compileCommand is not None and linkCommand is not None:
+    print("Generated OpenWhisk Compile command: %s" % compileCommand)
+    print("=========")
+    print("Generated OpenWhisk Link command: %s" % linkCommand)
+
+    with open(GENERATED_BUILD_SCRIPT, "a") as buildScript:
+        buildScript.write("#!/bin/bash\n")
+        buildScript.write("echo \"Compiling\"\n")
+        buildScript.write("%s\n" % compileCommand)
+        buildScript.write("swiftStatus=$?\n")
+        buildScript.write("echo swiftc status is $swiftStatus\n")
+        buildScript.write("if [[ \"$swiftStatus\" -eq \"0\" ]]; then\n")
+        buildScript.write("echo \"Linking\"\n")
+        buildScript.write("%s\n" % linkCommand)
+        buildScript.write("else\n")
+        buildScript.write(">2& echo \"Action did not compile\"\n")
+        buildScript.write("exit 1\n")
+        buildScript.write("fi")
+
+    os.chmod(GENERATED_BUILD_SCRIPT, 0o777)
+    sys.exit(0)
+else:
+    print("Cannot generate build script: compile or link command not found")
+    sys.exit(1)

core/swift3.1.1Action/spm-build/Package.swift

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2015-2016 IBM Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PackageDescription
+
+let package = Package(
+    name: "Action",
+        dependencies: [
+            .Package(url: "https://github.com/IBM-Swift/CCurl.git", "0.2.3"),
+            .Package(url: "https://github.com/IBM-Swift/Kitura-net.git", "1.7.10"),
+            .Package(url: "https://github.com/IBM-Swift/SwiftyJSON.git", "15.0.1"),
+            .Package(url: "https://github.com/watson-developer-cloud/swift-sdk.git", "0.16.0")
+        ]
+)

core/swift3Action/Dockerfile

@@ -61,8 +61,8 @@ ADD spm-build /swift3Action/spm-build
 
 # Build kitura net
 RUN touch /swift3Action/spm-build/main.swift
-RUN python /swift3Action/buildandrecord.py; rm /swift3Action/spm-build/.build/release/Action
+RUN python /swift3Action/buildandrecord.py && rm /swift3Action/spm-build/.build/release/Action
 #RUN cd /swift3Action/spm-build; swift build -c release; rm /swift3Action/spm-build/.build/release/Action
 ENV FLASK_PROXY_PORT 8080
 
-CMD ["/bin/bash", "-c", "cd /swift3Action && PYTHONIOENCODING='utf-8' python -u swift3runner.py"]
+CMD ["/bin/bash", "-c", "cd /swift3Action && PYTHONIOENCODING='utf-8' python -u swift3runner.py"]

docs/actions.md

@@ -680,7 +680,7 @@ docker run --rm -it -v "$(pwd):/owexec" openwhisk/swift3action bash
 This has created hello.zip in the same directory as hello.swift. 
 -Upload it to OpenWhisk with the action name helloSwifty:
   ```
-  wsk action update helloSwiftly hello.zip --kind swift:3
+  wsk action update helloSwiftly hello.zip --kind swift:3.1.1
   ```
 
 - To check how much faster it is, run 

docs/reference.md

@@ -354,6 +354,22 @@ Python 2 actions are executed using Python 2.7.12. This is the default runtime f
 - Werkzeug v0.12
 - zope.interface v4.3.3
 
+## Swift actions
+
+### Swift 3
+Swift 3 actions are executed using Swift 3.0.2  `--kind swift:3` or Swift 3.1.1 `--kind swift:3.1.1`, respectively.  The default `--kind swift:default` is Swift 3.0.2.
+
+Swift 3.0.2 actions can use the following packages:
+- KituraNet version 1.0.1, https://github.com/IBM-Swift/Kitura-net
+- SwiftyJSON version 14.2.0, https://github.com/IBM-Swift/SwiftyJSON
+- IBM Swift Watson SDK version 0.4.1, https://github.com/IBM-Swift/swift-watson-sdk
+
+Swift 3.1.1 actions can use the following packages:
+- KituraNet version 1.7.6, https://github.com/IBM-Swift/Kitura-net
+- SwiftyJSON version 15.0.1, https://github.com/IBM-Swift/SwiftyJSON
+- Watson Developer Cloud SDK version 0.16.0, https://github.com/watson-developer-cloud/swift-sdk
+
+
 ## Docker actions
 
 Docker actions run a user-supplied binary in a Docker container. The binary runs in a Docker image based on [python:3.6.1-alpine](https://hub.docker.com/r/library/python), so the binary must be compatible with this distribution.

settings.gradle

@@ -8,6 +8,7 @@ include 'core:actionProxy'
 include 'core:pythonAction'
 include 'core:python2Action'
 include 'core:swift3Action'
+include 'core:swift3.1.1Action'
 include 'core:javaAction'
 
 include 'tools:cli'

tests/build.gradle

@@ -32,6 +32,7 @@ test.dependsOn([
     ':core:python2Action:distDocker',
     ':core:javaAction:distDocker',
     ':core:swift3Action:distDocker',
+    ':core:swift3.1.1Action:distDocker',
     ':sdk:docker:distDocker',
     ':tests:dat:blackbox:badaction:distDocker',
     ':tests:dat:blackbox:badproxy:distDocker'

tests/src/test/scala/actionContainers/ActionContainer.scala

@@ -178,11 +178,11 @@ object ActionContainer {
         val f = for (
             entity <- Marshal(content).to[MessageEntity];
             request = HttpRequest(method = HttpMethods.POST, uri = uri, entity = entity);
-            response <- AkkaHttpUtils.singleRequest(request, 60.seconds, retryOnTCPErrors = true);
+            response <- AkkaHttpUtils.singleRequest(request, 90.seconds, retryOnTCPErrors = true);
             responseBody <- Unmarshal(response.entity).to[String]
         ) yield (response.status.intValue, Try(responseBody.parseJson.asJsObject).toOption)
 
-        Await.result(f, 1.minute)
+        Await.result(f, 90.seconds)
     }
 
     private class ActionContainerImpl() extends ActionContainer {

tests/src/test/scala/actionContainers/Swift311ActionContainerTests.scala

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package actionContainers
+
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class Swift311ActionContainerTests extends Swift3ActionContainerTests {
+    override lazy val swiftContainerImageName = "action-swift-v3.1.1"
+
+    override lazy val watsonCode = """
+                | import AlchemyDataNewsV1
+                | import ConversationV1
+                | import DiscoveryV1
+                | import DocumentConversionV1
+                | import NaturalLanguageClassifierV1
+                | import NaturalLanguageUnderstandingV1
+                | import PersonalityInsightsV3
+                | import RetrieveAndRankV1
+                | import ToneAnalyzerV3
+                | import TradeoffAnalyticsV1
+                | import VisualRecognitionV3
+                |
+                | func main(args: [String:Any]) -> [String:Any] {
+                |     return ["message": "I compiled and was able to import Watson SDKs"]
+                | }
+            """.stripMargin
+    override lazy val swiftBinaryName = "helloSwift311.zip"
+}

tests/src/test/scala/actionContainers/SwiftActionContainerTests.scala → tests/src/test/scala/actionContainers/Swift3ActionContainerTests.scala

@@ -29,7 +29,7 @@ import spray.json.JsString
 import common.TestUtils
 
 @RunWith(classOf[JUnitRunner])
-class SwiftActionContainerTests extends BasicActionRunnerTests with WskActorSystem {
+class Swift3ActionContainerTests extends BasicActionRunnerTests with WskActorSystem {
 
     // note: "out" will likely not be empty in some swift build as the compiler
     // prints status messages and there doesn't seem to be a way to quiet them
@@ -78,6 +78,16 @@ class SwiftActionContainerTests extends BasicActionRunnerTests with WskActorSyst
                 |     return [ "divBy0": div(x:5, y:0) ]
                 | }
             """.stripMargin
+    lazy val watsonCode ="""
+                | import RestKit
+                | import WeatherCompanyData
+                | import AlchemyVision
+                |
+                | func main(args: [String:Any]) -> [String:Any] {
+                |     return ["message": "I compiled and was able to import Watson SDKs"]
+                | }
+            """.stripMargin
+    lazy val swiftBinaryName = "helloSwift3.zip"
 
     // Helpers specific to swift actions
     override def withActionContainer(env: Map[String, String] = Map.empty)(code: ActionContainer => Unit) = {
@@ -238,7 +248,7 @@ class SwiftActionContainerTests extends BasicActionRunnerTests with WskActorSyst
     }
 
     it should "support pre-compiled binary in a zip file" in {
-        val zip = new File(TestUtils.getTestActionFilename("helloSwift.zip")).toPath
+        val zip = new File(TestUtils.getTestActionFilename(swiftBinaryName)).toPath
         val code = ResourceHelpers.readAsBase64(zip)
 
         val (out, err) = withActionContainer() { c =>
@@ -331,15 +341,7 @@ class SwiftActionContainerTests extends BasicActionRunnerTests with WskActorSyst
 
     it should "make Watson SDKs available to action authors" in {
         val (out, err) = withActionContainer() { c =>
-            val code = """
-                | import RestKit
-                | import WeatherCompanyData
-                | import AlchemyVision
-                |
-                | func main(args: [String:Any]) -> [String:Any] {
-                |     return ["message": "I compiled and was able to import Watson SDKs"]
-                | }
-            """.stripMargin
+            val code = watsonCode
 
             val (initCode, _) = c.init(initPayload(code))