diff --git a/.travis.yml b/.travis.yml index 526e25224..c8d230010 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,6 +105,7 @@ install: script: - | if [[ "$SHOULD_TEST" == "1" ]]; then + git fetch --unshallow make VERSION=${TRAVIS_TAG:-build-$TRAVIS_BUILD_ID} binary make controller-image CONTROLLER_IMAGE=$CONTROLLER_IMAGE make all-yaml diff --git a/docker/runtime/golang/Dockerfile b/docker/runtime/golang/Dockerfile new file mode 100644 index 000000000..eef4de32a --- /dev/null +++ b/docker/runtime/golang/Dockerfile @@ -0,0 +1,5 @@ +FROM bitnami/minideb:jessie + +USER 1000 + +CMD [ "/kubeless/server" ] diff --git a/docker/runtime/golang/Dockerfile.init b/docker/runtime/golang/Dockerfile.init new file mode 100644 index 000000000..25b1d1188 --- /dev/null +++ b/docker/runtime/golang/Dockerfile.init @@ -0,0 +1,17 @@ +FROM golang:1.10 + +# Install dep +RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh +RUN go get golang.org/x/net/context + +# Prepare function environment +RUN ln -s /kubeless $GOPATH/src/kubeless + +# Install controller +RUN mkdir -p $GOPATH/src/github.com/kubeless/kubeless/pkg/functions +ADD pkg/functions/* $GOPATH/src/github.com/kubeless/kubeless/pkg/functions +RUN mkdir -p $GOPATH/src/controller +ADD docker/runtime/golang/Gopkg.toml $GOPATH/src/controller/ +WORKDIR $GOPATH/src/controller/ +ADD docker/runtime/golang/kubeless.tpl.go $GOPATH/src/controller/ +RUN dep ensure diff --git a/docker/runtime/golang/Gopkg.toml b/docker/runtime/golang/Gopkg.toml new file mode 100644 index 000000000..a924024ff --- /dev/null +++ b/docker/runtime/golang/Gopkg.toml @@ -0,0 +1,6 @@ + +ignored = ["github.com/kubeless/kubeless/pkg/functions"] + +[[constraint]] + name = "github.com/prometheus/client_golang" + revision = "f504d69affe11ec1ccb2e5948127f86878c9fd57" diff --git a/docker/runtime/golang/Makefile b/docker/runtime/golang/Makefile new file mode 100644 index 000000000..791c1697c --- /dev/null +++ b/docker/runtime/golang/Makefile @@ -0,0 +1,5 @@ +init-image: + docker build -f Dockerfile.init -t kubeless/go-init:1.10 ../../../ + +runtime-image: + docker build -f Dockerfile -t kubeless/go:1.10 . diff --git a/docker/runtime/golang/kubeless.tpl.go b/docker/runtime/golang/kubeless.tpl.go new file mode 100644 index 000000000..de4489fdb --- /dev/null +++ b/docker/runtime/golang/kubeless.tpl.go @@ -0,0 +1,171 @@ +/* +Copyright (c) 2016-2017 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "golang.org/x/net/context" + "io/ioutil" + "log" + "net/http" + "os" + "strconv" + "time" + + "kubeless" + + "github.com/kubeless/kubeless/pkg/functions" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var ( + funcHandler = os.Getenv("FUNC_HANDLER") + timeout = os.Getenv("FUNC_TIMEOUT") + funcPort = os.Getenv("FUNC_PORT") + runtime = os.Getenv("FUNC_RUNTIME") + memoryLimit = os.Getenv("FUNC_MEMORY_LIMIT") + intTimeout int + funcContext functions.Context + funcHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "function_duration_seconds", + Help: "Duration of user function in seconds", + }, []string{"method"}) + funcCalls = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "function_calls_total", + Help: "Number of calls to user function", + }, []string{"method"}) + funcErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "function_failures_total", + Help: "Number of exceptions in user function", + }, []string{"method"}) +) + +func init() { + if timeout == "" { + timeout = "180" + } + if funcPort == "" { + funcPort = "8080" + } + funcContext = functions.Context{ + FunctionName: funcHandler, + Timeout: timeout, + Runtime: runtime, + MemoryLimit: memoryLimit, + } + var err error + intTimeout, err = strconv.Atoi(timeout) + if err != nil { + panic(err) + } + prometheus.MustRegister(funcHistogram, funcCalls, funcErrors) +} + +// Logging Functions, required to expose statusCode property +type loggingResponseWriter struct { + http.ResponseWriter + statusCode int +} + +func newLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter { + return &loggingResponseWriter{w, http.StatusOK} +} + +func (lrw *loggingResponseWriter) WriteHeader(code int) { + lrw.statusCode = code + lrw.ResponseWriter.WriteHeader(code) +} + +func logReq(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + lrw := newLoggingResponseWriter(w) + handler.ServeHTTP(lrw, r) + log.Printf("%s \"%s %s %s\" %d %s", r.RemoteAddr, r.Method, r.RequestURI, r.Proto, lrw.statusCode, r.UserAgent()) + if lrw.statusCode == 408 { + go func() { + // Give time to return timeout response + time.Sleep(time.Second) + log.Fatal("Request timeout. Forcing exit") + }() + } + }) +} + +// Handling Functions +func health(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("OK")) +} + +func handler(w http.ResponseWriter, r *http.Request) { + data, err := ioutil.ReadAll(r.Body) + if err != nil { + panic(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(intTimeout)*time.Second) + defer cancel() + event := functions.Event{ + Data: string(data), + EventID: r.Header.Get("event-id"), + EventType: r.Header.Get("event-type"), + EventTime: r.Header.Get("event-time"), + EventNamespace: r.Header.Get("event-namespace"), + Extensions: functions.Extension{ + Request: r, + Context: ctx, + }, + } + funcChannel := make(chan struct { + res string + err error + }, 1) + go func() { + funcCalls.With(prometheus.Labels{"method": r.Method}).Inc() + start := time.Now() + res, err := kubeless.<>(event, funcContext) + funcHistogram.With(prometheus.Labels{"method": r.Method}).Observe(time.Since(start).Seconds()) + pack := struct { + res string + err error + }{res, err} + funcChannel <- pack + }() + select { + case respPack := <-funcChannel: + if respPack.err != nil { + funcErrors.With(prometheus.Labels{"method": r.Method}).Inc() + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fmt.Sprintf("Error: %v", respPack.err))) + } else { + w.Write([]byte(respPack.res)) + } + // Send Timeout response + case <-ctx.Done(): + funcErrors.With(prometheus.Labels{"method": r.Method}).Inc() + w.WriteHeader(http.StatusRequestTimeout) + w.Write([]byte("Timeout exceeded")) + } +} + +func main() { + http.HandleFunc("/", handler) + http.HandleFunc("/healthz", health) + http.Handle("/metrics", promhttp.Handler()) + if err := http.ListenAndServe(fmt.Sprintf(":%s", funcPort), logReq(http.DefaultServeMux)); err != nil { + panic(err) + } +} diff --git a/docs/implementing-new-runtime.md b/docs/implementing-new-runtime.md index 3c508d6bb..4c956be7a 100755 --- a/docs/implementing-new-runtime.md +++ b/docs/implementing-new-runtime.md @@ -5,7 +5,8 @@ As of today, Kubeless supports the following languages: * Ruby * Node.js * Python -* .NET +* PHP +* Golang Each language interpreter is implemented as an image of a Docker container, dispatched by the Kubeless controller. @@ -13,38 +14,92 @@ All the structures that implement language support are coded in the file `langr If you want to extend it and make another language available it is necessary to change the following components: -## 1. Update the kubeless-config configmap +## 1. [Optional] Create an Init image for installing deps and compiling -In this configmap there is a set of runtime images. You need to update this set with an entry pointing to the repository of the Docker container image for the runtime of the new language. +The first step is to create an image for installing function dependencies or compile source code. This step is optional depending on the target language. If the runtime doesn't require compilation and there is already an available image with the necessary packages to install dependencies you can skip this step. -Usually there are three entries - one for HTTP triggers, another for event based functions and another one for the Init container that will be used to install dependencies in the build process. If your new runtime implementation will support only HTTP triggers, then create only two entries as follows: +In case a custom init image is required, create a Dockefile under the folder `docker/runtime//Dockerfile..init`. Note that you can skip `` if just one version is supported. -In the example below, a custom image for `dotnetcore` has been added. You can optionally add `imagePullSecrets` if they are necessary to pull the image from a private Docker registry. +The goal of this image is to have available any tools or files necessary to compile a function or install dependencies. It is not necessary to specify the `CMD` to compile or install dependencies, that will be specified in the Kubeless source code. + +In case that the function server needs to be compiled at this step, see the requirements [in the next section](create-a-runtime-image). + +See an example of an init image for [Go](https://github.com/kubeless/kubeless/blob/master/docker/runtime/golang/Dockerfile.init). + +## 2. Create a runtime image + +The second step is to generate the image that will be used to run the HTTP server that will load and execute functions. Usually at least a Dockerfile and a source code file written in the target language will be needed. Those files will be placed at: + +``` +docker/runtime//Dockerfile. +docker/runtime//kubeless. +``` + +Note that you can skip `` if just one version is supported. + +The HTTP server should satisfy the following requirements: + + - The file to load can be specified using an environment variable `MOD_NAME`. + - The function to load can be specified using an environment variable `FUNC_HANDLER`. + - The port used to expose the service can be modified using an environment variable `FUNC_PORT`. + - The server should return `200 - OK` to requests at `/healthz`. + - Functions should run `FUNC_TIMEOUT` as maximum. If, due to language limitations, it is not possible not stop the user function, at least a `408 - Timeout` response should be returned to the HTTP request. + - Functions should receive two parameters: `event` and `context` and should return the value that will be used as HTTP response. See [the functions standard signature](./runtimes#runtimes-interface) for more information. The information that will be available in `event` parameter will be received as HTTP headers. + - Requests should be served in parallel. + - Requests should be logged to stdout including date, HTTP method, requested path and status code of the reponse. + - Exceptions in the function should be catched. The server should not exit due to a function error. + - [Optional] The function should expose Prometheus statistics in the path `/metrics`. At least it should expose: + - Calls per HTTP method + - Errors per HTTP method + - Histogram with the execution time per HTTP method + +See an example of an runtime image for [Python](https://github.com/kubeless/kubeless/blob/master/docker/runtime/python/Dockerfile.2.7). + +## 3. Update the kubeless-config configmap + +In this configmap there is a set of images corresponding to the ones described in the previous sections. You need to add the references to these images along with general information about the language that will be added. + +There are two entries - one for the runtime image and another for the init container that will be used to install dependencies or compile the function in the build process. + +In the example below, a custom image for `go` has been added. You can optionally add `imagePullSecrets` if they are necessary to pull the image from a private Docker registry. ```patch runtime-images ... -+ - ID: "dotnetcore" -+ versions: -+ - name: "dotnetcore2" -+ version: "2.0" -+ httpImage: "mydocker/kubeless-netcore:latest" -+ initImage: "mydocker/kubeless-netcore-build:latest" -+ imagePullSecrets: -+ - imageSecret: "Secret" -+ depName: "requirements.xml" -+ fileNameSuffix: ".cs" ++ { ++ "ID": "go", ++ "compiled": true, ++ "versions": [ ++ { ++ "name": "go1.10", ++ "version": "1.10", ++ "runtimeImage": "andresmgot/go:1", ++ "initImage": "andresmgot/go-init:19", ++ "imagePullSecrets": [ ++ { ++ "imageSecret": "Secret" ++ } ++ ] ++ } ++ ], ++ "depName": "Gopkg.toml", ++ "fileNameSuffix": ".go" ++ } ``` -Restart the controller after updating the configmap. +Restart the controller after updating the configmap. In case that you want to submit the new runtime specify the new images in the file `kubeless.jsonnet` at the root of this repository. -## 2. Add the build instructions to include dependencies in the runtime +## 4. Add the instructions to intall dependencies in the runtime Each runtime has specific instructions to install its dependencies. These instructions are specified in the method `GetBuildContainer()`. About this method you should know: - - The folder with the function and the dependency files is mounted at `depsVolume.MountPath` - - Dependencies should be installed in the folder `runtimeVolume.MountPath` + - The runtime is specified as a string in the form of `` e.g. `go1.10` + - The folder with the function and the dependency files is mounted at `installVolume.MountPath`. This is the path in which the function dependencies should be installed. Any change outside this folder will be ignored. + +## 5. [Optional] Add the instructions to compile the runtime + +In case it is a compiled language you will need to add the required instructions to compile the source code in the method `GetCompilationContainer()`. As in `GetBuildContainer()` the result should be stored in the folder that `installVolume.MountPath` points to. Anything outside that folder will be ignored. -## 3. Update the deployment to load requirements for the runtime image +## 6. [Optional] Update the deployment to load requirements for the runtime image Some languages require to specify an environment variable in order to load dependencies from a certain path. If that is the case, update the function `updateDeployment()` to include the required environment variable: @@ -63,7 +118,7 @@ func UpdateDeployment(dpm *v1beta1.Deployment, depsPath, runtime string) { This function is called if there are requirements to be injected in your runtime or if it is a custom runtime. -## 4. Add examples +## 6. Add examples In order to demonstrate the usage of the new runtime it will be necessary to add at least three different examples: @@ -73,7 +128,7 @@ In order to demonstrate the usage of the new runtime it will be necessary to add The examples should be added to the folder `examples//` and should be added as well to the Makefile present in `examples/Makefile`. Note that the target should be `get-`, `post-` and `get--deps` for three examples above. -## 5. Add tests +## 7. Add tests For each new runtime, there should be integration tests that deploys the three examples above and check that the function is successfully deployed and that the output of the function is the expected one. For doing so add the counterpart `get--verify`, `post--verify` and `get--deps-verify` in the `examples/Makefile` and enable the execution of these tests in the script `test/integration-tests.bats`: diff --git a/docs/runtimes.md b/docs/runtimes.md index 28fc0d687..5dc5b0383 100644 --- a/docs/runtimes.md +++ b/docs/runtimes.md @@ -86,6 +86,62 @@ For python we use [Bottle](https://bottlepy.org) and we also add routes for heal For the case of Ruby we use [Sinatra](http://www.sinatrarb.com) as web framework and we add the routes required for the function and the health check. Monitoring is currently not supported yet for this framework. PR is welcome :-) +### Go HTTP Trigger + +The Go HTTP server doesn't include any framework since the native packages includes enough functionality to fit our needs. Since there is not a standard package that manages server logs that functionality is implemented in the same server. It is also required to implement the `ResponseWritter` interface in order to retrieve the Status Code of the response. + +### Debugging compilation + +If there is an error during the compilation of a function, the error message will be dumped to the termination log. If you see that the pod is crashed in a init container: + +``` +NAME READY STATUS RESTARTS AGE +get-go-6774465f95-x55lw 0/1 Init:CrashLoopBackOff 1 1m +``` + +That can mean that the compilation failed. You can obtain the compilation logs executing: + +```console +$ kubectl get pod -l function=get-go -o yaml +... + - containerID: docker://253fb677da4c3106780d8be225eeb5abf934a961af0d64168afe98159e0338c0 + image: andresmgot/go-init:1.10 + lastState: + terminated: + containerID: docker://253fb677da4c3106780d8be225eeb5abf934a961af0d64168afe98159e0338c0 + exitCode: 2 + finishedAt: 2018-04-06T09:01:16Z + message: | + # kubeless + /go/src/kubeless/handler.go:6:1: syntax error: non-declaration statement outside function body +... +``` + +You can see there that there is a syntax error in the line 6 of the function. You can also retrieve the same information with this one-liner: + +```console +$ kubectl get pod -l function=get-go -o go-template="{{range .items}}{{range .status.initContainerStatuses}}{{.lastState.terminated.message}}{{end}}{{end}}" + +# kubeless +/go/src/kubeless/handler.go:6:1: syntax error: non-declaration statement outside function body +``` + +### Timeout handling + +One peculiarity of the Go runtime is that the user has a `Context` object as part of the `Event.Extensions` parameter. This can be used to handle timeouts in the function. For example: + +```go +func Foo(event functions.Event, context functions.Context) (string, error) { + select { + case <-event.Extensions.Context.Done(): + return "", nil + case <-time.After(5 * time.Second): + } + return "Function returned after 5 seconds", nil +} +``` + +If the function above has a timeout smaller than 5 seconds it will exit and the code after the `select{}` won't be executed. # Scheduled Trigger diff --git a/examples/Makefile b/examples/Makefile index 497f8a75f..d7570396e 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -116,6 +116,48 @@ get-nodejs-multi: get-nodejs-multi-verify: kubeless function call get-nodejs-multi |egrep hello.world +get-go: + kubeless function deploy get-go --runtime go1.10 --handler handler.Foo --from-file golang/helloget.go + +get-go-verify: + kubeless function call get-go |egrep Hello.world + +get-go-custom-port: + kubeless function deploy get-go-custom-port --runtime go1.10 --handler helloget.Foo --from-file golang/helloget.go --port 8083 + +get-go-custom-port-verify: + kubectl get svc get-go-custom-port -o yaml | grep 'targetPort: 8083' + kubeless function call get-go-custom-port |egrep Hello.world + +timeout-go: + $(eval TMPDIR := $(shell mktemp -d)) + printf 'package kubeless\nimport "github.com/kubeless/kubeless/pkg/functions"\nfunc Foo(event functions.Event, context functions.Context) (string, error) {\nfor{\n}\nreturn "", nil\n}' > $(TMPDIR)/hello-loop.js + kubeless function deploy timeout-go --runtime go1.10 --handler helloget.Foo --from-file $(TMPDIR)/hello-loop.js --timeout 4 + rm -rf $(TMPDIR) + +timeout-go-verify: + $(eval MSG := $(shell kubeless function call timeout-go 2>&1 || true)) + echo $(MSG) | egrep Request.timeout.exceeded + +get-go-deps: + kubeless function deploy get-go-deps --runtime go1.10 --handler helloget.Hello --from-file golang/hellowithdeps.go --dependencies golang/Gopkg.toml + +get-go-deps-verify: + kubeless function call get-go-deps --data '{"hello": "world"}' + kubectl logs -l function=get-go-deps | grep -q 'level=info msg=.*hello.*world' + +post-go: + kubeless function deploy post-go --runtime go1.10 --handler hellowithdata.Handler --from-file golang/hellowithdata.go + +post-go-verify: + kubeless function call post-go --data '{"it-s": "alive"}'| egrep "it.*alive" + # Verify event context + logs=`kubectl logs -l function=post-go`; \ + echo $$logs | grep -q "it.*alive" && \ + echo $$logs | grep -q "UTC" && \ + echo $$logs | grep -q "application/json" && \ + echo $$logs | grep -q "cli.kubeless.io" + get-python-metadata: kubeless function deploy get-python-metadata --runtime python2.7 --handler helloget.foo --from-file python/helloget.py --env foo:bar,bar=foo,foo --memory 128Mi --label foo:bar,bar=foo,foobar @@ -461,4 +503,27 @@ pubsub-ruby-verify: done; \ $$found +pubsub-go: + kubeless topic create s3-go || true + kubeless function deploy pubsub-go --trigger-topic s3-go --runtime go1.10 --handler pubsub-go.Handler --from-file golang/hellowithdata.go + +pubsub-go-verify: + $(eval DATA := $(shell mktemp -u -t XXXXXXXX)) + kubeless topic publish --topic s3-go --data '{"payload":"$(DATA)"}' + number="1"; \ + timeout="60"; \ + found=false; \ + while [ $$number -le $$timeout ] ; do \ + pod=`kubectl get po -oname -l function=pubsub-go`; \ + logs=`kubectl logs $$pod | grep $(DATA)`; \ + if [ "$$logs" != "" ]; then \ + found=true; \ + break; \ + fi; \ + sleep 1; \ + number=`expr $$number + 1`; \ + done; \ + $$found + + pubsub: pubsub-python pubsub-nodejs pubsub-ruby diff --git a/examples/golang/Gopkg.toml b/examples/golang/Gopkg.toml new file mode 100644 index 000000000..016f1d598 --- /dev/null +++ b/examples/golang/Gopkg.toml @@ -0,0 +1,6 @@ + +ignored = ["github.com/kubeless/kubeless/pkg/functions"] + +[[constraint]] + name = "github.com/sirupsen/logrus" + branch = "master" diff --git a/examples/golang/helloget.go b/examples/golang/helloget.go new file mode 100644 index 000000000..bf9aa33f0 --- /dev/null +++ b/examples/golang/helloget.go @@ -0,0 +1,10 @@ +package kubeless + +import ( + "github.com/kubeless/kubeless/pkg/functions" +) + +// Foo sample function +func Foo(event functions.Event, context functions.Context) (string, error) { + return "Hello world!", nil +} diff --git a/examples/golang/hellowithdata.go b/examples/golang/hellowithdata.go new file mode 100644 index 000000000..6e33c4e3f --- /dev/null +++ b/examples/golang/hellowithdata.go @@ -0,0 +1,13 @@ +package kubeless + +import ( + "fmt" + + "github.com/kubeless/kubeless/pkg/functions" +) + +// Handler sample function with data +func Handler(event functions.Event, context functions.Context) (string, error) { + fmt.Println(event) + return event.Data, nil +} diff --git a/examples/golang/hellowithdeps.go b/examples/golang/hellowithdeps.go new file mode 100644 index 000000000..36c21fe5c --- /dev/null +++ b/examples/golang/hellowithdeps.go @@ -0,0 +1,12 @@ +package kubeless + +import ( + "github.com/kubeless/kubeless/pkg/functions" + "github.com/sirupsen/logrus" +) + +// Hello sample function with dependencies +func Hello(event functions.Event, context functions.Context) (string, error) { + logrus.Info(event.Data) + return "Hello world!", nil +} diff --git a/kubeless.jsonnet b/kubeless.jsonnet index ff879a37b..48ebb71d5 100644 --- a/kubeless.jsonnet +++ b/kubeless.jsonnet @@ -75,6 +75,7 @@ local deploymentConfig = '{}'; local runtime_images ='[ { "ID": "python", + "compiled": false, "versions": [ { "name": "python27", @@ -100,6 +101,7 @@ local runtime_images ='[ }, { "ID": "nodejs", + "compiled": false, "versions": [ { "name": "node6", @@ -119,6 +121,7 @@ local runtime_images ='[ }, { "ID": "ruby", + "compiled": false, "versions": [ { "name": "ruby24", @@ -132,6 +135,7 @@ local runtime_images ='[ }, { "ID": "php", + "compiled": false, "versions": [ { "name": "php72", @@ -142,6 +146,20 @@ local runtime_images ='[ ], "depName": "composer.json", "fileNameSuffix": ".php" + }, + { + "ID": "go", + "compiled": true, + "versions": [ + { + "name": "go1.10", + "version": "1.10", + "runtimeImage": "kubeless/go@sha256:bf72622344a54e4360f31d3fea5eb9dca2c96fbedc6f0ad7c54f3eb8fb7bd353", + "initImage": "kubeless/go-init@sha256:ec60becbac57ef775e42191739ee003fd1ce72a49b10da64065ee3ef984f63fd" + } + ], + "depName": "Gopkg.toml", + "fileNameSuffix": ".go" } ]'; diff --git a/pkg/functions/params.go b/pkg/functions/params.go new file mode 100644 index 000000000..87fa86105 --- /dev/null +++ b/pkg/functions/params.go @@ -0,0 +1,46 @@ +/* +Copyright (c) 2016-2017 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package functions + +import ( + "golang.org/x/net/context" + "net/http" +) + +// Extension includes a reference to the Event request and its Context (to handle timeouts) +type Extension struct { + Request *http.Request + Context context.Context +} + +// Event includes information about the event source +type Event struct { + Data string + EventID string + EventType string + EventTime string + EventNamespace string + Extensions Extension +} + +// Context includes information about the function environment +type Context struct { + FunctionName string + Timeout string + Runtime string + MemoryLimit string +} diff --git a/pkg/langruntime/langruntime.go b/pkg/langruntime/langruntime.go index e7875b817..0013758df 100644 --- a/pkg/langruntime/langruntime.go +++ b/pkg/langruntime/langruntime.go @@ -37,6 +37,7 @@ type ImageSecret struct { // and the supported versions type RuntimeInfo struct { ID string `yaml:"ID"` + Compiled bool `yaml:"compiled"` Versions []RuntimeVersion `yaml:"versions"` DepName string `yaml:"depName"` FileNameSuffix string `yaml:"fileNameSuffix"` @@ -210,6 +211,8 @@ func (l *Langruntimes) GetBuildContainer(runtime string, env []v1.EnvVar, instal case strings.Contains(runtime, "php"): command = "composer install -d " + installVolume.MountPath + case strings.Contains(runtime, "go"): + command = "cd $GOPATH/src/kubeless && dep ensure > /dev/termination-log 2>&1" } return v1.Container{ @@ -249,3 +252,41 @@ func (l *Langruntimes) UpdateDeployment(dpm *v1beta1.Deployment, depsPath, runti }) } } + +// RequiresCompilation returns if the given runtime requires compilation +func (l *Langruntimes) RequiresCompilation(runtime string) bool { + required := false + for _, runtimeInf := range l.AvailableRuntimes { + if strings.Contains(runtime, runtimeInf.ID) { + required = runtimeInf.Compiled + break + } + } + return required +} + +// GetCompilationContainer returns a Container definition based on a runtime +func (l *Langruntimes) GetCompilationContainer(runtime, funcName string, installVolume v1.VolumeMount) (v1.Container, error) { + versionInf, err := l.findRuntimeVersion(runtime) + if err != nil { + return v1.Container{}, err + } + var command string + switch { + case strings.Contains(runtime, "go"): + command = fmt.Sprintf( + "sed 's/<>/%s/g' $GOPATH/src/controller/kubeless.tpl.go > $GOPATH/src/controller/kubeless.go && "+ + "go build -o %s/server $GOPATH/src/controller/kubeless.go > /dev/termination-log 2>&1", funcName, installVolume.MountPath) + default: + return v1.Container{}, fmt.Errorf("Not found a valid compilation step for %s", runtime) + } + return v1.Container{ + Name: "compile", + Image: versionInf.InitImage, + Command: []string{"sh", "-c"}, + Args: []string{command}, + VolumeMounts: []v1.VolumeMount{installVolume}, + ImagePullPolicy: v1.PullIfNotPresent, + WorkingDir: installVolume.MountPath, + }, nil +} diff --git a/pkg/utils/k8sutil.go b/pkg/utils/k8sutil.go index 8a61af7a8..b81fe0356 100644 --- a/pkg/utils/k8sutil.go +++ b/pkg/utils/k8sutil.go @@ -809,6 +809,19 @@ func populatePodSpec(funcObj *kubelessApi.Function, lr *langruntime.Langruntimes depsInstallContainer, ) } + + // add compilation init container if needed + if lr.RequiresCompilation(funcObj.Spec.Runtime) { + _, funcName, err := splitHandler(funcObj.Spec.Handler) + compContainer, err := lr.GetCompilationContainer(funcObj.Spec.Runtime, funcName, runtimeVolumeMount) + if err != nil { + return err + } + result.InitContainers = append( + result.InitContainers, + compContainer, + ) + } return nil } diff --git a/script/validate-gofmt b/script/validate-gofmt index 3dc897014..addd85225 100755 --- a/script/validate-gofmt +++ b/script/validate-gofmt @@ -17,7 +17,7 @@ source "$(dirname "$BASH_SOURCE")/.validate" IFS=$'\n' -files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor/' || true) ) +files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor/\|kubeless.tpl.go' || true) ) unset IFS badFiles=() diff --git a/script/validate-lint b/script/validate-lint index bcb5a4256..8387072da 100755 --- a/script/validate-lint +++ b/script/validate-lint @@ -20,7 +20,7 @@ source "$(dirname "$BASH_SOURCE")/.validate" # of subpackages, vendoring excluded, as given by: # IFS=$'\n' -files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor/\|^pkg/client/\|^pkg/apis/kubeless/v1beta1/zz_generated.deepcopy.go\|^integration' || true) ) +files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor/\|^pkg/client/\|^pkg/apis/kubeless/v1beta1/zz_generated.deepcopy.go\|^integration\|kubeless.tpl.go' || true) ) unset IFS errors=() diff --git a/script/validate-vet b/script/validate-vet index 1d2684e48..8585e67a6 100755 --- a/script/validate-vet +++ b/script/validate-vet @@ -17,7 +17,7 @@ source "$(dirname "$BASH_SOURCE")/.validate" IFS=$'\n' -files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor/' || true) ) +files=( $(validate_diff --diff-filter=ACMR --name-only -- '*.go' | grep -v '^vendor/\|kubeless.tpl.go' || true) ) unset IFS failed=0 diff --git a/tests/integration-tests-kafka.bats b/tests/integration-tests-kafka.bats index 21d30bcc4..550d33f9e 100644 --- a/tests/integration-tests-kafka.bats +++ b/tests/integration-tests-kafka.bats @@ -51,6 +51,11 @@ load ../script/libtest verify_function pubsub-ruby kubeless_function_delete pubsub-ruby } +@test "Test function: pubsub-go" { + deploy_function pubsub-go + verify_function pubsub-go + kubeless_function_delete pubsub-go +} @test "Test topic list" { wait_for_kubeless_kafka_server_ready for topic in topic1 topic2; do diff --git a/tests/integration-tests-prebuilt.bats b/tests/integration-tests-prebuilt.bats index 69fbfc43f..5ab5e5fef 100644 --- a/tests/integration-tests-prebuilt.bats +++ b/tests/integration-tests-prebuilt.bats @@ -38,6 +38,15 @@ load ../script/libtest kubectl get deployment -o yaml get-python | grep image | grep $(minikube ip):5000 } +@test "Deploy a Golang function using the build system" { + deploy_function get-go-deps + wait_for_job get-go-deps + # Speed up pod start when the image is ready + restart_function get-go-deps + verify_function get-go-deps + kubectl get deployment -o yaml get-go-deps | grep image | grep $(minikube ip):5000 +} + @test "Test no-errors" { if kubectl logs -n kubeless -l kubeless=controller | grep "level=error"; then echo "Found errors in the controller logs" diff --git a/tests/integration-tests.bats b/tests/integration-tests.bats index 5df2281d9..e39d23a6d 100644 --- a/tests/integration-tests.bats +++ b/tests/integration-tests.bats @@ -32,12 +32,17 @@ load ../script/libtest deploy_function get-php deploy_function get-php-deps deploy_function timeout-php + deploy_function get-go + deploy_function get-go-custom-port + deploy_function get-go-deps + deploy_function timeout-go deploy_function get-python-metadata deploy_function post-python deploy_function post-python-custom-port deploy_function post-nodejs deploy_function post-ruby deploy_function post-php + deploy_function post-go deploy_function custom-get-python } @test "Test function: get-python" { @@ -113,6 +118,25 @@ load ../script/libtest verify_function timeout-php kubeless_function_delete timeout-php } +@test "Test function: get-go" { + verify_function get-go + kubeless_function_delete get-go +} +@test "Test function: get-go-deps" { + verify_function get-go-deps +} +@test "Test function: get-go-custom-port" { + verify_function get-go-custom-port + kubeless_function_delete get-go-custom-port +} +@test "Test function: timeout-go" { + verify_function timeout-go + kubeless_function_delete timeout-go +} +@test "Test function: post-go" { + verify_function post-go + kubeless_function_delete post-go +} @test "Test function: get-dotnetcore" { skip "This test is flaky until kubeless/kubeless/issues/395 is fixed" test_kubeless_function get-dotnetcore