diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..a538cf46d --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,306 @@ +version: 2 + +## Definitions +build_allways: &build_allways + filters: + tags: + only: /.*/ +defaults: &defaults + environment: + CONTROLLER_IMAGE_NAME: bitnami/kubeless-controller-manager + BUILDER_IMAGE_NAME: kubeless/function-image-builder + KAFKA_CONTROLLER_IMAGE_NAME: bitnami/kafka-trigger-controller + NATS_CONTROLLER_IMAGE_NAME: bitnami/nats-trigger-controller + CGO_ENABLED: "0" + TEST_DEBUG: "1" + GKE_VERSION: 1.8.8-gke.0 + MINIKUBE_VERSION: v0.25.2 +exports: &exports + # It is not possible to resolve env vars in the environment section: + # https://discuss.circleci.com/t/using-environment-variables-in-config-yml-not-working/14237 + run: | + CONTROLLER_TAG=${CIRCLE_TAG:-build-$CIRCLE_WORKFLOW_ID} + echo "export CONTROLLER_TAG=${CONTROLLER_TAG}" >> $BASH_ENV + echo "export CONTROLLER_IMAGE=${CONTROLLER_IMAGE_NAME}:${CONTROLLER_TAG}" >> $BASH_ENV + echo "export BUILDER_IMAGE=${BUILDER_IMAGE_NAME}:${CONTROLLER_TAG}" >> $BASH_ENV + echo "export KAFKA_CONTROLLER_IMAGE=${KAFKA_CONTROLLER_IMAGE_NAME}:${CONTROLLER_TAG}" >> $BASH_ENV + echo "export NATS_CONTROLLER_IMAGE=${NATS_CONTROLLER_IMAGE_NAME}:${CONTROLLER_TAG}" >> $BASH_ENV + echo "export KUBECFG_JPATH=/go/src/github.com/kubeless/kubeless/ksonnet-lib" >> $BASH_ENV + echo "export PATH=$(pwd)/bats/libexec:$PATH" >> $BASH_ENV +restore_workspace: &restore_workspace + run: | + make bootstrap + sudo cp -r /tmp/go/bin/* /usr/local/bin/ + cp -r /tmp/go/src/github.com/kubeless/kubeless/*yaml . +should_test: &should_test + run: | + case $CIRCLE_JOB in + # In case of GKE we will only want to build if it is + # a build of a branch in the kubeless/kubeless repository + GKE) + echo $CIRCLE_PULL_REQUESTS + if [[ -n "$GKE_ADMIN" && -z "$CIRCLE_PULL_REQUESTS" ]]; then + export SHOULD_TEST=1 + fi + ;; + # In kase of minikube+kafka we want to test it if + # it is a Pull Request related to Kafka (discovered from the PR title) + # or if the build is from the "master" branch + minikube_kafka) + if [[ -n "$CIRCLE_PULL_REQUESTS" ]]; then + pr_kafka_title=$(curl -H "Authorization: token $ACCESS_TOKEN" "https://api.github.com/repos/$TRAVIS_REPO_SLUG/pulls/${CIRCLE_PR_NUMBER}" | grep title || true) + fi + if [[ -z "$CIRCLE_PULL_REQUESTS" || "$pr_kafka_title" == "" || "$pr_kafka_title" =~ ^.*(Kafka|kafka|KAFKA).*$ ]]; then + export SHOULD_TEST=1 + fi + ;; + # In kase of minikube+NATS we want to test it if + # it is a Pull Request related to NATS (discovered from the PR title) + # or if the build is from the "master" branch + minikube_nats) + if [[ -n "$CIRCLE_PULL_REQUESTS" ]]; then + pr_nats_title=$(curl -H "Authorization: token $ACCESS_TOKEN" "https://api.github.com/repos/$TRAVIS_REPO_SLUG/pulls/${TRAVIS_PULL_REQUEST}" | grep title || true) + fi + if [[ -z "$CIRCLE_PULL_REQUESTS" || "$pr_nats_title" == "" || "$pr_nats_title" =~ ^.*(Nats|nats|NATS).*$ ]]; then + export SHOULD_TEST=1 + fi + ;; + esac + echo "Should test? $SHOULD_TEST" + if [[ "$SHOULD_TEST" != "1" ]]; then + circleci step halt + fi +#### End of definitions + +workflows: + version: 2 + kubeless: + jobs: + - build: + <<: *build_allways + - build_images: + <<: *build_allways + requires: + - build + - minikube: + <<: *build_allways + requires: + - build_images + - minikube_core_triggers: + <<: *build_allways + requires: + - build_images + - minikube_build_functions: + <<: *build_allways + requires: + - build_images + - minikube_kafka: + <<: *build_allways + requires: + - build_images + - minikube_nats: + <<: *build_allways + requires: + - build_images + - GKE: + <<: *build_allways + requires: + - build_images + - push_latest_images: + filters: + branches: + only: master + requires: + - minikube + - minikube_core_triggers + - minikube_build_functions + - minikube_kafka + - minikube_nats + - GKE + # TODO: Enable this when Travis release is disabled + # - release: + # filters: + # tags: + # only: /v.*/ + # branches: + # ignore: /.*/ + # requires: + # - minikube + # - minikube_core_triggers + # - minikube_build_functions + # - minikube_kafka + # - minikube_nats + # - GKE +jobs: + build: + <<: *defaults + docker: + - image: circleci/golang:1.9 + working_directory: /go/src/github.com/kubeless/kubeless + steps: + - checkout + - <<: *exports + - run: make bootstrap + - run: make VERSION=${CONTROLLER_TAG} binary + - run: make test + - run: make validation + - run: make all-yaml + - run: | + mkdir build-manifests + manifests=( + kubeless.yaml kubeless-non-rbac.yaml kubeless-openshift.yaml + kafka-zookeeper.yaml kafka-zookeeper-openshift.yaml + nats.yaml + ) + for f in "${manifests[@]}"; do + sed -i.bak 's/:latest/'":${CONTROLLER_TAG}"'/g' $f + cp $f build-manifests/ + done + - persist_to_workspace: + root: /go + paths: + - bin + - src/github.com/kubeless/kubeless/*yaml + - store_artifacts: + path: /go/bin/kubeless + - store_artifacts: + path: build-manifests + build_images: + <<: *defaults + docker: + - image: circleci/golang:1.9 + working_directory: /go/src/github.com/kubeless/kubeless + steps: + - setup_remote_docker + - checkout + - <<: *exports + - run: make bootstrap + - run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + - run: make controller-image CONTROLLER_IMAGE=$CONTROLLER_IMAGE + - run: docker push $CONTROLLER_IMAGE + - run: make kafka-controller-image KAFKA_CONTROLLER_IMAGE=$KAFKA_CONTROLLER_IMAGE + - run: docker push $KAFKA_CONTROLLER_IMAGE + - run: make nats-controller-image NATS_CONTROLLER_IMAGE=$NATS_CONTROLLER_IMAGE + - run: docker push $NATS_CONTROLLER_IMAGE + - run: make function-image-builder FUNCTION_IMAGE_BUILDER=$BUILDER_IMAGE + - run: docker push $BUILDER_IMAGE + minikube: + <<: *defaults + machine: true + steps: + - checkout + - attach_workspace: + at: /tmp/go + - <<: *exports + - <<: *restore_workspace + - run: ./script/integration-tests minikube deployment + - run: ./script/integration-tests minikube basic + minikube_core_triggers: + <<: *defaults + machine: true + steps: + - checkout + - <<: *exports + - attach_workspace: + at: /tmp/go + - <<: *restore_workspace + - run: sudo apt-get update -y + - run: sudo apt-get install -y apache2-utils + - run: ./script/integration-tests minikube deployment + - run: ./script/integration-tests minikube cronjob + - run: ./script/integration-tests minikube http + minikube_build_functions: + <<: *defaults + machine: true + steps: + - checkout + - <<: *exports + - attach_workspace: + at: /tmp/go + - <<: *restore_workspace + - run: "echo '{\"host\": \"unix:///var/run/docker.sock\", \"storage-driver\": \"overlay2\", \"insecure-registries\" : [\"0.0.0.0/0\"]}' > /tmp/daemon.json" + - run: sudo mv /tmp/daemon.json /etc/docker/daemon.json + - run: sudo service docker restart + - run: docker info + - run: docker run -d -p 5000:5000 --restart=always --name registry -v /data/docker-registry:/var/lib/registry registry:2 + - run: "sed -i.bak 's/enable-build-step: \"false\"/enable-build-step: \"true\"/g' kubeless.yaml" + - run: "sed -i.bak 's/function-registry-tls-verify: \"true\"/function-registry-tls-verify: \"false\"/g' kubeless.yaml" + - run: ./script/integration-tests minikube deployment + - run: ./script/integration-tests minikube prebuilt_functions + minikube_kafka: + <<: *defaults + machine: true + steps: + - <<: *should_test + - checkout + - <<: *exports + - attach_workspace: + at: /tmp/go + - <<: *restore_workspace + - run: ./script/integration-tests minikube deployment + - run: ./script/integration-tests minikube kafka + minikube_nats: + <<: *defaults + machine: true + steps: + - <<: *should_test + - checkout + - <<: *exports + - attach_workspace: + at: /tmp/go + - <<: *restore_workspace + - run: ./script/integration-tests minikube deployment + - run: ./script/integration-tests minikube nats + GKE: + <<: *defaults + docker: + - image: circleci/golang:1.9 + steps: + - <<: *should_test + - checkout + - <<: *exports + - attach_workspace: + at: /tmp/go + - <<: *restore_workspace + - run: ./script/enable-gcloud.sh $(pwd) > /dev/null + - run: echo "export ESCAPED_GKE_CLUSTER=$(echo ${GKE_CLUSTER}-ci-${CIRCLE_BRANCH} | sed 's/[^a-z0-9-]//g')" >> $BASH_ENV + - run: ./script/start-gke-env.sh $ESCAPED_GKE_CLUSTER $ZONE $GKE_VERSION $GKE_ADMIN > /dev/null + - run: ./script/integration-tests gke_${GKE_PROJECT}_${ZONE}_${ESCAPED_GKE_CLUSTER} deployment + - run: ./script/integration-tests gke_${GKE_PROJECT}_${ZONE}_${ESCAPED_GKE_CLUSTER} basic + - run: ./script/integration-tests gke_${GKE_PROJECT}_${ZONE}_${ESCAPED_GKE_CLUSTER} cronjob + - run: ./script/integration-tests gke_${GKE_PROJECT}_${ZONE}_${ESCAPED_GKE_CLUSTER} kafka + push_latest_images: + <<: *defaults + docker: + - image: circleci/golang:1.9 + steps: + - <<: *exports + - setup_remote_docker + - run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + - run: | + images=( + $CONTROLLER_IMAGE_NAME + $BUILDER_IMAGE_NAME + $KAFKA_CONTROLLER_IMAGE_NAME + $NATS_CONTROLLER_IMAGE_NAME + ) + for image in "${images[@]}"; do + echo "Pulling ${image}:${CONTROLLER_TAG}" + docker pull ${image}:${CONTROLLER_TAG} + docker tag ${image}:${CONTROLLER_TAG} ${image}:latest + docker push ${image}:latest + done + release: + <<: *defaults + docker: + - image: circleci/golang:1.9 + working_directory: /go/src/github.com/kubeless/kubeless + steps: + - <<: *exports + - checkout + - attach_workspace: + at: /tmp/go + - <<: *restore_workspace + - run: make VERSION=${CIRCLE_TAG} binary-cross + - run: for d in bundles/kubeless_*; do zip -r9 $d.zip $d/; done + - run: ./script/create_release.sh ${CIRCLE_TAG} diff --git a/Makefile b/Makefile index aab1ef8cb..eb3be8fb3 100644 --- a/Makefile +++ b/Makefile @@ -121,14 +121,14 @@ bootstrap: bats ksonnet-lib go get github.com/golang/lint/golint @if ! which kubecfg >/dev/null; then \ - wget -q -O $$GOPATH/bin/kubecfg https://github.com/ksonnet/kubecfg/releases/download/v0.6.0/kubecfg-$$(go env GOOS)-$$(go env GOARCH); \ - chmod +x $$GOPATH/bin/kubecfg; \ + sudo wget -q -O /usr/local/bin/kubecfg https://github.com/ksonnet/kubecfg/releases/download/v0.6.0/kubecfg-$$(go env GOOS)-$$(go env GOARCH); \ + sudo chmod +x /usr/local/bin/kubecfg; \ fi @if ! which kubectl >/dev/null; then \ KUBECTL_VERSION=$$(wget -qO- https://storage.googleapis.com/kubernetes-release/release/stable.txt); \ - wget -q -O $$GOPATH/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$$KUBECTL_VERSION/bin/$$(go env GOOS)/$$(go env GOARCH)/kubectl; \ - chmod +x $$GOPATH/bin/kubectl; \ + sudo wget -q -O /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$$KUBECTL_VERSION/bin/$$(go env GOOS)/$$(go env GOARCH)/kubectl; \ + sudo chmod +x /usr/local/bin/kubectl; \ fi build_and_test: diff --git a/chart/README.md b/chart/README.md index ad0adf06b..9b5226edf 100644 --- a/chart/README.md +++ b/chart/README.md @@ -15,3 +15,4 @@ helm init helm install --name kubeless --namespace kubeless ./kubeless ``` +After that, if you are having trouble deploying Kafka and Zookeeper, please check the specific guide [here](/docs/troubleshooting/#kafka-and-zookeeper-persistent-volume-creation) to create appropriate disks and PVs. If you are running Kubernetes in GKE,you can provision those persistent volumes manually deploying the manifests present in the [misc folder](https://github.com/kubeless/kubeless/tree/master/docs/misc). If you use other cloud provider, check [kubernetes docs](https://kubernetes.io/docs/concepts/storage/volumes/) to create these required volumes. diff --git a/chart/kubeless/README.md b/chart/kubeless/README.md index 4383cb3de..99214e174 100644 --- a/chart/kubeless/README.md +++ b/chart/kubeless/README.md @@ -6,4 +6,4 @@ It installs: * The controller * The Kubeless configuration * The UI -* A single node Kafka and Zookeeper setup +* A single node Kafka and Zookeeper setup \ No newline at end of file diff --git a/cmd/kubeless/autoscale/autoscaleDelete.go b/cmd/kubeless/autoscale/autoscaleDelete.go index b24841773..16d454e45 100644 --- a/cmd/kubeless/autoscale/autoscaleDelete.go +++ b/cmd/kubeless/autoscale/autoscaleDelete.go @@ -54,9 +54,9 @@ var autoscaleDeleteCmd = &cobra.Command{ if err != nil { logrus.Fatal(err) } - logrus.Infof("Removed Autoscaling rule from %s", funcName) + logrus.Infof("Remove Autoscaling rule from %s successfully", funcName) } else { - logrus.Fatalf("Not found an auto scale definition for %s", funcName) + logrus.Fatalf("Not found an autoscale definition for %s", funcName) } }, } diff --git a/cmd/kubeless/autoscale/autoscaleList.go b/cmd/kubeless/autoscale/autoscaleList.go index f54d02f55..89f769bfb 100644 --- a/cmd/kubeless/autoscale/autoscaleList.go +++ b/cmd/kubeless/autoscale/autoscaleList.go @@ -85,7 +85,7 @@ func printAutoscale(w io.Writer, ass []v2beta1.HorizontalPodAutoscaler, output s m := "" v := "" if len(i.Spec.Metrics) == 0 { - logrus.Errorf("The function autoscale %s isn't in correct format. It has no metric defined.", i.Name) + logrus.Errorf("The autoscale %s has bad format. It has no metric defined.", i.Name) continue } if i.Spec.Metrics[0].Object != nil { @@ -115,7 +115,7 @@ func printAutoscale(w io.Writer, ass []v2beta1.HorizontalPodAutoscaler, output s } fmt.Fprintln(w, string(b)) default: - return fmt.Errorf("Wrong output format. Please use only json|yaml") + return fmt.Errorf("Wrong output format. Only accept json|yaml file") } } } diff --git a/cmd/kubeless/function/deploy.go b/cmd/kubeless/function/deploy.go index c49c5ce33..d825edf48 100644 --- a/cmd/kubeless/function/deploy.go +++ b/cmd/kubeless/function/deploy.go @@ -17,7 +17,7 @@ limitations under the License. package function import ( - "io/ioutil" + "fmt" "strings" kubelessApi "github.com/kubeless/kubeless/pkg/apis/kubeless/v1beta1" @@ -113,6 +113,16 @@ var deployCmd = &cobra.Command{ logrus.Fatal(err) } + imagePullPolicy, err := cmd.Flags().GetString("image-pull-policy") + if err != nil { + logrus.Fatal(err) + } + + if imagePullPolicy != "IfNotPresent" && imagePullPolicy != "Always" && imagePullPolicy != "Never" { + err := fmt.Errorf("image-pull-policy must be {IfNotPresent|Always|Never}") + logrus.Fatal(err) + } + mem, err := cmd.Flags().GetString("memory") if err != nil { logrus.Fatal(err) @@ -142,11 +152,14 @@ var deployCmd = &cobra.Command{ funcDeps := "" if deps != "" { - bytes, err := ioutil.ReadFile(deps) + contentType, err := getContentType(deps) + if err != nil { + logrus.Fatal(err) + } + funcDeps, _, err = parseContent(deps, contentType) if err != nil { - logrus.Fatalf("Unable to read file %s: %v", deps, err) + logrus.Fatal(err) } - funcDeps = string(bytes) } if runtime == "" && runtimeImage == "" { @@ -163,7 +176,7 @@ var deployCmd = &cobra.Command{ "function": funcName, } - f, err := getFunctionDescription(cli, funcName, ns, handler, file, funcDeps, runtime, runtimeImage, mem, cpu, timeout, port, headless, envs, labels, secrets, defaultFunctionSpec) + f, err := getFunctionDescription(cli, funcName, ns, handler, file, funcDeps, runtime, runtimeImage, mem, cpu, timeout, imagePullPolicy, port, headless, envs, labels, secrets, defaultFunctionSpec) if err != nil { logrus.Fatal(err) @@ -209,7 +222,7 @@ var deployCmd = &cobra.Command{ func init() { deployCmd.Flags().StringP("runtime", "", "", "Specify runtime") deployCmd.Flags().StringP("handler", "", "", "Specify handler") - deployCmd.Flags().StringP("from-file", "", "", "Specify code file") + deployCmd.Flags().StringP("from-file", "", "", "Specify code file or a URL to the code file") deployCmd.Flags().StringSliceP("label", "", []string{}, "Specify labels of the function. Both separator ':' and '=' are allowed. For example: --label foo1=bar1,foo2:bar2") deployCmd.Flags().StringSliceP("secrets", "", []string{}, "Specify Secrets to be mounted to the functions container. For example: --secrets mySecret") deployCmd.Flags().StringArrayP("env", "", []string{}, "Specify environment variable of the function. Both separator ':' and '=' are allowed. For example: --env foo1=bar1,foo2:bar2") @@ -219,6 +232,7 @@ func init() { deployCmd.Flags().StringP("memory", "", "", "Request amount of memory, which is measured in bytes, for the function. It is expressed as a plain integer or a fixed-point interger with one of these suffies: E, P, T, G, M, K, Ei, Pi, Ti, Gi, Mi, Ki") deployCmd.Flags().StringP("cpu", "", "", "Request amount of cpu for the function, which is measured in units of cores. Please see the following link for more information: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu") deployCmd.Flags().StringP("runtime-image", "", "", "Custom runtime image") + deployCmd.Flags().StringP("image-pull-policy", "", "Always", "Image pull policy") deployCmd.Flags().StringP("timeout", "", "180", "Maximum timeout (in seconds) for the function to complete its execution") deployCmd.Flags().Bool("headless", false, "Deploy http-based function without a single service IP and load balancing support from Kubernetes. See: https://kubernetes.io/docs/concepts/services-networking/service/#headless-services") deployCmd.Flags().Int32("port", 8080, "Deploy http-based function with a custom port") diff --git a/cmd/kubeless/function/function.go b/cmd/kubeless/function/function.go index 8d2863c4a..b46f88d82 100644 --- a/cmd/kubeless/function/function.go +++ b/cmd/kubeless/function/function.go @@ -23,6 +23,8 @@ import ( "fmt" "io" "io/ioutil" + "net/http" + "net/url" "os" "path" "strings" @@ -120,21 +122,88 @@ func getFileSha256(file string) (string, error) { return "sha256:" + checksum, err } -func getContentType(filename string, fbytes []byte) string { +func getSha256(bytes []byte) (string, error) { + h := sha256.New() + _, err := h.Write(bytes) + if err != nil { + return "", err + } + checksum := hex.EncodeToString(h.Sum(nil)) + return "sha256:" + checksum, nil +} + +func getContentType(filename string) (string, error) { var contentType string - isText := utf8.ValidString(string(fbytes)) - if isText { - contentType = "text" - } else { - contentType = "base64" - if path.Ext(filename) == ".zip" { + + if strings.Index(filename, "http://") == 0 || strings.Index(filename, "https://") == 0 { + contentType = "url" + if path.Ext(strings.Split(filename, "?")[0]) == ".zip" { contentType += "+zip" } + } else { + fbytes, err := ioutil.ReadFile(filename) + if err != nil { + return "", err + } + isText := utf8.ValidString(string(fbytes)) + if isText { + contentType = "text" + } else { + contentType = "base64" + if path.Ext(filename) == ".zip" { + contentType += "+zip" + } + } + } + return contentType, nil +} + +func parseContent(file, contentType string) (string, string, error) { + var checksum, content string + + if strings.Contains(contentType, "url") { + + functionURL, err := url.Parse(file) + if err != nil { + return "", "", err + } + resp, err := http.Get(functionURL.String()) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + + functionBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", "", err + } + content = string(functionBytes) + checksum, err = getSha256(functionBytes) + if err != nil { + return "", "", err + } + + } else { + + functionBytes, err := ioutil.ReadFile(file) + if err != nil { + return "", "", err + } + if contentType == "text" { + content = string(functionBytes) + } else { + content = base64.StdEncoding.EncodeToString(functionBytes) + } + checksum, err = getFileSha256(file) + if err != nil { + return "", "", err + } } - return contentType + + return content, checksum, nil } -func getFunctionDescription(cli kubernetes.Interface, funcName, ns, handler, file, deps, runtime, runtimeImage, mem, cpu, timeout string, port int32, headless bool, envs, labels []string, secrets []string, defaultFunction kubelessApi.Function) (*kubelessApi.Function, error) { +func getFunctionDescription(cli kubernetes.Interface, funcName, ns, handler, file, deps, runtime, runtimeImage, mem, cpu, timeout string, imagePullPolicy string, port int32, headless bool, envs, labels []string, secrets []string, defaultFunction kubelessApi.Function) (*kubelessApi.Function, error) { function := defaultFunction function.TypeMeta = metav1.TypeMeta{ @@ -146,20 +215,22 @@ func getFunctionDescription(cli kubernetes.Interface, funcName, ns, handler, fil } if file != "" { - functionBytes, err := ioutil.ReadFile(file) + contentType, err := getContentType(file) if err != nil { return nil, err } - function.Spec.FunctionContentType = getContentType(file, functionBytes) - if function.Spec.FunctionContentType == "text" { - function.Spec.Function = string(functionBytes) - } else { - function.Spec.Function = base64.StdEncoding.EncodeToString(functionBytes) - } - function.Spec.Checksum, err = getFileSha256(file) + functionContent, checksum, err := parseContent(file, contentType) if err != nil { return nil, err } + if strings.Contains(contentType, "url") { + // set the function to be the URL provided on the command line + function.Spec.Function = file + } else { + function.Spec.Function = functionContent + } + function.Spec.Checksum = checksum + function.Spec.FunctionContentType = contentType } if deps != "" { @@ -225,9 +296,10 @@ func getFunctionDescription(cli kubernetes.Interface, funcName, ns, handler, fil } function.Spec.Deployment.Spec.Template.Spec.Containers = []v1.Container{ { - Env: funcEnv, - Resources: resources, - Image: runtimeImage, + ImagePullPolicy: v1.PullPolicy(imagePullPolicy), + Env: funcEnv, + Resources: resources, + Image: runtimeImage, }, } diff --git a/cmd/kubeless/function/function_test.go b/cmd/kubeless/function/function_test.go index adb9be9fd..9232e8cd9 100644 --- a/cmd/kubeless/function/function_test.go +++ b/cmd/kubeless/function/function_test.go @@ -18,8 +18,11 @@ package function import ( "archive/zip" + "fmt" "io" "io/ioutil" + "net/http" + "net/http/httptest" "os" "reflect" "testing" @@ -100,7 +103,7 @@ func TestGetFunctionDescription(t *testing.T) { file.Close() defer os.Remove(file.Name()) // clean up - result, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) + result, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "10", "Always", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) if err != nil { t.Error(err) @@ -147,7 +150,8 @@ func TestGetFunctionDescription(t *testing.T) { v1.ResourceCPU: parsedCPU, }, }, - Image: "test-image", + Image: "test-image", + ImagePullPolicy: v1.PullAlways, VolumeMounts: []v1.VolumeMount{ { Name: "secretName-vol", @@ -186,7 +190,7 @@ func TestGetFunctionDescription(t *testing.T) { } // It should take the default values - result2, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "", "", "", "", "", "", "", "", 8080, false, []string{}, []string{}, []string{}, expectedFunction) + result2, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "", "", "", "", "", "", "", "", "Always", 8080, false, []string{}, []string{}, []string{}, expectedFunction) if err != nil { t.Error(err) @@ -207,7 +211,7 @@ func TestGetFunctionDescription(t *testing.T) { file.Close() defer os.Remove(file.Name()) // clean up - result3, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler2", file.Name(), "dependencies2", "runtime2", "test-image2", "256Mi", "100m", "20", 8080, false, []string{"TEST=2"}, []string{"test=2"}, []string{"secret2"}, expectedFunction) + result3, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler2", file.Name(), "dependencies2", "runtime2", "test-image2", "256Mi", "100m", "20", "Always", 8080, false, []string{"TEST=2"}, []string{"test=2"}, []string{"secret2"}, expectedFunction) if err != nil { t.Error(err) @@ -254,7 +258,8 @@ func TestGetFunctionDescription(t *testing.T) { v1.ResourceCPU: parsedCPU2, }, }, - Image: "test-image2", + Image: "test-image2", + ImagePullPolicy: v1.PullAlways, VolumeMounts: []v1.VolumeMount{ { Name: "secretName-vol", @@ -332,7 +337,7 @@ func TestGetFunctionDescription(t *testing.T) { file.Close() zipW.Close() - result4, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", newfile.Name(), "dependencies", "runtime", "", "", "", "", 8080, false, []string{}, []string{}, []string{}, expectedFunction) + result4, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", newfile.Name(), "dependencies", "runtime", "", "", "", "", "Always", 8080, false, []string{}, []string{}, []string{}, expectedFunction) if err != nil { t.Error(err) } @@ -341,7 +346,7 @@ func TestGetFunctionDescription(t *testing.T) { } // It should maintain previous HPA definition - result5, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{}, kubelessApi.Function{ + result5, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "10", "Always", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{}, kubelessApi.Function{ Spec: kubelessApi.FunctionSpec{ HorizontalPodAutoscaler: v2beta1.HorizontalPodAutoscaler{ @@ -356,7 +361,7 @@ func TestGetFunctionDescription(t *testing.T) { } // It should set the Port and headless service properly - result6, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "", 9091, true, []string{}, []string{}, []string{}, kubelessApi.Function{}) + result6, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "", "Always", 9091, true, []string{}, []string{}, []string{}, kubelessApi.Function{}) expectedPort := v1.ServicePort{ Name: "http-function-port", Port: 9091, @@ -370,4 +375,123 @@ func TestGetFunctionDescription(t *testing.T) { if result6.Spec.ServiceSpec.ClusterIP != v1.ClusterIPNone { t.Errorf("Unexpected clusterIP %v", result6.Spec.ServiceSpec.ClusterIP) } + + // it should create a function from a URL + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "function") + })) + defer ts.Close() + + expectedURLFunction := kubelessApi.Function{ + TypeMeta: metav1.TypeMeta{ + Kind: "Function", + APIVersion: "kubeless.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + Labels: map[string]string{ + "test": "1", + }, + }, + Spec: kubelessApi.FunctionSpec{ + Handler: "file.handler", + Runtime: "runtime", + Function: ts.URL, + Checksum: "sha256:78f9ac018e554365069108352dacabb7fbd15246edf19400677e3b54fe24e126", + FunctionContentType: "url", + Deps: "dependencies", + Timeout: "10", + Deployment: v1beta1.Deployment{ + Spec: v1beta1.DeploymentSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Env: []v1.EnvVar{{ + Name: "TEST", + Value: "1", + }}, + Resources: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: parsedMem, + v1.ResourceCPU: parsedCPU, + }, + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: parsedMem, + v1.ResourceCPU: parsedCPU, + }, + }, + Image: "test-image", + ImagePullPolicy: v1.PullAlways, + VolumeMounts: []v1.VolumeMount{ + { + Name: "secretName-vol", + MountPath: "/secretName", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "secretName-vol", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "secretName", + }, + }, + }, + }, + }, + }, + }, + }, + ServiceSpec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {Name: "http-function-port", Protocol: "TCP", Port: 8080, TargetPort: intstr.FromInt(8080)}, + }, + Selector: map[string]string{ + "test": "1", + }, + Type: v1.ServiceTypeClusterIP, + }, + }, + } + + result7, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", ts.URL, "dependencies", "runtime", "test-image", "128Mi", "", "10", "Always", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) + + if err != nil { + t.Error(err) + } + + if !reflect.DeepEqual(expectedURLFunction, *result7) { + t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result7) + } + // end test + + // it should handle zip files from a URL and detect url+zip encoding + zipBytes, err := ioutil.ReadFile(newfile.Name()) + if err != nil { + t.Error(err) + } + + ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(zipBytes) + })) + defer ts2.Close() + + expectedURLFunction.Spec.FunctionContentType = "url+zip" + expectedURLFunction.Spec.Function = ts2.URL + "/test.zip" + result8, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", ts2.URL+"/test.zip", "dependencies", "runtime", "test-image", "128Mi", "", "10", "Always", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) + if err != nil { + t.Error(err) + } + if result8.Spec.FunctionContentType != "url+zip" { + t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result8) + } + if result8.Spec.Function != ts2.URL+"/test.zip" { + t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result8) + } + // end test + } diff --git a/cmd/kubeless/function/logs.go b/cmd/kubeless/function/logs.go index 7b8c9c4b0..3fdef2e10 100644 --- a/cmd/kubeless/function/logs.go +++ b/cmd/kubeless/function/logs.go @@ -57,7 +57,7 @@ var logsCmd = &cobra.Command{ } readyPod, err := utils.GetReadyPod(pods) if err != nil { - logrus.Fatalf("Can't find the function pod: %v", err) + logrus.Fatalf("No function pod is running: %v", err) } podLog := &v1.PodLogOptions{ Container: funcName, diff --git a/cmd/kubeless/function/update.go b/cmd/kubeless/function/update.go index c39cbb5ae..17a49e534 100644 --- a/cmd/kubeless/function/update.go +++ b/cmd/kubeless/function/update.go @@ -17,7 +17,7 @@ limitations under the License. package function import ( - "io/ioutil" + "fmt" "strings" "github.com/kubeless/kubeless/pkg/langruntime" @@ -93,6 +93,16 @@ var updateCmd = &cobra.Command{ logrus.Fatal(err) } + imagePullPolicy, err := cmd.Flags().GetString("image-pull-policy") + if err != nil { + logrus.Fatal(err) + } + + if imagePullPolicy != "IfNotPresent" && imagePullPolicy != "Always" && imagePullPolicy != "Never" { + err := fmt.Errorf("image-pull-policy must be {IfNotPresent|Always|Never}") + logrus.Fatal(err) + } + mem, err := cmd.Flags().GetString("memory") if err != nil { logrus.Fatal(err) @@ -114,11 +124,14 @@ var updateCmd = &cobra.Command{ } funcDeps := "" if deps != "" { - bytes, err := ioutil.ReadFile(deps) + contentType, err := getContentType(deps) + if err != nil { + logrus.Fatal(err) + } + funcDeps, _, err = parseContent(deps, contentType) if err != nil { - logrus.Fatalf("Unable to read file %s: %v", deps, err) + logrus.Fatal(err) } - funcDeps = string(bytes) } headless, err := cmd.Flags().GetBool("headless") if err != nil { @@ -137,7 +150,7 @@ var updateCmd = &cobra.Command{ logrus.Fatal(err) } - f, err := getFunctionDescription(cli, funcName, ns, handler, file, funcDeps, runtime, runtimeImage, mem, cpu, timeout, port, headless, envs, labels, secrets, previousFunction) + f, err := getFunctionDescription(cli, funcName, ns, handler, file, funcDeps, runtime, runtimeImage, mem, cpu, timeout, imagePullPolicy, port, headless, envs, labels, secrets, previousFunction) if err != nil { logrus.Fatal(err) } @@ -159,7 +172,7 @@ var updateCmd = &cobra.Command{ func init() { updateCmd.Flags().StringP("runtime", "", "", "Specify runtime") updateCmd.Flags().StringP("handler", "", "", "Specify handler") - updateCmd.Flags().StringP("from-file", "", "", "Specify code file") + updateCmd.Flags().StringP("from-file", "", "", "Specify code file or a URL to the code file") updateCmd.Flags().StringP("memory", "", "", "Request amount of memory for the function") updateCmd.Flags().StringP("cpu", "", "", "Request amount of cpu for the function.") updateCmd.Flags().StringSliceP("label", "", []string{}, "Specify labels of the function") @@ -168,6 +181,7 @@ func init() { updateCmd.Flags().StringP("namespace", "", "", "Specify namespace for the function") updateCmd.Flags().StringP("dependencies", "", "", "Specify a file containing list of dependencies for the function") updateCmd.Flags().StringP("runtime-image", "", "", "Custom runtime image") + updateCmd.Flags().StringP("image-pull-policy", "", "Always", "Image pull policy") updateCmd.Flags().StringP("timeout", "", "180", "Maximum timeout (in seconds) for the function to complete its execution") updateCmd.Flags().Bool("headless", false, "Deploy http-based function without a single service IP and load balancing support from Kubernetes. See: https://kubernetes.io/docs/concepts/services-networking/service/#headless-services") updateCmd.Flags().Int32("port", 8080, "Deploy http-based function with a custom port") diff --git a/docker/runtime/golang/Dockerfile b/docker/runtime/golang/Dockerfile index eef4de32a..405f0511b 100644 --- a/docker/runtime/golang/Dockerfile +++ b/docker/runtime/golang/Dockerfile @@ -1,5 +1,7 @@ FROM bitnami/minideb:jessie +RUN install_packages ca-certificates + USER 1000 CMD [ "/kubeless/server" ] diff --git a/docker/runtime/golang/kubeless.tpl.go b/docker/runtime/golang/kubeless.tpl.go index de4489fdb..699ab447d 100644 --- a/docker/runtime/golang/kubeless.tpl.go +++ b/docker/runtime/golang/kubeless.tpl.go @@ -126,6 +126,7 @@ func handler(w http.ResponseWriter, r *http.Request) { EventNamespace: r.Header.Get("event-namespace"), Extensions: functions.Extension{ Request: r, + Response: w, Context: ctx, }, } diff --git a/docker/runtime/nodejs/kubeless.js b/docker/runtime/nodejs/kubeless.js index 9e66b5fdd..8be50753b 100644 --- a/docker/runtime/nodejs/kubeless.js +++ b/docker/runtime/nodejs/kubeless.js @@ -84,7 +84,7 @@ function modExecute(handler, req, res, end) { 'event-time': req.get('event-time'), 'event-namespace': req.get('event-namespace'), data, - 'extensions': { request: req }, + 'extensions': { request: req, response: res }, }; Promise.resolve(func(event, context)) // Finalize diff --git a/docker/runtime/php/Controller.php b/docker/runtime/php/Controller.php index ee1d91995..88c2983a3 100644 --- a/docker/runtime/php/Controller.php +++ b/docker/runtime/php/Controller.php @@ -40,7 +40,7 @@ public function __construct() * * @return void */ - private function runFunction(Request $request) + private function runFunction(Request $request, Response $response) { set_time_limit($this->timeout); ob_start(); @@ -63,12 +63,14 @@ private function runFunction(Request $request) 'event-namespace' => $_SERVER['HTTP_EVENT_NAMESPACE'], 'extensions' => (object) array( 'request' => $request, + 'response' => $response, ) ); $res = call_user_func($this->function, $event, $this->functionContext); + $event->extensions->response->getBody()->write($res); ob_end_clean(); chdir($this->currentDir); - return $res; + return $event->extensions->response; } else { sleep($this->timeout); posix_kill($pid, SIGKILL); @@ -106,10 +108,8 @@ public function root(Request $request, Response $response, array $args) { try { $this->validate(); - $ret = $this->runFunction($request); - $response->getBody()->write($ret); - return $response; + return $this->runFunction($request, $response); } catch (\Kubeless\TimeoutFunctionException $e) { $res = $response->withStatus(408); diff --git a/docs/advanced-function-deployment.md b/docs/advanced-function-deployment.md index 98e82d363..dd12091a9 100644 --- a/docs/advanced-function-deployment.md +++ b/docs/advanced-function-deployment.md @@ -25,9 +25,9 @@ The fields that a Function specification can contain are: - Runtime: Runtime ID and version that the function will use. It should match one of the availables in the [Kubeless configuration](/docs/function-controller-configuration). - Timeout: Maximum timeout for the given function. After that time, the function execution will be terminated. - Handler: Pair of `.`. When using `zip` in `function-content-type` the `` will be used to find the file with the function to expose. In other case it will be used just as a final file name. `` is used to select the function to run from the exported functions of ``. This field is mandatory and should match with an exported function. - - Deps: Dependencies of the function. The format of this field will depend on the runtime, e.g. a `package.json` for NodeJS functions or a `Gemfile` for Ruby. + - Deps: Dependencies of the function. The format of this field will depend on the runtime, e.g. a `package.json` for NodeJS functions or a `Gemfile` for Ruby. This field supports the use of URLs to specify dependency files. - Checksum: SHA256 of the function content. - - Function content type: Content type of the function. Current supported values are `zip` or `text`. If the content is encoded in base64 the suffix `+base64` should be added. + - Function content type: Content type of the function. Current supported values are `base64`, `url` or `text`. If the content is zipped the suffix `+zip` should be added. - Function: Function content. Apart from the basic parameters, it is possible to add the specification of a `Deployment`, a `Service` or an `Horizontal Pod Autoscaler` that Kubeless will use to generate them. diff --git a/docs/architecture.md b/docs/architecture.md index cfc4c10fc..62caca385 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -43,8 +43,8 @@ Kubeless leverages multiple concepts of Kubernetes in order to support deploy fu - Each event source is modelled as a separate Trigger CRD object - Separate Custom Resource Definitions controller to handle CRUD operations corresponding to CRD object - Deployment / Pod to run the corresponding runtime. -- Config map to inject function's code to the runtime pod. -- Init container to load the dependencies that function might have. +- Configmap to inject function's code to the runtime pod. +- Init-container to load the dependencies that function might have. - Service to expose function. - Ingress resources to expose functions externally @@ -325,9 +325,9 @@ Use "kubeless [command] --help" for more information about a command. ## Implementation -Kubeless controller is written in Go programming language, and uses the Kubernetes go client to interact with the Kubernetes API server. +Kubeless controller is written in Go programming language, and uses the Kubernetes client-go to interact with the Kubernetes apiserver. -Kubeless CLI is written in Go as well, using the popular cli library `github.com/spf13/cobra`. Basically it is a bundle of HTTP requests and kubectl commands. We send http requests to the Kubernetes API server in order to 'crud' CRD objects. Checkout [the cmd folder](https://github.com/kubeless/kubeless/tree/master/cmd/kubeless) for more details. +Kubeless CLI is written in Go as well, using the popular cli library `github.com/spf13/cobra`. Basically it is a bundle of HTTP requests and kubectl commands. We send http requests to the Kubernetes apiserver in order to 'crud' CRD objects. Checkout [the cmd folder](https://github.com/kubeless/kubeless/tree/master/cmd/kubeless) for more details. ## Directory structure diff --git a/docs/kubeless-functions.md b/docs/kubeless-functions.md index 5ccbd4768..0787d0061 100644 --- a/docs/kubeless-functions.md +++ b/docs/kubeless-functions.md @@ -16,6 +16,7 @@ event: event-namespace: "kafkatriggers.kubeless.io" # Event emitter extensions: # Optional parameters request: ... # Reference to the request received + response: ... # Reference to the response to send # (specific properties will depend on the function language) context: function-name: "pubsub-nodejs" diff --git a/docs/misc/kafka-pv.yaml b/docs/misc/kafka-pv-gke.yaml similarity index 100% rename from docs/misc/kafka-pv.yaml rename to docs/misc/kafka-pv-gke.yaml diff --git a/docs/misc/zk-pv.yaml b/docs/misc/zookeeper-pv-gke.yaml similarity index 100% rename from docs/misc/zk-pv.yaml rename to docs/misc/zookeeper-pv-gke.yaml diff --git a/docs/quick-start.md b/docs/quick-start.md index 43dac50e3..0f2b2ba0b 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -15,7 +15,7 @@ There are several kubeless manifests being shipped for multiple k8s environments For example, this below is a show case of deploying kubeless to a non-RBAC Kubernetes cluster. ```console -$ export RELEASE=v1.0.0-alpha.1 +$ export RELEASE=$(curl -s https://api.github.com/repos/kubeless/kubeless/releases/latest | grep tag_name | cut -d '"' -f 4) $ kubectl create ns kubeless $ kubectl create -f https://github.com/kubeless/kubeless/releases/download/$RELEASE/kubeless-non-rbac-$RELEASE.yaml diff --git a/examples/Makefile b/examples/Makefile index 48ac51e4e..655a535d6 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -49,6 +49,18 @@ get-python-36: get-python-36-verify: kubeless function call get-python-36 |egrep hello.world +get-python-url-deps: + kubeless function deploy get-python-url-deps --runtime python2.7 --handler helloget.foo --from-file https://raw.githubusercontent.com/kubeless/kubeless/v1.0.0-alpha.1/examples/python/hellowithdeps.py --dependencies https://raw.githubusercontent.com/kubeless/kubeless/v1.0.0-alpha.1/examples/python/requirements.txt + +get-python-url-deps-verify: + kubeless function call get-python-url-deps |egrep Google + +get-node-url-zip: + kubeless function deploy get-node-url-zip --runtime nodejs6 --handler index.helloGet --from-file https://github.com/tkeech1/kubelessfunction/blob/master/nodejs/helloFunctions.zip?raw=true + +get-node-url-zip-verify: + kubeless function call get-node-url-zip |egrep hello.world + scheduled-get-python: kubeless function deploy scheduled-get-python --schedule "* * * * *" --runtime python2.7 --handler helloget.foo --from-file python/helloget.py diff --git a/kubeless-non-rbac.jsonnet b/kubeless-non-rbac.jsonnet index c91806d88..127bf393b 100644 --- a/kubeless-non-rbac.jsonnet +++ b/kubeless-non-rbac.jsonnet @@ -106,13 +106,13 @@ local runtime_images ='[ { "name": "node6", "version": "6", - "runtimeImage": "kubeless/nodejs@sha256:61c5a10aacb709c4575a09a4aa28f822b2d008c0dbf4aa0b124705ee9ca143f9", + "runtimeImage": "kubeless/nodejs@sha256:0a8a72af4cc3bfbfd4fe9bd309cbf486e7493d0dc32a691673b3f0d3fae07487", "initImage": "node:6.10" }, { "name": "node8", "version": "8", - "runtimeImage": "kubeless/nodejs@sha256:fc1aa96e55116400ee13d664a655dfb2025ded91858ebfd5fc0c8f0d6b923eba", + "runtimeImage": "kubeless/nodejs@sha256:76ee28dc7e3613845fface2d1c56afc2e6e2c6d6392c724795a7ccc2f5e60582", "initImage": "node:8" } ], @@ -140,7 +140,7 @@ local runtime_images ='[ { "name": "php72", "version": "7.2", - "runtimeImage": "kubeless/php@sha256:4e44ab60f597e93097bf9f5ea91d58bd9c308bf206043db2a9809ec16a8ff2f4", + "runtimeImage": "kubeless/php@sha256:b605bb6b5ae3b1a2a93570939296618904259d7767a14002fa9733e66d59849b", "initImage": "composer:1.6" } ], @@ -154,8 +154,8 @@ local runtime_images ='[ { "name": "go1.10", "version": "1.10", - "runtimeImage": "kubeless/go@sha256:bf72622344a54e4360f31d3fea5eb9dca2c96fbedc6f0ad7c54f3eb8fb7bd353", - "initImage": "kubeless/go-init@sha256:e262f70639594b3a9e3481843171ecbbe82e84b786825ebe28bc1a3ae89310d3" + "runtimeImage": "kubeless/go@sha256:e2fd49f09b6ff8c9bac6f1592b3119ea74237c47e2955a003983e08524cb3ae5", + "initImage": "kubeless/go-init@sha256:d0812c4e8351bfd95d0574efd23613cff2664d6a57af4ed0a20ebc651382d476" } ], "depName": "Gopkg.toml", diff --git a/pkg/controller/function_controller.go b/pkg/controller/function_controller.go index e420538a8..34fbc6cde 100644 --- a/pkg/controller/function_controller.go +++ b/pkg/controller/function_controller.go @@ -447,6 +447,9 @@ func functionObjChanged(oldFunctionObj, newFunctionObj *kubelessApi.Function) bo oldSpec := &newFunctionObj.Spec if newSpec.Function != oldSpec.Function || + // compare checksum since the url content type uses Function field to pass the URL for the function + // comparing the checksum ensures that if the function code has changed but the URL remains the same, the function will get redeployed + newSpec.Checksum != oldSpec.Checksum || newSpec.Handler != oldSpec.Handler || newSpec.FunctionContentType != oldSpec.FunctionContentType || newSpec.Deps != oldSpec.Deps || diff --git a/pkg/functions/params.go b/pkg/functions/params.go index 87fa86105..c1c9f93ba 100644 --- a/pkg/functions/params.go +++ b/pkg/functions/params.go @@ -23,8 +23,9 @@ import ( // Extension includes a reference to the Event request and its Context (to handle timeouts) type Extension struct { - Request *http.Request - Context context.Context + Request *http.Request + Response http.ResponseWriter + Context context.Context } // Event includes information about the event source diff --git a/pkg/utils/k8sutil.go b/pkg/utils/k8sutil.go index 4ceada3e0..00a5338d2 100644 --- a/pkg/utils/k8sutil.go +++ b/pkg/utils/k8sutil.go @@ -409,7 +409,14 @@ func GetPodsByLabel(c kubernetes.Interface, ns, k, v string) (*v1.PodList, error // GetReadyPod returns the first pod has passed the liveness probe check func GetReadyPod(pods *v1.PodList) (v1.Pod, error) { for _, pod := range pods.Items { - if pod.Status.ContainerStatuses[0].Ready { + isPodRunning := true + for _, containerStatus := range pod.Status.ContainerStatuses { + if !containerStatus.Ready { + isPodRunning = false + break + } + } + if isPodRunning { return pod, nil } } @@ -433,6 +440,10 @@ func getProvisionContainer(function, checksum, fileName, handler, contentType, r decodedFile := "/tmp/func.decoded" prepareCommand = appendToCommand(prepareCommand, fmt.Sprintf("base64 -d < %s > %s", originFile, decodedFile)) originFile = decodedFile + } else if strings.Contains(contentType, "url") { + fromURLFile := "/tmp/func.fromurl" + prepareCommand = appendToCommand(prepareCommand, fmt.Sprintf("curl %s -L --silent --output %s", function, fromURLFile)) + originFile = fromURLFile } else if strings.Contains(contentType, "text") || contentType == "" { // Assumming that function is plain text // So we don't need to preprocess it @@ -642,7 +653,7 @@ func getFileName(handler, funcContentType, runtime string, lr *langruntime.Langr return "", err } filename := modName - if funcContentType == "text" || funcContentType == "" { + if funcContentType == "text" || funcContentType == "" || funcContentType == "url" { // We can only guess the extension if the function is specified as plain text runtimeInf, err := lr.GetRuntimeInfo(runtime) if err == nil { diff --git a/pkg/utils/k8sutil_test.go b/pkg/utils/k8sutil_test.go index 1f884f190..735145fee 100644 --- a/pkg/utils/k8sutil_test.go +++ b/pkg/utils/k8sutil_test.go @@ -1106,6 +1106,24 @@ func TestGetProvisionContainer(t *testing.T) { t.Errorf("Unexpected command: %s", c.Args[0]) } + // If the content type is url it should use curl + c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.py", "sha256:abc1234", "", "test.foo", "url", "python2.7", "unzip", rvol, dvol, lr) + if err != nil { + t.Errorf("Unexpected error: %s", err) + } + if !strings.HasPrefix(c.Args[0], "curl https://raw.githubusercontent.com/test/test/test/test.py -L --silent --output /tmp/func.fromurl") { + t.Errorf("Unexpected command: %s", c.Args[0]) + } + + // If the content type is url it should use curl + c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.py", "sha256:abc1234", "", "test.foo", "url+zip", "python2.7", "unzip", rvol, dvol, lr) + if err != nil { + t.Errorf("Unexpected error: %s", err) + } + if !strings.HasPrefix(c.Args[0], "curl https://raw.githubusercontent.com/test/test/test/test.py -L --silent --output /tmp/func.fromurl") { + t.Errorf("Unexpected command: %s", c.Args[0]) + } + } func TestInitializeEmptyMapsInDeployment(t *testing.T) { diff --git a/script/binary b/script/binary index 52907fc58..2e37b4c34 100755 --- a/script/binary +++ b/script/binary @@ -16,19 +16,25 @@ set -e + GIT_COMMIT=$(git describe --tags --dirty) BUILD_FLAGS=(-ldflags="-w -X github.com/kubeless/kubeless/pkg/version.Version=${GIT_COMMIT}") # Get rid of existing binary -rm -f ${GOPATH%%:*}/bin/kubeless* +echo "Removing Old Kubeless binaries" +rm -f ${GOPATH%%:*}/bin/kubeless +rm -f ${GOPATH%%:*}/bin/kafka-trigger-controller +rm -f ${GOPATH%%:*}/bin/kubeless-controller-manager +rm -f ${GOPATH%%:*}/bin/nats-trigger-controller +echo "Build Kubeless Components binaries" # Build binary go install \ "${BUILD_FLAGS[@]}" \ github.com/kubeless/kubeless/cmd/... if [ $? -eq 0 ]; then - echo "Build successful. Program saved at ${GOPATH%%:*}/bin" + echo "Build Kubeless Components successful. Program saved at ${GOPATH%%:*}/bin" else - echo "Build failed." + echo "Build Kubeless Components failed." fi diff --git a/script/cluster-up-minikube.sh b/script/cluster-up-minikube.sh index dbd1e52f4..6394d5313 100755 --- a/script/cluster-up-minikube.sh +++ b/script/cluster-up-minikube.sh @@ -29,7 +29,7 @@ MINIKUBE_VERSION=${MINIKUBE_VERSION:?} install_bin() { local exe=${1:?} - test -n "${TRAVIS}" && sudo install -v ${exe} /usr/local/bin || install ${exe} ${GOPATH:?}/bin + sudo install -v ${exe} /usr/local/bin || install ${exe} ${GOPATH:?}/bin } # Travis ubuntu trusty env doesn't have nsenter, needed for VM-less minikube diff --git a/script/create_release.sh b/script/create_release.sh new file mode 100755 index 000000000..46d3b61c6 --- /dev/null +++ b/script/create_release.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -e + +# TODO: Use kubeless +REPO_NAME=kubeless +REPO_DOMAIN=kubeless +TAG=${1:?} + +PROJECT_DIR=$(cd $(dirname $0)/.. && pwd) + +source $(dirname $0)/release_utils.sh + +if [[ -z "$REPO_NAME" || -z "$REPO_DOMAIN" ]]; then + echo "Github repository not specified" > /dev/stderr + exit 1 +fi + +if [[ -z "$ACCESS_TOKEN" ]]; then + echo "Unable to release: Github Token not specified" > /dev/stderr + exit 1 +fi + +repo_check=`curl -H "Authorization: token $ACCESS_TOKEN" -s https://api.github.com/repos/$REPO_DOMAIN/$REPO_NAME` +if [[ $repo_check == *"Not Found"* ]]; then + echo "Not found a Github repository for $REPO_DOMAIN/$REPO_NAME, it is not possible to publish it" > /dev/stderr + exit 1 +else + RELEASE_ID=$(release_tag $1 $REPO_DOMAIN $REPO_NAME | jq '.id') +fi + +manifests=( + kubeless kubeless-non-rbac kubeless-openshift + kafka-zookeeper kafka-zookeeper-openshift + nats + ) +for f in "${manifests[@]}"; do + cp ${PROJECT_DIR}/${f}.yaml ${PROJECT_DIR}/${f}-${TAG}.yaml + upload_asset $REPO_DOMAIN $REPO_NAME "$RELEASE_ID" "${PROJECT_DIR}/${f}-${TAG}.yaml" +done +for f in `ls ${PROJECT_DIR}/bundles/kubeless_*.zip`; do + upload_asset $REPO_DOMAIN $REPO_NAME $RELEASE_ID $f +done diff --git a/script/enable-gcloud.sh b/script/enable-gcloud.sh new file mode 100755 index 000000000..18ca24b67 --- /dev/null +++ b/script/enable-gcloud.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e +BUILD_DIR=${1:?} +export GOOGLE_APPLICATION_CREDENTIALS=$BUILD_DIR/client_secrets.json +echo $GCLOUD_KEY > $GOOGLE_APPLICATION_CREDENTIALS +if [ ! -d $HOME/gcloud/google-cloud-sdk ]; then + mkdir -p $HOME/gcloud && + wget -q https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-187.0.0-linux-x86_64.tar.gz --directory-prefix=$HOME/gcloud && + cd $HOME/gcloud && + tar xzf google-cloud-sdk-187.0.0-linux-x86_64.tar.gz && + printf '\ny\n\ny\ny\n' | ./google-cloud-sdk/install.sh && + sudo ln -s $HOME/gcloud/google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud + cd $BUILD_DIR; +fi +gcloud -q config set project $GKE_PROJECT +if [ -a $GOOGLE_APPLICATION_CREDENTIALS ]; then + gcloud -q auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS; +fi diff --git a/script/integration-tests b/script/integration-tests index 2e80841a3..462dfccf2 100755 --- a/script/integration-tests +++ b/script/integration-tests @@ -36,11 +36,6 @@ which bats > /dev/null || { exit 255 } -install_bin() { - local exe=${1:?} - test -n "${TRAVIS}" && sudo install -v ${exe} /usr/local/bin || install ${exe} ${GOPATH:?}/bin -} - # Start a k8s cluster (minikube, dind) if not running kubectl get nodes --context=${INTEGRATION_TESTS_CTX:?} || { cluster_up=./script/cluster-up-${INTEGRATION_TESTS_CTX}.sh diff --git a/script/release_utils.sh b/script/release_utils.sh new file mode 100755 index 000000000..42da2ac67 --- /dev/null +++ b/script/release_utils.sh @@ -0,0 +1,99 @@ +#!/bin/bash +set -e + +function commit_list { + local tag=${1:?} + local repo_domain=${2:?} + local repo_name=${3:?} + git fetch --tags + local previous_tag=`curl -H "Authorization: token $ACCESS_TOKEN" -s https://api.github.com/repos/$repo_domain/$repo_name/tags | jq --raw-output '.[1].name'` + local release_notes=`git log $previous_tag..$tag --oneline` + local parsed_release_notes=$(echo "$release_notes" | sed -n -e 'H;${x;s/\n/\\n- /g;s/^\\n//;s/"/\\"/g;p;}') + echo $parsed_release_notes +} + +function get_release_notes { + local tag=${1:?} + local repo_domain=${2:?} + local repo_name=${3:?} + commits=`commit_list $tag $repo_domain $repo_name` + notes=$(echo "\ +This release includes the following commits and features:\\n\ +$commits\\n\\n\ +To install this latest version, use the manifest that is part of the release:\\n\ +\\n\ +**WITH RBAC ENABLED:**\\n\ +\\n\ +\`\`\`console\\n\ +kubectl create ns kubeless\\n\ +kubectl create -f https://github.com/kubeless/kubeless/releases/download/$tag/kubeless-$tag.yaml \\n\ +\`\`\`\\n\ +\\n\ +**WITHOUT RBAC:**\\n\ +\\n\ +\`\`\`console\\n\ +kubectl create ns kubeless\\n\ +kubectl create -f https://github.com/kubeless/kubeless/releases/download/$tag/kubeless-non-rbac-$tag.yaml \\n\ +\`\`\`\\n\ +**OPENSHIFT:**\\n\ +\\n\ +\`\`\`console\\n\ +oc create ns kubeless\\n\ +oc create -f https://github.com/kubeless/kubeless/releases/download/$tag/kubeless-openshift-$tag.yaml \\n\ +# Kafka\\n\ +oc create -f https://github.com/kubeless/kubeless/releases/download/$tag/kafka-zookeeper-openshift-$tag.yaml \\n\ +\`\`\`\\n\ +") + echo "${notes}" +} + +function get_release_body { + local tag=${1:?} + local repo_domain=${2:?} + local repo_name=${3:?} + local release_notes=$(get_release_notes $tag $repo_domain $repo_name) + echo '{ + "tag_name": "'$tag'", + "target_commitish": "master", + "name": "'$tag'", + "body": "'$release_notes'", + "draft": true, + "prerelease": false + }' +} + +function update_release_tag { + local tag=${1:?} + local repo_domain=${2:?} + local repo_name=${3:?} + local release_id=$(curl -H "Authorization: token $ACCESS_TOKEN" -s https://api.github.com/repos/$repo_domain/$repo_name/releases | jq --raw-output '.[0].id') + local body=$(get_release_body $tag $repo_domain $repo_name) + local release=`curl -H "Authorization: token $ACCESS_TOKEN" -s --request PATCH --data $body https://api.github.com/repos/$repo_domain/$repo_name/releases/$release_id` + echo $release +} + +function release_tag { + local tag=$1 + local repo_domain=${2:?} + local repo_name=${3:?} + local body=$(get_release_body $tag $repo_domain $repo_name) + local release=`curl -H "Authorization: token $ACCESS_TOKEN" -s --request POST --data "$body" https://api.github.com/repos/$REPO_DOMAIN/$REPO_NAME/releases` + echo $release +} + +function upload_asset { + local repo_domain=${1:?} + local repo_name=${2:?} + local release_id=${3:?} + local asset=${4:?} + local filename=$(basename $asset) + if [[ "$filename" == *".zip" ]]; then + local content_type="application/zip" + elif [[ "$filename" == *".yaml" ]]; then + local content_type="text/yaml" + fi + curl -H "Authorization: token $ACCESS_TOKEN" \ + -H "Content-Type: $content_type" \ + --data-binary @"$asset" \ + "https://uploads.github.com/repos/$repo_domain/$repo_name/releases/$release_id/assets?name=$filename" +} diff --git a/script/upload_release_notes.sh b/script/upload_release_notes.sh index 070fec494..508a0f343 100755 --- a/script/upload_release_notes.sh +++ b/script/upload_release_notes.sh @@ -4,61 +4,7 @@ set -e REPO_NAME=kubeless REPO_DOMAIN=kubeless -function commit_list { - local tag=$1 - git fetch --tags - local previous_tag=`curl -H "Authorization: token $ACCESS_TOKEN" -s https://api.github.com/repos/$REPO_DOMAIN/$REPO_NAME/tags | jq --raw-output '.[1].name'` - local release_notes=`git log $previous_tag..$tag --oneline` - local parsed_release_notes=$(echo "$release_notes" | sed -n -e 'H;${x;s/\n/\\n- /g;s/^\\n//;s/"/\\"/g;p;}') - echo $parsed_release_notes -} - -function get_release_notes { - commits=`commit_list $tag` - notes=$(echo "\ -This release includes the following commits and features:\\n\ -$commits\\n\\n\ -To install this latest version, use the manifest that is part of the release:\\n\ -\\n\ -**WITH RBAC ENABLED:**\\n\ -\\n\ -\`\`\`console\\n\ -kubectl create ns kubeless\\n\ -kubectl create -f https://github.com/kubeless/kubeless/releases/download/$tag/kubeless-$tag.yaml \\n\ -\`\`\`\\n\ -\\n\ -**WITHOUT RBAC:**\\n\ -\\n\ -\`\`\`console\\n\ -kubectl create ns kubeless\\n\ -kubectl create -f https://github.com/kubeless/kubeless/releases/download/$tag/kubeless-non-rbac-$tag.yaml \\n\ -\`\`\`\\n\ -**OPENSHIFT:**\\n\ -\\n\ -\`\`\`console\\n\ -oc create ns kubeless\\n\ -oc create -f https://github.com/kubeless/kubeless/releases/download/$tag/kubeless-openshift-$tag.yaml \\n\ -# Kafka\\n\ -oc create -f https://github.com/kubeless/kubeless/releases/download/$tag/kafka-zookeeper-openshift-$tag.yaml \\n\ -\`\`\`\\n\ -") - echo "${notes}" -} - -function release_tag { - local tag=$1 - local release_notes=$(get_release_notes $tag) - local release_id=$(curl -H "Authorization: token $ACCESS_TOKEN" -s https://api.github.com/repos/$REPO_DOMAIN/$REPO_NAME/releases | jq --raw-output '.[0].id') - local release=`curl -H "Authorization: token $ACCESS_TOKEN" -s --request PATCH --data "{ - \"tag_name\": \"$tag\", - \"target_commitish\": \"master\", - \"name\": \"$tag\", - \"body\": \"$release_notes\", - \"draft\": true, - \"prerelease\": false - }" https://api.github.com/repos/$REPO_DOMAIN/$REPO_NAME/releases/$release_id` - echo $release -} +source $(dirname $0)/release_utils.sh if [[ -z "$REPO_NAME" || -z "$REPO_DOMAIN" ]]; then echo "Github repository not specified" > /dev/stderr @@ -75,5 +21,5 @@ if [[ $repo_check == *"Not Found"* ]]; then echo "Not found a Github repository for $REPO_DOMAIN/$REPO_NAME, it is not possible to publish it" > /dev/stderr exit 1 else - release_tag $1 + update_release_tag $1 $REPO_DOMAIN $REPO_DOMAIN fi diff --git a/tests/integration-tests.bats b/tests/integration-tests.bats index 70de1ffb1..c3ae9d52a 100644 --- a/tests/integration-tests.bats +++ b/tests/integration-tests.bats @@ -45,6 +45,8 @@ load ../script/libtest deploy_function post-php deploy_function post-go deploy_function custom-get-python + deploy_function get-python-url-deps + deploy_function get-node-url-zip } @test "Test function: get-python" { verify_function get-python @@ -187,4 +189,12 @@ load ../script/libtest false fi } +@test "Test function: get-python-url-deps" { + verify_function get-python-url-deps + kubeless_function_delete get-python-url-deps +} +@test "Test function: get-node-url-zip" { + verify_function get-node-url-zip + kubeless_function_delete get-node-url-zip +} # vim: ts=2 sw=2 si et syntax=sh