From cce20f2e90a582508f6d83d78d12f054b197f027 Mon Sep 17 00:00:00 2001 From: todd Date: Mon, 30 Apr 2018 21:54:44 -0400 Subject: [PATCH 01/23] Added --from-url command line flag --- cmd/kubeless/function/deploy.go | 8 +++- cmd/kubeless/function/function.go | 39 ++++++++++++++++++- cmd/kubeless/function/function_test.go | 52 +++++++++++++++++++++++--- cmd/kubeless/function/update.go | 8 +++- 4 files changed, 98 insertions(+), 9 deletions(-) diff --git a/cmd/kubeless/function/deploy.go b/cmd/kubeless/function/deploy.go index c49c5ce33..b9dea96c6 100644 --- a/cmd/kubeless/function/deploy.go +++ b/cmd/kubeless/function/deploy.go @@ -85,6 +85,11 @@ var deployCmd = &cobra.Command{ logrus.Fatal(err) } + fromURL, err := cmd.Flags().GetString("from-url") + if err != nil { + logrus.Fatal(err) + } + file, err := cmd.Flags().GetString("from-file") if err != nil { logrus.Fatal(err) @@ -163,7 +168,7 @@ var deployCmd = &cobra.Command{ "function": funcName, } - f, err := getFunctionDescription(cli, funcName, ns, handler, file, funcDeps, runtime, runtimeImage, mem, cpu, timeout, port, headless, envs, labels, secrets, defaultFunctionSpec) + f, err := getFunctionDescription(cli, funcName, ns, handler, fromURL, file, funcDeps, runtime, runtimeImage, mem, cpu, timeout, port, headless, envs, labels, secrets, defaultFunctionSpec) if err != nil { logrus.Fatal(err) @@ -210,6 +215,7 @@ func init() { deployCmd.Flags().StringP("runtime", "", "", "Specify runtime") deployCmd.Flags().StringP("handler", "", "", "Specify handler") deployCmd.Flags().StringP("from-file", "", "", "Specify code file") + deployCmd.Flags().StringP("from-url", "", "", "Specify a URL to the raw code file. For example, --from-url https://raw.githubusercontent.com////") deployCmd.Flags().StringSliceP("label", "", []string{}, "Specify labels of the function. Both separator ':' and '=' are allowed. For example: --label foo1=bar1,foo2:bar2") deployCmd.Flags().StringSliceP("secrets", "", []string{}, "Specify Secrets to be mounted to the functions container. For example: --secrets mySecret") deployCmd.Flags().StringArrayP("env", "", []string{}, "Specify environment variable of the function. Both separator ':' and '=' are allowed. For example: --env foo1=bar1,foo2:bar2") diff --git a/cmd/kubeless/function/function.go b/cmd/kubeless/function/function.go index 8d2863c4a..4abe81e67 100644 --- a/cmd/kubeless/function/function.go +++ b/cmd/kubeless/function/function.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "io/ioutil" + "net/http" "os" "path" "strings" @@ -120,6 +121,16 @@ func getFileSha256(file string) (string, error) { return "sha256:" + checksum, err } +func getHttpSha256(bytes []byte) (string, error) { + h := sha256.New() + _, err := h.Write(bytes) + if err != nil { + return "", err + } + checksum := hex.EncodeToString(h.Sum(nil)) + return "sha256:" + checksum, nil +} + func getContentType(filename string, fbytes []byte) string { var contentType string isText := utf8.ValidString(string(fbytes)) @@ -134,7 +145,7 @@ func getContentType(filename string, fbytes []byte) string { return contentType } -func getFunctionDescription(cli kubernetes.Interface, funcName, ns, handler, file, deps, runtime, runtimeImage, mem, cpu, timeout string, port int32, headless bool, envs, labels []string, secrets []string, defaultFunction kubelessApi.Function) (*kubelessApi.Function, error) { +func getFunctionDescription(cli kubernetes.Interface, funcName, ns, handler, fromURL, file, deps, runtime, runtimeImage, mem, cpu, timeout string, port int32, headless bool, envs, labels []string, secrets []string, defaultFunction kubelessApi.Function) (*kubelessApi.Function, error) { function := defaultFunction function.TypeMeta = metav1.TypeMeta{ @@ -145,6 +156,32 @@ func getFunctionDescription(cli kubernetes.Interface, funcName, ns, handler, fil function.Spec.Handler = handler } + // --from-file will override --from-url + if fromURL != "" && file == "" { + resp, err := http.Get(fromURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + functionBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + function.Spec.FunctionContentType = getContentType(fromURL, functionBytes) + if function.Spec.FunctionContentType == "text" { + function.Spec.Function = string(functionBytes) + } else { + function.Spec.Function = base64.StdEncoding.EncodeToString(functionBytes) + } + checksum, err := getHttpSha256(functionBytes) + if err != nil { + return nil, err + } + function.Spec.Checksum = checksum + } + if file != "" { functionBytes, err := ioutil.ReadFile(file) if err != nil { diff --git a/cmd/kubeless/function/function_test.go b/cmd/kubeless/function/function_test.go index adb9be9fd..d165d3059 100644 --- a/cmd/kubeless/function/function_test.go +++ b/cmd/kubeless/function/function_test.go @@ -18,8 +18,11 @@ package function import ( "archive/zip" + "fmt" "io" "io/ioutil" + "net/http" + "net/http/httptest" "os" "reflect" "testing" @@ -100,7 +103,7 @@ func TestGetFunctionDescription(t *testing.T) { file.Close() defer os.Remove(file.Name()) // clean up - result, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) + result, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", "", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) if err != nil { t.Error(err) @@ -186,7 +189,7 @@ func TestGetFunctionDescription(t *testing.T) { } // It should take the default values - result2, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "", "", "", "", "", "", "", "", 8080, false, []string{}, []string{}, []string{}, expectedFunction) + result2, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "", "", "", "", "", "", "", "", "", 8080, false, []string{}, []string{}, []string{}, expectedFunction) if err != nil { t.Error(err) @@ -195,6 +198,23 @@ func TestGetFunctionDescription(t *testing.T) { t.Errorf("Unexpected result. Expecting:\n %+v\n Received %+v\n", expectedFunction, *result2) } + // it should create a function from a URL + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "function") + })) + defer ts.Close() + + result7, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", ts.URL, "", "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) + + if err != nil { + t.Error(err) + } + + if !reflect.DeepEqual(expectedFunction, *result7) { + t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedFunction, *result7) + } + // end test + // Given parameters should take precedence from default values file, err = ioutil.TempFile("", "test") if err != nil { @@ -207,7 +227,7 @@ func TestGetFunctionDescription(t *testing.T) { file.Close() defer os.Remove(file.Name()) // clean up - result3, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler2", file.Name(), "dependencies2", "runtime2", "test-image2", "256Mi", "100m", "20", 8080, false, []string{"TEST=2"}, []string{"test=2"}, []string{"secret2"}, expectedFunction) + result3, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler2", "", file.Name(), "dependencies2", "runtime2", "test-image2", "256Mi", "100m", "20", 8080, false, []string{"TEST=2"}, []string{"test=2"}, []string{"secret2"}, expectedFunction) if err != nil { t.Error(err) @@ -332,7 +352,7 @@ func TestGetFunctionDescription(t *testing.T) { file.Close() zipW.Close() - result4, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", newfile.Name(), "dependencies", "runtime", "", "", "", "", 8080, false, []string{}, []string{}, []string{}, expectedFunction) + result4, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", "", newfile.Name(), "dependencies", "runtime", "", "", "", "", 8080, false, []string{}, []string{}, []string{}, expectedFunction) if err != nil { t.Error(err) } @@ -341,7 +361,7 @@ func TestGetFunctionDescription(t *testing.T) { } // It should maintain previous HPA definition - result5, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{}, kubelessApi.Function{ + result5, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", "", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{}, kubelessApi.Function{ Spec: kubelessApi.FunctionSpec{ HorizontalPodAutoscaler: v2beta1.HorizontalPodAutoscaler{ @@ -356,7 +376,7 @@ func TestGetFunctionDescription(t *testing.T) { } // It should set the Port and headless service properly - result6, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "", 9091, true, []string{}, []string{}, []string{}, kubelessApi.Function{}) + result6, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", "", file.Name(), "dependencies", "runtime", "test-image", "128Mi", "", "", 9091, true, []string{}, []string{}, []string{}, kubelessApi.Function{}) expectedPort := v1.ServicePort{ Name: "http-function-port", Port: 9091, @@ -370,4 +390,24 @@ func TestGetFunctionDescription(t *testing.T) { if result6.Spec.ServiceSpec.ClusterIP != v1.ClusterIPNone { t.Errorf("Unexpected clusterIP %v", result6.Spec.ServiceSpec.ClusterIP) } + + // it should handle zip files from a URL and detect base64+zip encoding + zipBytes, err := ioutil.ReadFile(newfile.Name()) + if err != nil { + t.Error(err) + } + + ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(zipBytes) + })) + defer ts2.Close() + + result8, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", ts2.URL+"/test.zip", "", "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) + if err != nil { + t.Error(err) + } + if result8.Spec.FunctionContentType != "base64+zip" { + t.Errorf("Should return base64+zip, received %s", result8.Spec.FunctionContentType) + } + // end test } diff --git a/cmd/kubeless/function/update.go b/cmd/kubeless/function/update.go index c39cbb5ae..73a0a0d7a 100644 --- a/cmd/kubeless/function/update.go +++ b/cmd/kubeless/function/update.go @@ -59,6 +59,11 @@ var updateCmd = &cobra.Command{ logrus.Fatal(err) } + fromURL, err := cmd.Flags().GetString("from-url") + if err != nil { + logrus.Fatal(err) + } + file, err := cmd.Flags().GetString("from-file") if err != nil { logrus.Fatal(err) @@ -137,7 +142,7 @@ var updateCmd = &cobra.Command{ logrus.Fatal(err) } - f, err := getFunctionDescription(cli, funcName, ns, handler, file, funcDeps, runtime, runtimeImage, mem, cpu, timeout, port, headless, envs, labels, secrets, previousFunction) + f, err := getFunctionDescription(cli, funcName, ns, handler, fromURL, file, funcDeps, runtime, runtimeImage, mem, cpu, timeout, port, headless, envs, labels, secrets, previousFunction) if err != nil { logrus.Fatal(err) } @@ -160,6 +165,7 @@ func init() { updateCmd.Flags().StringP("runtime", "", "", "Specify runtime") updateCmd.Flags().StringP("handler", "", "", "Specify handler") updateCmd.Flags().StringP("from-file", "", "", "Specify code file") + updateCmd.Flags().StringP("from-url", "", "", "Specify a URL to the raw code file. For example, --from-url https://raw.githubusercontent.com////") updateCmd.Flags().StringP("memory", "", "", "Request amount of memory for the function") updateCmd.Flags().StringP("cpu", "", "", "Request amount of cpu for the function.") updateCmd.Flags().StringSliceP("label", "", []string{}, "Specify labels of the function") From 887de6e1d6cd73d37904a160c045f7d0a26f75b1 Mon Sep 17 00:00:00 2001 From: todd Date: Mon, 30 Apr 2018 22:46:42 -0400 Subject: [PATCH 02/23] modified method name to getHTTPSha256 --- cmd/kubeless/function/function.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/kubeless/function/function.go b/cmd/kubeless/function/function.go index 4abe81e67..5afe059ea 100644 --- a/cmd/kubeless/function/function.go +++ b/cmd/kubeless/function/function.go @@ -121,7 +121,7 @@ func getFileSha256(file string) (string, error) { return "sha256:" + checksum, err } -func getHttpSha256(bytes []byte) (string, error) { +func getHTTPSha256(bytes []byte) (string, error) { h := sha256.New() _, err := h.Write(bytes) if err != nil { From 0f1fb97f66677b0a54beeee36abc7cfe12633c53 Mon Sep 17 00:00:00 2001 From: todd Date: Mon, 30 Apr 2018 22:53:28 -0400 Subject: [PATCH 03/23] modified method name to getHTTPSha256 in function call --- cmd/kubeless/function/function.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/kubeless/function/function.go b/cmd/kubeless/function/function.go index 5afe059ea..f597d22b4 100644 --- a/cmd/kubeless/function/function.go +++ b/cmd/kubeless/function/function.go @@ -175,7 +175,7 @@ func getFunctionDescription(cli kubernetes.Interface, funcName, ns, handler, fro } else { function.Spec.Function = base64.StdEncoding.EncodeToString(functionBytes) } - checksum, err := getHttpSha256(functionBytes) + checksum, err := getHTTPSha256(functionBytes) if err != nil { return nil, err } From 13da68be8ed1f4f1ad41e2d7445c8e2621a48bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BA=E6=B5=91?= <1004815462@qq.com> Date: Fri, 4 May 2018 17:27:58 +0800 Subject: [PATCH 04/23] change some terms in architecture.md (#728) --- docs/architecture.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index cfc4c10fc..62caca385 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -43,8 +43,8 @@ Kubeless leverages multiple concepts of Kubernetes in order to support deploy fu - Each event source is modelled as a separate Trigger CRD object - Separate Custom Resource Definitions controller to handle CRUD operations corresponding to CRD object - Deployment / Pod to run the corresponding runtime. -- Config map to inject function's code to the runtime pod. -- Init container to load the dependencies that function might have. +- Configmap to inject function's code to the runtime pod. +- Init-container to load the dependencies that function might have. - Service to expose function. - Ingress resources to expose functions externally @@ -325,9 +325,9 @@ Use "kubeless [command] --help" for more information about a command. ## Implementation -Kubeless controller is written in Go programming language, and uses the Kubernetes go client to interact with the Kubernetes API server. +Kubeless controller is written in Go programming language, and uses the Kubernetes client-go to interact with the Kubernetes apiserver. -Kubeless CLI is written in Go as well, using the popular cli library `github.com/spf13/cobra`. Basically it is a bundle of HTTP requests and kubectl commands. We send http requests to the Kubernetes API server in order to 'crud' CRD objects. Checkout [the cmd folder](https://github.com/kubeless/kubeless/tree/master/cmd/kubeless) for more details. +Kubeless CLI is written in Go as well, using the popular cli library `github.com/spf13/cobra`. Basically it is a bundle of HTTP requests and kubectl commands. We send http requests to the Kubernetes apiserver in order to 'crud' CRD objects. Checkout [the cmd folder](https://github.com/kubeless/kubeless/tree/master/cmd/kubeless) for more details. ## Directory structure From c6e506892474c39b2d2fa7b0b085f608c0ab2f49 Mon Sep 17 00:00:00 2001 From: todd Date: Sun, 6 May 2018 13:30:58 -0400 Subject: [PATCH 05/23] modified PR 726 to use curl to retrieve function code in getProvisionContainer --- cmd/kubeless/function/function.go | 22 ++--- cmd/kubeless/function/function_test.go | 120 +++++++++++++++++++------ pkg/controller/function_controller.go | 3 + pkg/utils/k8sutil.go | 6 +- pkg/utils/k8sutil_test.go | 9 ++ 5 files changed, 121 insertions(+), 39 deletions(-) diff --git a/cmd/kubeless/function/function.go b/cmd/kubeless/function/function.go index f597d22b4..c0b768c4f 100644 --- a/cmd/kubeless/function/function.go +++ b/cmd/kubeless/function/function.go @@ -20,6 +20,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "errors" "fmt" "io" "io/ioutil" @@ -121,7 +122,7 @@ func getFileSha256(file string) (string, error) { return "sha256:" + checksum, err } -func getHTTPSha256(bytes []byte) (string, error) { +func getSha256(bytes []byte) (string, error) { h := sha256.New() _, err := h.Write(bytes) if err != nil { @@ -156,8 +157,12 @@ func getFunctionDescription(cli kubernetes.Interface, funcName, ns, handler, fro function.Spec.Handler = handler } - // --from-file will override --from-url - if fromURL != "" && file == "" { + if fromURL != "" && file != "" { + err := errors.New("either --from-url or --from-file should be provided, not both") + return nil, err + } + + if fromURL != "" { resp, err := http.Get(fromURL) if err != nil { return nil, err @@ -169,17 +174,14 @@ func getFunctionDescription(cli kubernetes.Interface, funcName, ns, handler, fro return nil, err } - function.Spec.FunctionContentType = getContentType(fromURL, functionBytes) - if function.Spec.FunctionContentType == "text" { - function.Spec.Function = string(functionBytes) - } else { - function.Spec.Function = base64.StdEncoding.EncodeToString(functionBytes) - } - checksum, err := getHTTPSha256(functionBytes) + checksum, err := getSha256(functionBytes) if err != nil { return nil, err } function.Spec.Checksum = checksum + + function.Spec.FunctionContentType = "url" + function.Spec.Function = fromURL } if file != "" { diff --git a/cmd/kubeless/function/function_test.go b/cmd/kubeless/function/function_test.go index d165d3059..3517a1eff 100644 --- a/cmd/kubeless/function/function_test.go +++ b/cmd/kubeless/function/function_test.go @@ -198,23 +198,6 @@ func TestGetFunctionDescription(t *testing.T) { t.Errorf("Unexpected result. Expecting:\n %+v\n Received %+v\n", expectedFunction, *result2) } - // it should create a function from a URL - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, "function") - })) - defer ts.Close() - - result7, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", ts.URL, "", "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) - - if err != nil { - t.Error(err) - } - - if !reflect.DeepEqual(expectedFunction, *result7) { - t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedFunction, *result7) - } - // end test - // Given parameters should take precedence from default values file, err = ioutil.TempFile("", "test") if err != nil { @@ -391,23 +374,104 @@ func TestGetFunctionDescription(t *testing.T) { t.Errorf("Unexpected clusterIP %v", result6.Spec.ServiceSpec.ClusterIP) } - // it should handle zip files from a URL and detect base64+zip encoding - zipBytes, err := ioutil.ReadFile(newfile.Name()) - if err != nil { - t.Error(err) + // it should create a function from a URL + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "function") + })) + defer ts.Close() + + expectedURLFunction := kubelessApi.Function{ + TypeMeta: metav1.TypeMeta{ + Kind: "Function", + APIVersion: "kubeless.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + Labels: map[string]string{ + "test": "1", + }, + }, + Spec: kubelessApi.FunctionSpec{ + Handler: "file.handler", + Runtime: "runtime", + Function: ts.URL, + Checksum: "sha256:78f9ac018e554365069108352dacabb7fbd15246edf19400677e3b54fe24e126", + FunctionContentType: "url", + Deps: "dependencies", + Timeout: "10", + Deployment: v1beta1.Deployment{ + Spec: v1beta1.DeploymentSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Env: []v1.EnvVar{{ + Name: "TEST", + Value: "1", + }}, + Resources: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: parsedMem, + v1.ResourceCPU: parsedCPU, + }, + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: parsedMem, + v1.ResourceCPU: parsedCPU, + }, + }, + Image: "test-image", + VolumeMounts: []v1.VolumeMount{ + { + Name: "secretName-vol", + MountPath: "/secretName", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "secretName-vol", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "secretName", + }, + }, + }, + }, + }, + }, + }, + }, + ServiceSpec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + {Name: "http-function-port", Protocol: "TCP", Port: 8080, TargetPort: intstr.FromInt(8080)}, + }, + Selector: map[string]string{ + "test": "1", + }, + Type: v1.ServiceTypeClusterIP, + }, + }, } - ts2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write(zipBytes) - })) - defer ts2.Close() + result7, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", ts.URL, "", "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) - result8, err := getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", ts2.URL+"/test.zip", "", "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) if err != nil { t.Error(err) } - if result8.Spec.FunctionContentType != "base64+zip" { - t.Errorf("Should return base64+zip, received %s", result8.Spec.FunctionContentType) + + if !reflect.DeepEqual(expectedURLFunction, *result7) { + t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedURLFunction, *result7) + } + // end test + + // it should return an error if both --from-url and --from-file are given as command line flags + _, err = getFunctionDescription(fake.NewSimpleClientset(), "test", "default", "file.handler", "fromURL", "fromFile", "dependencies", "runtime", "test-image", "128Mi", "", "10", 8080, false, []string{"TEST=1"}, []string{"test=1"}, []string{"secretName"}, kubelessApi.Function{}) + + if err == nil { + t.Error(err) } // end test + } diff --git a/pkg/controller/function_controller.go b/pkg/controller/function_controller.go index e420538a8..34fbc6cde 100644 --- a/pkg/controller/function_controller.go +++ b/pkg/controller/function_controller.go @@ -447,6 +447,9 @@ func functionObjChanged(oldFunctionObj, newFunctionObj *kubelessApi.Function) bo oldSpec := &newFunctionObj.Spec if newSpec.Function != oldSpec.Function || + // compare checksum since the url content type uses Function field to pass the URL for the function + // comparing the checksum ensures that if the function code has changed but the URL remains the same, the function will get redeployed + newSpec.Checksum != oldSpec.Checksum || newSpec.Handler != oldSpec.Handler || newSpec.FunctionContentType != oldSpec.FunctionContentType || newSpec.Deps != oldSpec.Deps || diff --git a/pkg/utils/k8sutil.go b/pkg/utils/k8sutil.go index 4ceada3e0..b6867228b 100644 --- a/pkg/utils/k8sutil.go +++ b/pkg/utils/k8sutil.go @@ -433,6 +433,10 @@ func getProvisionContainer(function, checksum, fileName, handler, contentType, r decodedFile := "/tmp/func.decoded" prepareCommand = appendToCommand(prepareCommand, fmt.Sprintf("base64 -d < %s > %s", originFile, decodedFile)) originFile = decodedFile + } else if strings.Contains(contentType, "url") { + fromURLFile := "/tmp/func.fromurl" + prepareCommand = appendToCommand(prepareCommand, fmt.Sprintf("curl %s --silent --output %s", function, fromURLFile)) + originFile = fromURLFile } else if strings.Contains(contentType, "text") || contentType == "" { // Assumming that function is plain text // So we don't need to preprocess it @@ -642,7 +646,7 @@ func getFileName(handler, funcContentType, runtime string, lr *langruntime.Langr return "", err } filename := modName - if funcContentType == "text" || funcContentType == "" { + if funcContentType == "text" || funcContentType == "" || funcContentType == "url"{ // We can only guess the extension if the function is specified as plain text runtimeInf, err := lr.GetRuntimeInfo(runtime) if err == nil { diff --git a/pkg/utils/k8sutil_test.go b/pkg/utils/k8sutil_test.go index 1f884f190..fbd3a604d 100644 --- a/pkg/utils/k8sutil_test.go +++ b/pkg/utils/k8sutil_test.go @@ -1106,6 +1106,15 @@ func TestGetProvisionContainer(t *testing.T) { t.Errorf("Unexpected command: %s", c.Args[0]) } + // If the content type is url it should use curl + c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.py", "sha256:abc1234", "", "test.foo", "url", "python2.7", "unzip", rvol, dvol, lr) + if err != nil { + t.Errorf("Unexpected error: %s", err) + } + if !strings.HasPrefix(c.Args[0], "curl https://raw.githubusercontent.com/test/test/test/test.py --silent --output /tmp/func.fromurl") { + t.Errorf("Unexpected command: %s", c.Args[0]) + } + } func TestInitializeEmptyMapsInDeployment(t *testing.T) { From e9ef9fbf19264f5bbe5c290be86f4a8c41e85d2b Mon Sep 17 00:00:00 2001 From: todd Date: Sun, 6 May 2018 14:03:09 -0400 Subject: [PATCH 06/23] code formatting changes to pass build --- pkg/utils/k8sutil.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/k8sutil.go b/pkg/utils/k8sutil.go index b6867228b..cd51065c7 100644 --- a/pkg/utils/k8sutil.go +++ b/pkg/utils/k8sutil.go @@ -646,7 +646,7 @@ func getFileName(handler, funcContentType, runtime string, lr *langruntime.Langr return "", err } filename := modName - if funcContentType == "text" || funcContentType == "" || funcContentType == "url"{ + if funcContentType == "text" || funcContentType == "" || funcContentType == "url" { // We can only guess the extension if the function is specified as plain text runtimeInf, err := lr.GetRuntimeInfo(runtime) if err == nil { From 8d1966c45486f2b6b067c8a9ceee7a72939675ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BA=E6=B5=91?= <1004815462@qq.com> Date: Mon, 7 May 2018 12:32:19 +0800 Subject: [PATCH 07/23] fix cmd/kubeless/autoscale some typo (#729) --- cmd/kubeless/autoscale/autoscaleDelete.go | 4 ++-- cmd/kubeless/autoscale/autoscaleList.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/kubeless/autoscale/autoscaleDelete.go b/cmd/kubeless/autoscale/autoscaleDelete.go index b24841773..16d454e45 100644 --- a/cmd/kubeless/autoscale/autoscaleDelete.go +++ b/cmd/kubeless/autoscale/autoscaleDelete.go @@ -54,9 +54,9 @@ var autoscaleDeleteCmd = &cobra.Command{ if err != nil { logrus.Fatal(err) } - logrus.Infof("Removed Autoscaling rule from %s", funcName) + logrus.Infof("Remove Autoscaling rule from %s successfully", funcName) } else { - logrus.Fatalf("Not found an auto scale definition for %s", funcName) + logrus.Fatalf("Not found an autoscale definition for %s", funcName) } }, } diff --git a/cmd/kubeless/autoscale/autoscaleList.go b/cmd/kubeless/autoscale/autoscaleList.go index f54d02f55..89f769bfb 100644 --- a/cmd/kubeless/autoscale/autoscaleList.go +++ b/cmd/kubeless/autoscale/autoscaleList.go @@ -85,7 +85,7 @@ func printAutoscale(w io.Writer, ass []v2beta1.HorizontalPodAutoscaler, output s m := "" v := "" if len(i.Spec.Metrics) == 0 { - logrus.Errorf("The function autoscale %s isn't in correct format. It has no metric defined.", i.Name) + logrus.Errorf("The autoscale %s has bad format. It has no metric defined.", i.Name) continue } if i.Spec.Metrics[0].Object != nil { @@ -115,7 +115,7 @@ func printAutoscale(w io.Writer, ass []v2beta1.HorizontalPodAutoscaler, output s } fmt.Fprintln(w, string(b)) default: - return fmt.Errorf("Wrong output format. Please use only json|yaml") + return fmt.Errorf("Wrong output format. Only accept json|yaml file") } } } From d9c23befa2956c2f610686601bb60011b17e61dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BA=E6=B5=91?= <1004815462@qq.com> Date: Tue, 8 May 2018 15:28:39 +0800 Subject: [PATCH 08/23] pv for kafka (#735) * pv for kafka * Update README.md Fix some typos and links --- chart/README.md | 1 + chart/kubeless/README.md | 2 +- docs/misc/{kafka-pv.yaml => kafka-pv-gke.yaml} | 0 docs/misc/{zk-pv.yaml => zookeeper-pv-gke.yaml} | 0 4 files changed, 2 insertions(+), 1 deletion(-) rename docs/misc/{kafka-pv.yaml => kafka-pv-gke.yaml} (100%) rename docs/misc/{zk-pv.yaml => zookeeper-pv-gke.yaml} (100%) diff --git a/chart/README.md b/chart/README.md index ad0adf06b..9b5226edf 100644 --- a/chart/README.md +++ b/chart/README.md @@ -15,3 +15,4 @@ helm init helm install --name kubeless --namespace kubeless ./kubeless ``` +After that, if you are having trouble deploying Kafka and Zookeeper, please check the specific guide [here](/docs/troubleshooting/#kafka-and-zookeeper-persistent-volume-creation) to create appropriate disks and PVs. If you are running Kubernetes in GKE,you can provision those persistent volumes manually deploying the manifests present in the [misc folder](https://github.com/kubeless/kubeless/tree/master/docs/misc). If you use other cloud provider, check [kubernetes docs](https://kubernetes.io/docs/concepts/storage/volumes/) to create these required volumes. diff --git a/chart/kubeless/README.md b/chart/kubeless/README.md index 4383cb3de..99214e174 100644 --- a/chart/kubeless/README.md +++ b/chart/kubeless/README.md @@ -6,4 +6,4 @@ It installs: * The controller * The Kubeless configuration * The UI -* A single node Kafka and Zookeeper setup +* A single node Kafka and Zookeeper setup \ No newline at end of file diff --git a/docs/misc/kafka-pv.yaml b/docs/misc/kafka-pv-gke.yaml similarity index 100% rename from docs/misc/kafka-pv.yaml rename to docs/misc/kafka-pv-gke.yaml diff --git a/docs/misc/zk-pv.yaml b/docs/misc/zookeeper-pv-gke.yaml similarity index 100% rename from docs/misc/zk-pv.yaml rename to docs/misc/zookeeper-pv-gke.yaml From 147424da453e476bee4530c1554bb9c9c4ec9ac5 Mon Sep 17 00:00:00 2001 From: Andres Date: Tue, 8 May 2018 15:05:52 +0200 Subject: [PATCH 09/23] Point to latest release in quick-start guide --- docs/quick-start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quick-start.md b/docs/quick-start.md index 43dac50e3..0f2b2ba0b 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -15,7 +15,7 @@ There are several kubeless manifests being shipped for multiple k8s environments For example, this below is a show case of deploying kubeless to a non-RBAC Kubernetes cluster. ```console -$ export RELEASE=v1.0.0-alpha.1 +$ export RELEASE=$(curl -s https://api.github.com/repos/kubeless/kubeless/releases/latest | grep tag_name | cut -d '"' -f 4) $ kubectl create ns kubeless $ kubectl create -f https://github.com/kubeless/kubeless/releases/download/$RELEASE/kubeless-non-rbac-$RELEASE.yaml From 082209d8fdfca56c41fccc37c93b8748764072c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BA=E6=B5=91?= <1004815462@qq.com> Date: Wed, 9 May 2018 15:34:43 +0800 Subject: [PATCH 10/23]