Added URL support for --from-file command line flag

.circleci/config.yml

@@ -0,0 +1,306 @@
+version: 2
+
+## Definitions
+build_allways: &build_allways
+  filters:
+    tags:
+      only: /.*/
+defaults: &defaults
+  environment:
+    CONTROLLER_IMAGE_NAME: bitnami/kubeless-controller-manager
+    BUILDER_IMAGE_NAME: kubeless/function-image-builder
+    KAFKA_CONTROLLER_IMAGE_NAME: bitnami/kafka-trigger-controller
+    NATS_CONTROLLER_IMAGE_NAME: bitnami/nats-trigger-controller
+    CGO_ENABLED: "0"
+    TEST_DEBUG: "1"
+    GKE_VERSION: 1.8.8-gke.0
+    MINIKUBE_VERSION: v0.25.2
+exports: &exports
+  # It is not possible to resolve env vars in the environment section:
+  # https://discuss.circleci.com/t/using-environment-variables-in-config-yml-not-working/14237
+  run: |
+    CONTROLLER_TAG=${CIRCLE_TAG:-build-$CIRCLE_WORKFLOW_ID}
+    echo "export CONTROLLER_TAG=${CONTROLLER_TAG}" >> $BASH_ENV
+    echo "export CONTROLLER_IMAGE=${CONTROLLER_IMAGE_NAME}:${CONTROLLER_TAG}" >> $BASH_ENV
+    echo "export BUILDER_IMAGE=${BUILDER_IMAGE_NAME}:${CONTROLLER_TAG}" >> $BASH_ENV
+    echo "export KAFKA_CONTROLLER_IMAGE=${KAFKA_CONTROLLER_IMAGE_NAME}:${CONTROLLER_TAG}" >> $BASH_ENV
+    echo "export NATS_CONTROLLER_IMAGE=${NATS_CONTROLLER_IMAGE_NAME}:${CONTROLLER_TAG}" >> $BASH_ENV
+    echo "export KUBECFG_JPATH=/go/src/github.com/kubeless/kubeless/ksonnet-lib" >> $BASH_ENV
+    echo "export PATH=$(pwd)/bats/libexec:$PATH" >> $BASH_ENV
+restore_workspace: &restore_workspace
+  run: |
+    make bootstrap
+    sudo cp -r /tmp/go/bin/* /usr/local/bin/
+    cp -r /tmp/go/src/github.com/kubeless/kubeless/*yaml .
+should_test: &should_test
+  run: |
+    case $CIRCLE_JOB in
+    # In case of GKE we will only want to build if it is
+    # a build of a branch in the kubeless/kubeless repository
+    GKE)
+      echo $CIRCLE_PULL_REQUESTS
+      if [[ -n "$GKE_ADMIN" && -z "$CIRCLE_PULL_REQUESTS" ]]; then
+        export SHOULD_TEST=1
+      fi
+      ;;
+    # In kase of minikube+kafka we want to test it if
+    # it is a Pull Request related to Kafka (discovered from the PR title)
+    # or if the build is from the "master" branch
+    minikube_kafka)
+      if [[ -n "$CIRCLE_PULL_REQUESTS" ]]; then
+        pr_kafka_title=$(curl -H "Authorization: token $ACCESS_TOKEN" "https://api.github.com/repos/$TRAVIS_REPO_SLUG/pulls/${CIRCLE_PR_NUMBER}" | grep title || true)
+      fi
+      if [[ -z "$CIRCLE_PULL_REQUESTS" || "$pr_kafka_title" == "" || "$pr_kafka_title" =~ ^.*(Kafka|kafka|KAFKA).*$ ]]; then
+        export SHOULD_TEST=1
+      fi
+      ;;
+    # In kase of minikube+NATS we want to test it if
+    # it is a Pull Request related to NATS (discovered from the PR title)
+    # or if the build is from the "master" branch
+    minikube_nats)
+      if [[ -n "$CIRCLE_PULL_REQUESTS" ]]; then
+        pr_nats_title=$(curl -H "Authorization: token $ACCESS_TOKEN" "https://api.github.com/repos/$TRAVIS_REPO_SLUG/pulls/${TRAVIS_PULL_REQUEST}" | grep title || true)
+      fi
+      if [[ -z "$CIRCLE_PULL_REQUESTS" || "$pr_nats_title" == "" || "$pr_nats_title" =~ ^.*(Nats|nats|NATS).*$ ]]; then
+        export SHOULD_TEST=1
+      fi
+      ;;
+    esac
+    echo "Should test? $SHOULD_TEST"
+    if [[ "$SHOULD_TEST" != "1" ]]; then
+      circleci step halt
+    fi
+#### End of definitions
+
+workflows:
+  version: 2
+  kubeless:
+    jobs:
+      - build:
+          <<: *build_allways
+      - build_images:
+          <<: *build_allways
+          requires:
+            - build
+      - minikube:
+          <<: *build_allways
+          requires:
+            - build_images
+      - minikube_core_triggers:
+          <<: *build_allways
+          requires:
+            - build_images
+      - minikube_build_functions:
+          <<: *build_allways
+          requires:
+            - build_images
+      - minikube_kafka:
+          <<: *build_allways
+          requires:
+            - build_images
+      - minikube_nats:
+          <<: *build_allways
+          requires:
+            - build_images
+      - GKE:
+          <<: *build_allways
+          requires:
+            - build_images
+      - push_latest_images:
+          filters:
+            branches:
+              only: master
+          requires:
+            - minikube
+            - minikube_core_triggers
+            - minikube_build_functions
+            - minikube_kafka
+            - minikube_nats
+            - GKE
+      # TODO: Enable this when Travis release is disabled
+      # - release:
+      #     filters:
+      #       tags:
+      #         only: /v.*/
+      #       branches:
+      #         ignore: /.*/
+      #     requires:
+      #       - minikube
+      #       - minikube_core_triggers
+      #       - minikube_build_functions
+      #       - minikube_kafka
+      #       - minikube_nats
+      #       - GKE
+jobs:
+  build:
+    <<: *defaults
+    docker:
+      - image: circleci/golang:1.9 
+    working_directory: /go/src/github.com/kubeless/kubeless
+    steps:
+      - checkout
+      - <<: *exports
+      - run: make bootstrap
+      - run: make VERSION=${CONTROLLER_TAG} binary
+      - run: make test
+      - run: make validation
+      - run: make all-yaml
+      - run: |
+          mkdir build-manifests
+          manifests=(
+            kubeless.yaml kubeless-non-rbac.yaml kubeless-openshift.yaml
+            kafka-zookeeper.yaml kafka-zookeeper-openshift.yaml
+            nats.yaml
+          )
+          for f in "${manifests[@]}"; do
+            sed -i.bak 's/:latest/'":${CONTROLLER_TAG}"'/g' $f
+            cp $f build-manifests/
+          done
+      - persist_to_workspace:
+          root: /go
+          paths: 
+            - bin
+            - src/github.com/kubeless/kubeless/*yaml
+      - store_artifacts:
+          path: /go/bin/kubeless
+      - store_artifacts:
+          path: build-manifests
+  build_images:
+    <<: *defaults
+    docker:
+      - image: circleci/golang:1.9 
+    working_directory: /go/src/github.com/kubeless/kubeless
+    steps:
+      - setup_remote_docker
+      - checkout
+      - <<: *exports
+      - run: make bootstrap
+      - run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"      
+      - run: make controller-image CONTROLLER_IMAGE=$CONTROLLER_IMAGE
+      - run: docker push $CONTROLLER_IMAGE
+      - run: make kafka-controller-image KAFKA_CONTROLLER_IMAGE=$KAFKA_CONTROLLER_IMAGE
+      - run: docker push $KAFKA_CONTROLLER_IMAGE
+      - run: make nats-controller-image NATS_CONTROLLER_IMAGE=$NATS_CONTROLLER_IMAGE
+      - run: docker push $NATS_CONTROLLER_IMAGE
+      - run: make function-image-builder FUNCTION_IMAGE_BUILDER=$BUILDER_IMAGE
+      - run: docker push $BUILDER_IMAGE
+  minikube:
+    <<: *defaults
+    machine: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/go
+      - <<: *exports
+      - <<: *restore_workspace
+      - run: ./script/integration-tests minikube deployment
+      - run: ./script/integration-tests minikube basic
+  minikube_core_triggers:
+    <<: *defaults
+    machine: true
+    steps:
+      - checkout
+      - <<: *exports
+      - attach_workspace:
+          at: /tmp/go
+      - <<: *restore_workspace
+      - run: sudo apt-get update -y
+      - run: sudo apt-get install -y apache2-utils
+      - run: ./script/integration-tests minikube deployment
+      - run: ./script/integration-tests minikube cronjob
+      - run: ./script/integration-tests minikube http
+  minikube_build_functions:
+    <<: *defaults
+    machine: true
+    steps:
+      - checkout
+      - <<: *exports
+      - attach_workspace:
+          at: /tmp/go
+      - <<: *restore_workspace
+      - run: "echo '{\"host\": \"unix:///var/run/docker.sock\", \"storage-driver\": \"overlay2\", \"insecure-registries\" : [\"0.0.0.0/0\"]}' > /tmp/daemon.json"
+      - run: sudo mv /tmp/daemon.json /etc/docker/daemon.json
+      - run: sudo service docker restart
+      - run: docker info
+      - run: docker run -d -p 5000:5000 --restart=always --name registry -v /data/docker-registry:/var/lib/registry registry:2
+      - run: "sed -i.bak 's/enable-build-step: \"false\"/enable-build-step: \"true\"/g' kubeless.yaml"
+      - run: "sed -i.bak 's/function-registry-tls-verify: \"true\"/function-registry-tls-verify: \"false\"/g' kubeless.yaml"
+      - run: ./script/integration-tests minikube deployment
+      - run: ./script/integration-tests minikube prebuilt_functions
+  minikube_kafka:
+    <<: *defaults
+    machine: true
+    steps:
+      - <<: *should_test
+      - checkout
+      - <<: *exports
+      - attach_workspace:
+          at: /tmp/go
+      - <<: *restore_workspace
+      - run: ./script/integration-tests minikube deployment
+      - run: ./script/integration-tests minikube kafka
+  minikube_nats:
+    <<: *defaults
+    machine: true
+    steps:
+      - <<: *should_test
+      - checkout
+      - <<: *exports
+      - attach_workspace:
+          at: /tmp/go
+      - <<: *restore_workspace
+      - run: ./script/integration-tests minikube deployment
+      - run: ./script/integration-tests minikube nats
+  GKE:
+    <<: *defaults
+    docker:
+      - image: circleci/golang:1.9 
+    steps:
+      - <<: *should_test
+      - checkout
+      - <<: *exports
+      - attach_workspace:
+          at: /tmp/go
+      - <<: *restore_workspace
+      - run: ./script/enable-gcloud.sh $(pwd) > /dev/null
+      - run: echo "export ESCAPED_GKE_CLUSTER=$(echo ${GKE_CLUSTER}-ci-${CIRCLE_BRANCH} | sed 's/[^a-z0-9-]//g')" >> $BASH_ENV
+      - run: ./script/start-gke-env.sh $ESCAPED_GKE_CLUSTER $ZONE $GKE_VERSION $GKE_ADMIN > /dev/null
+      - run: ./script/integration-tests gke_${GKE_PROJECT}_${ZONE}_${ESCAPED_GKE_CLUSTER} deployment
+      - run: ./script/integration-tests gke_${GKE_PROJECT}_${ZONE}_${ESCAPED_GKE_CLUSTER} basic
+      - run: ./script/integration-tests gke_${GKE_PROJECT}_${ZONE}_${ESCAPED_GKE_CLUSTER} cronjob
+      - run: ./script/integration-tests gke_${GKE_PROJECT}_${ZONE}_${ESCAPED_GKE_CLUSTER} kafka
+  push_latest_images:
+    <<: *defaults
+    docker:
+      - image: circleci/golang:1.9 
+    steps:
+      - <<: *exports
+      - setup_remote_docker
+      - run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"   
+      - run: |
+          images=( 
+            $CONTROLLER_IMAGE_NAME
+            $BUILDER_IMAGE_NAME 
+            $KAFKA_CONTROLLER_IMAGE_NAME 
+            $NATS_CONTROLLER_IMAGE_NAME
+          )
+          for image in "${images[@]}"; do
+            echo "Pulling ${image}:${CONTROLLER_TAG}"
+            docker pull ${image}:${CONTROLLER_TAG}
+            docker tag ${image}:${CONTROLLER_TAG} ${image}:latest
+            docker push ${image}:latest
+          done
+  release:
+    <<: *defaults
+    docker:
+      - image: circleci/golang:1.9 
+    working_directory: /go/src/github.com/kubeless/kubeless
+    steps:
+      - <<: *exports
+      - checkout
+      - attach_workspace:
+          at: /tmp/go
+      - <<: *restore_workspace
+      - run: make VERSION=${CIRCLE_TAG} binary-cross
+      - run: for d in bundles/kubeless_*; do zip -r9 $d.zip $d/; done
+      - run: ./script/create_release.sh ${CIRCLE_TAG}

Makefile

@@ -121,14 +121,14 @@ bootstrap: bats ksonnet-lib
 	go get github.com/golang/lint/golint
 
 	@if ! which kubecfg >/dev/null; then \
-	wget -q -O $$GOPATH/bin/kubecfg https://github.com/ksonnet/kubecfg/releases/download/v0.6.0/kubecfg-$$(go env GOOS)-$$(go env GOARCH); \
-	chmod +x $$GOPATH/bin/kubecfg; \
+	sudo wget -q -O /usr/local/bin/kubecfg https://github.com/ksonnet/kubecfg/releases/download/v0.6.0/kubecfg-$$(go env GOOS)-$$(go env GOARCH); \
+	sudo chmod +x /usr/local/bin/kubecfg; \
 	fi
 
 	@if ! which kubectl >/dev/null; then \
 	KUBECTL_VERSION=$$(wget -qO- https://storage.googleapis.com/kubernetes-release/release/stable.txt); \
-	wget -q -O $$GOPATH/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$$KUBECTL_VERSION/bin/$$(go env GOOS)/$$(go env GOARCH)/kubectl; \
-	chmod +x $$GOPATH/bin/kubectl; \
+	sudo wget -q -O /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$$KUBECTL_VERSION/bin/$$(go env GOOS)/$$(go env GOARCH)/kubectl; \
+	sudo chmod +x /usr/local/bin/kubectl; \
 	fi
 
 build_and_test:

chart/README.md

@@ -15,3 +15,4 @@ helm init
 helm install --name kubeless --namespace kubeless ./kubeless
 ```
 
+After that, if you are having trouble deploying Kafka and Zookeeper, please check the specific guide [here](/docs/troubleshooting/#kafka-and-zookeeper-persistent-volume-creation) to create appropriate disks and PVs. If you are running Kubernetes in GKE,you can provision those persistent volumes manually deploying the manifests present in the [misc folder](https://github.com/kubeless/kubeless/tree/master/docs/misc). If you use other cloud provider, check [kubernetes docs](https://kubernetes.io/docs/concepts/storage/volumes/) to create these required volumes.

chart/kubeless/README.md

@@ -6,4 +6,4 @@ It installs:
 * The controller
 * The Kubeless configuration
 * The UI
-* A single node Kafka and Zookeeper setup
+* A single node Kafka and Zookeeper setup

cmd/kubeless/autoscale/autoscaleDelete.go

@@ -54,9 +54,9 @@ var autoscaleDeleteCmd = &cobra.Command{
 			if err != nil {
 				logrus.Fatal(err)
 			}
-			logrus.Infof("Removed Autoscaling rule from %s", funcName)
+			logrus.Infof("Remove Autoscaling rule from %s successfully", funcName)
 		} else {
-			logrus.Fatalf("Not found an auto scale definition for %s", funcName)
+			logrus.Fatalf("Not found an autoscale definition for %s", funcName)
 		}
 	},
 }

cmd/kubeless/autoscale/autoscaleList.go

@@ -85,7 +85,7 @@ func printAutoscale(w io.Writer, ass []v2beta1.HorizontalPodAutoscaler, output s
 			m := ""
 			v := ""
 			if len(i.Spec.Metrics) == 0 {
-				logrus.Errorf("The function autoscale %s isn't in correct format. It has no metric defined.", i.Name)
+				logrus.Errorf("The autoscale %s has bad format. It has no metric defined.", i.Name)
 				continue
 			}
 			if i.Spec.Metrics[0].Object != nil {
@@ -115,7 +115,7 @@ func printAutoscale(w io.Writer, ass []v2beta1.HorizontalPodAutoscaler, output s
 				}
 				fmt.Fprintln(w, string(b))
 			default:
-				return fmt.Errorf("Wrong output format. Please use only json|yaml")
+				return fmt.Errorf("Wrong output format. Only accept json|yaml file")
 			}
 		}
 	}