Add function logs support (#53)

Documentation/Architecture.md

@@ -110,4 +110,22 @@ be loaded with any function.
 When poolmgr needs to create a service for a function, it calls
 fetcher to fetch the function.  Fetcher downloads the function into a
 volume shared between fetcher and this environment container.  Poolmgr
-then requests the container to load the function.
+then requests the container to load the function.
+
+Logger
+-----------
+
+Logger helps to forward function logs to centralized db service for log
+persistence. Currently only influxdb is supported to store logs.
+Following is a diagram describe how log service works:
+
+![Logger Diagram](https://cloud.githubusercontent.com/assets/202578/23100399/b0e3ea00-f6ba-11e6-8f2f-6588cfef2e84.png)
+
+1. Pool manager choose a pod from pool to execute user function
+2. Pool manager makes a HTTP POST to logger helper, once helper receives 
+the request it creates a symlink to container log for fluentd.
+3. Fluentd reads log from symlink and pipes to influxdb
+4. `fission function logs ...` retrieve event logs from influxdb with 
+optional log filter
+5. Pool manager removes function pod from pool
+6. Pool manager asks logger helper to stop piping logs, logger removes symlink.

INSTALL.md

@@ -7,6 +7,7 @@
      * [Get and Run Fission: GKE or other Cloud](#get-and-run-fission-gke-or-other-cloud)
      * [Install the client CLI](#install-the-client-cli)
      * [Run an example](#run-an-example)
+     * [Enable Persistent Function Logs (Optional)](#enable-persistent-function-logs)
 
 ## Running Fission on your Cluster
 
@@ -110,3 +111,46 @@ Finally, you're ready to use Fission!
   $ curl http://$FISSION_ROUTER/hello
   Hello, world!
 ```
+
+
+### Enable Persistent Function Logs (Optional)
+
+Fission uses InfluxDB to store logs and fluentd to forward them from
+function pods into InfluxDB.  To setup both InfluxDB and fluentd:
+
+Edit `fission-logger.yaml` to add a username and password for the
+Influxdb deployment.  Then:
+
+```
+  $ kubectl create -f fission-logger.yaml
+```
+
+On the client side,
+
+If you're using minikube or a local cluster:
+
+```
+$ export FISSION_LOGDB=http://$(minikube ip):31315
+```
+
+If you're using GKE or other cloud:
+
+```
+$ export FISSION_LOGDB=http://$(kubectl --namespace fission get svc influxdb -o=jsonpath='{..ip}'):8086
+```
+
+That's it for setup.  You can now use this to view function logs:
+
+```
+  $ fission function logs --name hello
+```
+
+You can also list the all the pods that have hosted the function
+(including ones that aren't alive any more) and view logs for a
+particular pod:
+
+```
+  $ fission function pods --name hello
+
+  $ fission function logs --name hello --pod <pod name>
+```

README.md

@@ -182,6 +182,9 @@ Finally, you're ready to use Fission!
   Hello, world!
 ```
 
+You can also set up persistence for logs: [instructions here](INSTALL.md).
+
+
 Compiling Fission
 =================
 

fission-bundle/main.go

@@ -7,6 +7,7 @@ import (
 	"github.com/docopt/docopt-go"
 	"github.com/fission/fission/controller"
 	"github.com/fission/fission/kubewatcher"
+	"github.com/fission/fission/logger"
 	"github.com/fission/fission/poolmgr"
 	"github.com/fission/fission/router"
 )
@@ -47,6 +48,11 @@ func runKubeWatcher(controllerUrl, routerUrl string) {
 	}
 }
 
+func runLogger() {
+	logger.Start()
+	log.Fatalf("Error: Logger exited.")
+}
+
 func getPort(portArg interface{}) int {
 	portArgStr := portArg.(string)
 	port, err := strconv.Atoi(portArgStr)
@@ -80,6 +86,7 @@ Usage:
   fission-bundle --routerPort=<port> [--controllerUrl=<url> --poolmgrUrl=<url>]
   fission-bundle --poolmgrPort=<port> [--controllerUrl=<url> --namespace=<namespace>]
   fission-bundle --kubewatcher [--controllerUrl=<url> --routerUrl=<url>]
+  fission-bundle --logger
 Options:
   --controllerPort=<port>  Port that the controller should listen on.
   --routerPort=<port>      Port that the router should listen on.
@@ -91,6 +98,7 @@ Options:
   --filepath=<filepath>    Directory to store functions in.
   --namespace=<namespace>  Kubernetes namespace in which to run function containers. Defaults to 'fission-function'.
   --kubewatcher            Start Kubernetes events watcher.
+  --logger                 Start logger.
 `
 	arguments, err := docopt.Parse(usage, nil, true, "fission-bundle", false)
 	if err != nil {
@@ -123,5 +131,9 @@ Options:
 		runKubeWatcher(controllerUrl, routerUrl)
 	}
 
+	if arguments["--logger"] == true {
+		runLogger()
+	}
+
 	select {}
 }

fission-cloud.yaml

@@ -30,3 +30,22 @@ spec:
     nodePort: 31313
   selector:
     svc: controller
+
+---
+# If you want to switch to external db service, please checkout 
+# sample Kubernetes Service config in fission-logger.yaml.
+apiVersion: v1
+kind: Service
+metadata:
+  name: influxdb
+  namespace: fission
+  labels:
+    svc: influxdb
+spec:
+  type: LoadBalancer
+  ports:
+  - port: 8086
+    targetPort: 8086
+    nodePort: 31315
+  selector:
+    svc: influxdb

fission-logger.yaml

@@ -0,0 +1,122 @@
+# Sample Kubernetes Service config for external db service. 
+#
+# apiVersion: v1
+# kind: Service
+# metadata:
+#   name: influxdb
+#   namespace: fission
+# spec:
+#   ports:
+#   - name: "8086"
+#     port: 8086
+#     targetPort: 8086
+#     protocol: TCP
+
+# ---
+# apiVersion: v1
+# kind: Endpoints
+# metadata:
+#   name: influxdb
+#   namespace: fission
+# subsets:
+#   - addresses:
+#     - ip: replace.influxdb.host.here
+#     ports:
+#     - name: "8086"
+#       port: 8086
+#       protocol: TCP
+
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: influxdb
+  namespace: fission
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        svc: influxdb
+    spec:
+      containers:
+      - name: influxdb
+        image: tutum/influxdb
+        env:
+          - name: PRE_CREATE_DB
+            value: fissionFunctionLog
+          # Create a random username/password for InfluxDB here, and
+          # repeat it in the fluentd spec below.
+          - name: ADMIN_USER
+            value: ""
+          - name: INFLUXDB_INIT_PWD
+            value: ""
+
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: logger
+  namespace: fission
+spec:
+  template:
+    metadata:
+      labels:
+        svc: logger
+    spec:
+      containers:
+        - name: logger
+          image: fission/fission-bundle
+          imagePullPolicy: IfNotPresent
+          command: ["/fission-bundle"]
+          args: ["--logger"]
+          volumeMounts:
+            - name: container-log
+              mountPath: /var/log/containers
+              readOnly: true
+            - name: docker-log
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+            - name: fission-log
+              mountPath: /var/log/fission
+              readOnly: false
+          ports:
+            - containerPort: 1234
+              hostPort: 1234
+              protocol: TCP
+        - name: fluentd
+          image: fission/fluentd
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: INFLUXDB_ADDRESS
+              value: influxdb
+            - name: INFLUXDB_PORT
+              value: "8086"
+            - name: INFLUXDB_DBNAME
+              value: "fissionFunctionLog"
+            # Username/password for fluentd to push logs to
+            # influxdb. This should match the influxdb
+            # username/passowrd defined above.
+            - name: INFLUXDB_USERNAME
+              value: ""
+            - name: INFLUXDB_PASSWD
+              value: ""
+          volumeMounts:
+            - name: container-log
+              mountPath: /var/log/containers
+              readOnly: true
+            - name: docker-log
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+            - name: fission-log
+              mountPath: /var/log/fission
+              readOnly: false
+      volumes:
+        - name: container-log
+          hostPath:
+              path: /var/log/containers
+        - name: docker-log
+          hostPath:
+              path: /var/lib/docker/containers
+        - name: fission-log
+          hostPath:
+              path: /var/log/fission

fission-nodeport.yaml

@@ -30,3 +30,22 @@ spec:
     nodePort: 31313
   selector:
     svc: controller
+
+---
+# If you want to switch to external db service, please checkout 
+# sample Kubernetes Service config in fission-logger.yaml.
+apiVersion: v1
+kind: Service
+metadata:
+  name: influxdb
+  namespace: fission
+  labels:
+    svc: influxdb
+spec:
+  type: NodePort
+  ports:
+  - port: 8086
+    targetPort: 8086
+    nodePort: 31315
+  selector:
+    svc: influxdb