Permalink
Browse files

Log file access in K8s pods (#445)

  • Loading branch information...
DaoWen authored and shamsimam committed Sep 12, 2018
1 parent 5404a98 commit 5d07cf8e8e539de1e6de6ab42abb24d62f4d0045
@@ -0,0 +1,6 @@
FROM nginx:1.15-alpine

COPY fileserver-start /bin/
COPY nginx.conf /root/nginx.conf.template
RUN mkdir -p /srv/www && echo 'Hello!' > /srv/www/hello.txt
CMD ["fileserver-start"]
@@ -0,0 +1,19 @@
# Waiter-Kubernetes Fileserver Sidecar Container

Waiter on Marathon+Mesos uses the Mesos file-browsing REST API
to give users access to their service instances' local files (e.g., logs).
To provide a similar experience for Waiter on Kubernetes users,
we add a fileserver as a sidecar-container in each Waiter-managed pod.
The sidecar-container runs an nginx server,
mounting the same home directory volume used in the main app container,
and serving that mounted directory using automatic JSON directory indexing.

## Settings

By default, the nginx server binds to port 9090 in its container.
The default port can be overridden by setting `WAITER_FILESERVER_PORT`
in the container's environment to the desired port number.

## Building the Docker image

docker build -t twosigma/waiter-fileserver .
@@ -0,0 +1,11 @@
#!/bin/sh

# Set default server port
: ${WAITER_FILESERVER_PORT:=9090}
export WAITER_FILESERVER_PORT

# Generate server config from template
envsubst </root/nginx.conf.template >/root/nginx.conf

# Start server in non-daemon mode
nginx -c /root/nginx.conf
@@ -0,0 +1,20 @@
daemon off;
worker_processes 1;

events {
use epoll;
}

http {
server {
gzip on;
gzip_min_length 512;
gzip_types *;
listen ${WAITER_FILESERVER_PORT};
root /srv/www;
location / {
autoindex on;
autoindex_format json;
}
}
}
@@ -334,6 +334,18 @@
;; The default factory function accepts an option for the docker container to use in the pod.
:default-container-image "twosigma/kitchen:latest"}

;; Configuration for creating and querying a sidecar container in each Waiter Service Instance's Kuberentes pod
;; running a server for directory listings (in JSON) and serving file contents.
;; Waiter expects the directory listings in the format returned by nginx's autoindex module when configured for JSON.
;; See the behavior of the default container image below for reference.
;; The sidecar container is only added to Waiter-managed pods if the :port number is set in this map,
;; otherwise, the file browsing API is disabled (always returning an empty array).
;; You must ensure that the fileserver port number does not intersect with the pod-base-port range.
:fileserver {:image "twosigma/waiter-fileserver:latest"
:port 9090
:resources {:cpu 0.1 :mem 128}
:scheme "http"}

;; HTTP options that will be used when accessing the Kubernetes API:
:http-options {:conn-timeout 10000
:socket-timeout 10000}
@@ -21,8 +21,8 @@
; ---------- Scheduling ----------

:scheduler-config {:kind :kubernetes
:kubernetes {:url "http://localhost:8001"
:replicaset-spec-file-path "./specs/k8s-default-pod.edn"}}
:kubernetes {:fileserver {:port 591}
:url "http://localhost:8001"}}

; ---------- Error Handling ----------

@@ -184,7 +184,7 @@

(deftest ^:parallel ^:integration-fast test-basic-logs
(testing-using-waiter-url
(if (or (using-cook? waiter-url) (using-marathon? waiter-url))
(if (contains? #{"cook" "kubernetes" "marathon"} (scheduler-kind waiter-url))
(let [waiter-headers {:x-waiter-name (rand-name)}
{:keys [cookies router-id service-id]} (make-request-with-debug-info waiter-headers #(make-kitchen-request waiter-url %))
router-url (get (routers waiter-url) router-id)]
@@ -371,7 +371,7 @@
response-json (api-request http-client request-url
:body (utils/clj->json spec-json)
:request-method :post)]
(replicaset->Service response-json)))
(some-> response-json replicaset->Service)))

(defn- delete-service
"Delete the Kubernetes ReplicaSet corresponding to a Waiter Service.
@@ -413,6 +413,7 @@

; The Waiter Scheduler protocol implementation for Kubernetes
(defrecord KubernetesScheduler [api-server-url
fileserver
http-client
max-patch-retries
max-name-length
@@ -511,9 +512,35 @@
:result :failed
:message "Error while scaling waiter service"})))

(retrieve-directory-content [_ service-id instance-id _ relative-directory]
;; TODO (#357) - get access to the working directory contents in the pod
[])
(retrieve-directory-content
[{:keys [http-client] {:keys [port scheme]} :fileserver}
_ _ host browse-path]
(let [auth-str @k8s-api-auth-str
headers (when auth-str {"Authorization" auth-str})
browse-path (if (string/blank? browse-path) "/" browse-path)
browse-path (cond->
browse-path
(not (string/ends-with? browse-path "/"))
(str "/")
(not (string/starts-with? browse-path "/"))
(->> (str "/")))
target-url (str scheme "://" host ":" port browse-path)]
(when port
(ss/try+
(let [result (http-utils/http-request
http-client
target-url
:accept "application/json"
:content-type "application/json"
:headers headers)]
(for [{entry-name :name entry-type :type :as entry} result]
(if (= "file" entry-type)
(assoc entry :url (str target-url entry-name))
(assoc entry :path (str browse-path entry-name)))))
(catch [:client http-client] response
(log/error "request to fileserver failed: " target-url response))
(catch Throwable t
(log/error t "request to fileserver failed"))))))

(service-id->state [_ service-id]
{:failed-instances (vals (get @service-id->failed-instances-transient-store service-id))
@@ -525,7 +552,7 @@

(defn default-replicaset-builder
"Factory function which creates a Kubernetes ReplicaSet spec for the given Waiter Service."
[{:keys [orchestrator-name pod-base-port replicaset-api-version
[{:keys [fileserver orchestrator-name pod-base-port replicaset-api-version
service-id->password-fn] :as scheduler}
service-id
{:strs [backend-proto cmd cpus grace-period-secs health-check-interval-secs
@@ -553,45 +580,69 @@
health-check-url (sd/service-description->health-check-url service-description)
memory (str mem "Mi")
ssl? (= "https" backend-protocol-lower)]
{:kind "ReplicaSet"
:apiVersion replicaset-api-version
:metadata {:annotations {:waiter/service-id service-id}
:labels {:app k8s-name
:managed-by orchestrator-name}
:name k8s-name}
:spec {:replicas min-instances
:selector {:matchLabels {:app k8s-name
:managed-by orchestrator-name}}
:template {:metadata {:annotations {:waiter/port-count (str ports)
:waiter/protocol backend-protocol-lower
:waiter/service-id service-id}
:labels {:app k8s-name
:managed-by orchestrator-name}}
:spec {:containers [{:command ["/usr/bin/waiter-init" cmd]
:env env
:image default-container-image
:imagePullPolicy "IfNotPresent"
:livenessProbe {:httpGet {:path health-check-url
:port port0
:scheme backend-protocol-upper}
:failureThreshold health-check-max-consecutive-failures
:initialDelaySeconds grace-period-secs
:periodSeconds health-check-interval-secs
:timeoutSeconds 1}
:name k8s-name
:ports [{:containerPort port0}]
:readinessProbe {:httpGet {:path health-check-url
:port port0
:scheme backend-protocol-upper}
:failureThreshold 1
:periodSeconds health-check-interval-secs
:timeoutSeconds 1}
:resources {:limits {:cpu cpus
:memory memory}
:requests {:cpu cpus
:memory memory}}
:workingDir home-path}]
:terminationGracePeriodSeconds 0}}}}))
(cond->
{:kind "ReplicaSet"
:apiVersion replicaset-api-version
:metadata {:annotations {:waiter/service-id service-id}
:labels {:app k8s-name
:managed-by orchestrator-name}
:name k8s-name}
:spec {:replicas min-instances
:selector {:matchLabels {:app k8s-name
:managed-by orchestrator-name}}
:template {:metadata {:annotations {:waiter/port-count (str ports)
:waiter/protocol backend-protocol-lower
:waiter/service-id service-id}
:labels {:app k8s-name
:managed-by orchestrator-name}}
:spec {:containers [{:command ["/usr/bin/waiter-init" cmd]
:env env
:image default-container-image
:imagePullPolicy "IfNotPresent"
:livenessProbe {:httpGet {:path health-check-url
:port port0
:scheme backend-protocol-upper}
:failureThreshold health-check-max-consecutive-failures
:initialDelaySeconds grace-period-secs
:periodSeconds health-check-interval-secs
:timeoutSeconds 1}
:name "waiter-app"
:ports [{:containerPort port0}]
:readinessProbe {:httpGet {:path health-check-url
:port port0
:scheme backend-protocol-upper}
:failureThreshold 1
:periodSeconds health-check-interval-secs
:timeoutSeconds 1}
:resources {:limits {:cpu cpus
:memory memory}
:requests {:cpu cpus
:memory memory}}
:volumeMounts [{:mountPath home-path
:name "user-home"}]
:workingDir home-path}]
:volumes [{:name "user-home"
:emptyDir {}}]
:terminationGracePeriodSeconds 0}}}}
;; Optional fileserver sidecar container
(integer? (:port fileserver))
(update-in
[:spec :template :spec :containers]
conj
(let [{:keys [image port] {:keys [cpu mem]} :resources} fileserver
memory (str mem "Mi")]
{:command ["/bin/fileserver-start"]
:env [{:name "WAITER_FILESERVER_PORT"
:value (str port)}]
:image image
:imagePullPolicy "IfNotPresent"
:name "waiter-fileserver"
:ports [{:containerPort port}]
:resources {:limits {:cpu cpu :memory memory}
:requests {:cpu cpu :memory memory}}
:volumeMounts [{:mountPath "/srv/www"
:name "user-home"}]
:workingDir home-path})))))

(defn start-auth-renewer
"Initialize the k8s-api-auth-str atom,
@@ -618,8 +669,13 @@
[{:keys [authentication http-options max-patch-retries max-name-length orchestrator-name
pod-base-port pod-suffix-length replicaset-api-version replicaset-spec-builder
scheduler-name scheduler-state-chan scheduler-syncer-interval-secs service-id->service-description-fn
service-id->password-fn url start-scheduler-syncer-fn]}]
{:pre [(utils/pos-int? (:socket-timeout http-options))
service-id->password-fn url start-scheduler-syncer-fn]
{fileserver-port :port fileserver-scheme :scheme :as fileserver} :fileserver}]
{:pre [(or (nil? fileserver-port)
(and (integer? fileserver-port)
(< 0 fileserver-port 65535)))
(re-matches #"https?" fileserver-scheme)
(utils/pos-int? (:socket-timeout http-options))
(utils/pos-int? (:conn-timeout http-options))
(utils/non-neg-int? max-patch-retries)
(utils/pos-int? max-name-length)
@@ -656,6 +712,7 @@
(when authentication
(start-auth-renewer authentication))
(->KubernetesScheduler url
fileserver
http-client
max-patch-retries
max-name-length
@@ -289,8 +289,11 @@
:mesos-slave-port 5051
:search-interval-days 10}
:kubernetes {; Default values are not provided below for the following keys:
; :authentication :url
; :authentication [:fileserver :port] :url
:factory-fn 'waiter.scheduler.kubernetes/kubernetes-scheduler
:fileserver {:image "twosigma/waiter-fileserver:latest"
:resources {:cpu 0.1 :mem 128}
:scheme "http"}
:http-options {:conn-timeout 10000
:socket-timeout 10000}
:max-patch-retries 5
@@ -850,7 +850,7 @@
(log/debug "router url with slots assigned:" router-url)
router-url))

(defn- scheduler-kind
(defn scheduler-kind
"Returns the configured :scheduler-config :kind"
[waiter-url & {:keys [verbose] :or {verbose false}}]
(setting waiter-url [:scheduler-config :kind] :verbose verbose))
@@ -865,10 +865,15 @@
(:gracePeriodSeconds (first (:healthChecks (:app app-info-map))))))

(defn using-cook?
"Returns true if Waiter is configured to use Marathon for scheduling"
"Returns true if Waiter is configured to use Cook for scheduling"
[waiter-url]
(= "cook" (scheduler-kind waiter-url :verbose true)))

(defn using-k8s?
"Returns true if Waiter is configured to use Kubernetes for scheduling"
[waiter-url]
(= "kubernetes" (scheduler-kind waiter-url :verbose true)))

(defn using-marathon?
"Returns true if Waiter is configured to use Marathon for scheduling"
[waiter-url]
Oops, something went wrong.

0 comments on commit 5d07cf8

Please sign in to comment.