From 145c3241109fa1562fff9f1c785c56dac25182a6 Mon Sep 17 00:00:00 2001 From: bilalcaliskan Date: Tue, 7 Dec 2021 01:43:59 +0300 Subject: [PATCH] varnish deployment files are added, closes #23 --- README.md | 22 +- .../invalidator/clusterrolebinding.yaml | 14 + .../deployment.yaml} | 53 +--- deployment/invalidator/kustomization.yaml | 10 + deployment/invalidator/service.yaml | 23 ++ deployment/invalidator/serviceaccount.yaml | 7 + deployment/varnish/default.vcl | 249 ++++++++++++++++++ deployment/varnish/deployment.yaml | 38 +++ deployment/varnish/kustomization.yaml | 13 + deployment/varnish/service.yaml | 14 + internal/web/handlers.go | 41 --- internal/web/utils.go | 1 - 12 files changed, 387 insertions(+), 98 deletions(-) create mode 100644 deployment/invalidator/clusterrolebinding.yaml rename deployment/{sample.yaml => invalidator/deployment.yaml} (57%) create mode 100644 deployment/invalidator/kustomization.yaml create mode 100644 deployment/invalidator/service.yaml create mode 100644 deployment/invalidator/serviceaccount.yaml create mode 100644 deployment/varnish/default.vcl create mode 100644 deployment/varnish/deployment.yaml create mode 100644 deployment/varnish/kustomization.yaml create mode 100644 deployment/varnish/service.yaml diff --git a/README.md b/README.md index f39bbc4..c858664 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Go version](https://img.shields.io/github/go-mod/go-version/bilalcaliskan/varnish-cache-invalidator)](https://github.com/bilalcaliskan/varnish-cache-invalidator) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -This tool basically multiplexes **BAN** and **PURGE** requests on [Varnish Cache](https://github.com/varnishcache/varnish-cache) +This tool basically multiplexes **PURGE** requests on [Varnish Cache](https://github.com/varnishcache/varnish-cache) instances at the same time to manage the cache properly. If you are using Varnish Enterprise, you already have that feature. varnish-cache-invalidator can be used for standalone Varnish instances or Varnish pods inside a Kubernetes cluster. @@ -25,7 +25,7 @@ Please check all the command line arguments on **Configuration** section. Requir **Kubernetes mode** In that mode, varnish-cache-invalidator discovers kube-apiserver for running [Varnish](https://github.com/varnishcache/varnish-cache) pods inside -Kubernetes and multiplexes **BAN** and **PURGE** requests on them at the same time to manage the cache properly. +Kubernetes and multiplexes **PURGE** requests on them at the same time to manage the cache properly. Please check all the command line arguments on **Configuration** section. Required args for Kubernetes mode: ``` @@ -34,11 +34,21 @@ Please check all the command line arguments on **Configuration** section. Requir ## Installation ### Kubernetes +varnish-cache invalidator requires Kustomize for in-kubernetes installations. You can refer [here](https://kustomize.io/) for +Kustomize installation. + Varnish-cache-invalidator can be run inside a Kubernetes cluster to multiplex requests for in-cluster Varnish containers. -You can use [sample deployment file](deployment/sample.yaml) to deploy your Kubernetes cluster. +You can use [deployment/invalidator](deployment/invalidator) folder to deploy it with sample configuration. For that, please +run below command in the [deployment/invalidator](deployment/invalidator) directory: +```shell +$ kustomize build . | kubectl apply -f - +``` +If you need to also deploy Varnish on Kubernetes, you can use [deployment/varnish](deployment/varnish) folder to deploy it +with sample [default.vcl](deployment/varnish/default.vcl). For that, please run below command in the [deployment/varnish](deployment/varnish) +directory: ```shell -$ kubectl create -f config/sample.yaml +$ kustomize build . | kubectl apply -f - ``` ### Standalone @@ -52,7 +62,7 @@ $ ./varnish-cache-invalidator --inCluster=false --targetHosts 10.0.0.100:6081,10 ## Configuration Varnish-cache-invalidator can be customized with several command line arguments. You can pass command line arguments via -[sample deployment file](deployment/sample.yaml). Here is the list of arguments you can pass: +[sample deployment file](deployment/invalidator/sample.yaml). Here is the list of arguments you can pass: ``` --inCluster bool InCluster is the boolean flag if varnish-cache-invalidator is running inside cluster or not, @@ -85,7 +95,7 @@ varnish-cache-invalidator uses [client-go](https://github.com/kubernetes/client- with `kube-apiserver`. [client-go](https://github.com/kubernetes/client-go) uses the [service account token](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) mounted inside the Pod at the `/var/run/secrets/kubernetes.io/serviceaccount` path while initializing the client. -If you have RBAC enabled on your cluster, when you applied the sample deployment file [deployment/sample.yaml](deployment/sample.yaml), +If you have RBAC enabled on your cluster, when you applied the sample deployment file [deployment/sample.yaml](deployment/invalidator/sample.yaml), it will create required serviceaccount and clusterrolebinding and then use that serviceaccount to be used by our varnish-cache-invalidator pods. diff --git a/deployment/invalidator/clusterrolebinding.yaml b/deployment/invalidator/clusterrolebinding.yaml new file mode 100644 index 0000000..b8113d8 --- /dev/null +++ b/deployment/invalidator/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: varnish-cache-invalidator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: view +subjects: + - kind: ServiceAccount + name: varnish-cache-invalidator + namespace: default diff --git a/deployment/sample.yaml b/deployment/invalidator/deployment.yaml similarity index 57% rename from deployment/sample.yaml rename to deployment/invalidator/deployment.yaml index 73bc7d9..1792d07 100644 --- a/deployment/sample.yaml +++ b/deployment/invalidator/deployment.yaml @@ -1,28 +1,5 @@ --- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: varnish-cache-invalidator - namespace: default - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: varnish-cache-invalidator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: view -subjects: - - kind: ServiceAccount - name: varnish-cache-invalidator - namespace: default - ---- - apiVersion: apps/v1 kind: Deployment metadata: @@ -46,9 +23,9 @@ spec: containers: - command: [ "./main" ] args: [ - "--varnishNamespace", "default", - "--varnishLabel", "'app=varnish'", - "--inCluster=true" + "--varnishNamespace", "default", + "--varnishLabel", "'app=varnish'", + "--inCluster=true" ] image: 'docker.io/bilalcaliskan/varnish-cache-invalidator:latest' imagePullPolicy: Always @@ -74,27 +51,3 @@ spec: tcpSocket: port: 3000 timeoutSeconds: 10 - ---- - -apiVersion: v1 -kind: Service -metadata: - labels: - app: varnish-cache-invalidator - name: varnish-cache-invalidator -spec: - ports: - - name: 3000-tcp - port: 3000 - protocol: TCP - targetPort: 3000 - - name: 3001-tcp - port: 3001 - protocol: TCP - targetPort: 3001 - selector: - app: varnish-cache-invalidator - deployment: varnish-cache-invalidator - sessionAffinity: None - type: NodePort diff --git a/deployment/invalidator/kustomization.yaml b/deployment/invalidator/kustomization.yaml new file mode 100644 index 0000000..4f79183 --- /dev/null +++ b/deployment/invalidator/kustomization.yaml @@ -0,0 +1,10 @@ +--- + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - serviceaccount.yaml + - clusterrolebinding.yaml + - deployment.yaml + - service.yaml diff --git a/deployment/invalidator/service.yaml b/deployment/invalidator/service.yaml new file mode 100644 index 0000000..6b5eea0 --- /dev/null +++ b/deployment/invalidator/service.yaml @@ -0,0 +1,23 @@ +--- + +apiVersion: v1 +kind: Service +metadata: + labels: + app: varnish-cache-invalidator + name: varnish-cache-invalidator +spec: + ports: + - name: 3000-tcp + port: 3000 + protocol: TCP + targetPort: 3000 + - name: 3001-tcp + port: 3001 + protocol: TCP + targetPort: 3001 + selector: + app: varnish-cache-invalidator + deployment: varnish-cache-invalidator + sessionAffinity: None + type: NodePort diff --git a/deployment/invalidator/serviceaccount.yaml b/deployment/invalidator/serviceaccount.yaml new file mode 100644 index 0000000..e415ffd --- /dev/null +++ b/deployment/invalidator/serviceaccount.yaml @@ -0,0 +1,7 @@ +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: varnish-cache-invalidator + namespace: default diff --git a/deployment/varnish/default.vcl b/deployment/varnish/default.vcl new file mode 100644 index 0000000..f7258a8 --- /dev/null +++ b/deployment/varnish/default.vcl @@ -0,0 +1,249 @@ +vcl 4.1; + +import directors; +import std; + +backend default { + .host = "varnish-cache:80"; +} + +sub vcl_init { + # Called when VCL is loaded, before any requests pass through it. + # Typically used to initialize VMODs. + + new vdir = directors.round_robin(); + vdir.add_backend(default); +} + +sub vcl_pipe { + # Called upon entering pipe mode. + # In this mode, the request is passed on to the backend, and any further data from both the client + # and backend is passed on unaltered until either end closes the connection. Basically, Varnish will + # degrade into a simple TCP proxy, shuffling bytes back and forth. For a connection in pipe mode, + # no other VCL subroutine will ever get called after vcl_pipe. + + # Note that only the first request to the backend will have + # X-Forwarded-For set. If you use X-Forwarded-For and want to + # have it set for all requests, make sure to have: + # set bereq.http.connection = "close"; + # here. It is not set by default as it might break some broken web + # applications, like IIS with NTLM authentication. + + # set bereq.http.Connection = "Close"; + + # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html) + if (req.http.upgrade) { + set bereq.http.upgrade = req.http.upgrade; + } + + return (pipe); +} + +sub vcl_pass { + # Called upon entering pass mode. In this mode, the request is passed on to the backend, and the + # backend's response is passed on to the client, but is not entered into the cache. Subsequent + # requests submitted over the same client connection are handled normally. + + # return (pass); +} + +# The data on which the hashing will take place +sub vcl_hash { + # Called after vcl_recv to create a hash value for the request. This is used as a key + # to look up the object in Varnish. + + hash_data(req.url); + + if (req.http.host) { + hash_data(req.http.host); + } else { + hash_data(server.ip); + } + + # hash cookies for requests that have them + if (req.http.Cookie) { + hash_data(req.http.Cookie); + } + + # Cache the HTTP vs HTTPs separately + if (req.http.X-Forwarded-Proto) { + hash_data(req.http.X-Forwarded-Proto); + } +} + +sub vcl_hit { + # Called when a cache lookup is successful. + + if (obj.ttl >= 0s) { + # A pure unadultered hit, deliver it + return (deliver); + } + + # https://www.varnish-cache.org/docs/trunk/users-guide/vcl-grace.html + # When several clients are requesting the same page Varnish will send one request to the backend and place the others + # on hold while fetching one copy from the backend. In some products this is called request coalescing and Varnish does + # this automatically. + # If you are serving thousands of hits per second the queue of waiting requests can get huge. There are two potential + # problems - one is a thundering herd problem - suddenly releasing a thousand threads to serve content might send the + # load sky high. Secondly - nobody likes to wait. To deal with this we can instruct Varnish to keep the objects in cache + # beyond their TTL and to serve the waiting requests somewhat stale content. + + # if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) { + # return (deliver); + # } else { + # return (miss); + # } + + # We have no fresh fish. Lets look at the stale ones. + if (std.healthy(req.backend_hint)) { + # Backend is healthy. Limit age to 10s. + if (obj.ttl + 10s > 0s) { + # set req.http.grace = "normal(limited)"; + return (deliver); + } + } else { + # backend is sick - use full grace + if (obj.ttl + obj.grace > 0s) { + # set req.http.grace = "full"; + return (deliver); + } + } +} + +sub vcl_miss { + # Called after a cache lookup if the requested document was not found in the cache. Its purpose + # is to decide whether or not to attempt to retrieve the document from the backend, and which + # backend to use. + + return (fetch); +} + +# The routine when we deliver the HTTP request to the user +# Last chance to modify headers that are sent to the client +sub vcl_deliver { + # Called before a cached object is delivered to the client. + + # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed + if (obj.hits > 0) { + set resp.http.X-Cache = "HIT"; + } else { + set resp.http.X-Cache = "MISS"; + } + + # Please note that obj.hits behaviour changed in 4.0, now it counts per objecthead, not per object + # and obj.hits may not be reset in some cases where bans are in use. See bug 1492 for details. + # So take hits with a grain of salt + set resp.http.X-Cache-Hits = obj.hits; + + # Remove some headers: PHP version + unset resp.http.X-Powered-By; + + # Remove some headers: Apache version & OS + unset resp.http.Server; + unset resp.http.X-Drupal-Cache; + unset resp.http.X-Varnish; + unset resp.http.Via; + unset resp.http.Link; + unset resp.http.X-Generator; + + return (deliver); +} + +sub vcl_purge { + # Only handle actual PURGE HTTP methods, everything else is discarded + if (req.method == "PURGE") { + # restart request + set req.http.X-Purge = "Yes"; + return(restart); + } +} + +sub vcl_synth { + if (resp.status == 720) { + # We use this special error status 720 to force redirects with 301 (permanent) redirects + # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html")); + set resp.http.Location = resp.reason; + set resp.status = 301; + return (deliver); + } elseif (resp.status == 721) { + # And we use error status 721 to force redirects with a 302 (temporary) redirect + # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html")); + set resp.http.Location = resp.reason; + set resp.status = 302; + return (deliver); + } + return (deliver); +} + +sub vcl_fini { + # Called when VCL is discarded only after all requests have exited the VCL. + # Typically used to clean up VMODs. + + return (ok); +} + +sub vcl_recv { + # Called at the beginning of a request, after the complete request has been received and parsed. + # Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable, + # which backend to use. + # also used to modify the request + + set req.backend_hint = vdir.backend(); # send all traffic to the vdir director + + # Normalize the header if it exists, remove the port (in case you're testing this on various TCP ports) + if (req.http.Host) { + set req.http.Host = regsub(req.http.Host, ":[0-9]+", ""); + } + + # Remove the proxy header (see https://httpoxy.org/#mitigate-varnish) + unset req.http.proxy; + + # Normalize the query arguments + set req.url = std.querysort(req.url); + + if (req.method == "PURGE") { + return (purge); + } + + # Only deal with "normal" types + if (req.method != "GET" && req.method != "HEAD" && req.method != "PUT" && + req.method != "POST" && req.method != "TRACE" && req.method != "OPTIONS" && + req.method != "PATCH" && req.method != "DELETE") { + return (pipe); + } + + # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html) + if (req.http.Upgrade ~ "(?i)websocket") { + return (pipe); + } + + # Only cache GET or HEAD requests. This makes sure the POST requests are always passed. + if (req.method != "GET" && req.method != "HEAD") { + return (pass); + } + + + # Backend declaration + if (req.http.host == "varnish-cache") { + set req.backend_hint = default; + if (! req.url ~ "/") { + return(pass); + } + } +} + +sub vcl_backend_response { + # Happens after we have read the response headers from the backend. + # Here you clean the response headers, removing silly Set-Cookie headers + # and other mistakes your backend does. + + set beresp.ttl = 5m; + set beresp.grace = 30m; + + # Don't cache 50x responses + if (beresp.status == 500 || beresp.status == 502 || beresp.status == 503 || beresp.status == 504) { + return (abandon); + } + + return (deliver); +} diff --git a/deployment/varnish/deployment.yaml b/deployment/varnish/deployment.yaml new file mode 100644 index 0000000..a5fae65 --- /dev/null +++ b/deployment/varnish/deployment.yaml @@ -0,0 +1,38 @@ +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: varnish-cache + namespace: default +spec: + replicas: 2 + selector: + matchLabels: + app: varnish-cache + template: + metadata: + name: varnish-cache + labels: + app: varnish-cache + spec: + containers: + - name: varnish-cache + image: varnish:7.0.1 + imagePullPolicy: IfNotPresent + env: + - name: VARNISH_SIZE + value: 128m + ports: + - containerPort: 80 + volumeMounts: + - mountPath: /etc/varnish/default.vcl + name: varnish-config + subPath: default.vcl + volumes: + - name: varnish-config + configMap: + name: varnish-config + items: + - key: default.vcl + path: default.vcl diff --git a/deployment/varnish/kustomization.yaml b/deployment/varnish/kustomization.yaml new file mode 100644 index 0000000..c06bf91 --- /dev/null +++ b/deployment/varnish/kustomization.yaml @@ -0,0 +1,13 @@ +--- + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - deployment.yaml + - service.yaml + +configMapGenerator: + - name: varnish-config + files: + - default.vcl diff --git a/deployment/varnish/service.yaml b/deployment/varnish/service.yaml new file mode 100644 index 0000000..b2d8c52 --- /dev/null +++ b/deployment/varnish/service.yaml @@ -0,0 +1,14 @@ +--- + +apiVersion: v1 +kind: Service +metadata: + name: varnish-cache + namespace: default +spec: + ports: + - name: "http" + port: 80 + targetPort: 80 + selector: + app: varnish-cache diff --git a/internal/web/handlers.go b/internal/web/handlers.go index 4c515d5..46ef38a 100644 --- a/internal/web/handlers.go +++ b/internal/web/handlers.go @@ -8,47 +8,6 @@ import ( "go.uber.org/zap" ) -func banHandler(w http.ResponseWriter, r *http.Request) { - var successCount int - var response string - logger = logger.With(zap.String("requestMethod", "BAN")) - banRegex := r.Header.Get("ban-regex") - if banRegex == "" { - logger.Error("Unable to make a request to Varnish targets, header ban-regex must be set!") - http.Error(w, "Header ban-regex must be set!", http.StatusBadRequest) - return - } - - for _, v := range options.VarnishInstances { - req, _ := http.NewRequest("BAN", *v, nil) - req.Header.Set("ban-url", banRegex) - logger.Info("Making BAN request", zap.String("targetHost", *v)) - res, err := client.Do(req) - if err != nil { - logger.Error("An error occurred while making BAN request", zap.String("targetHost", *v), - zap.String("error", err.Error())) - } - - if res != nil && res.StatusCode == http.StatusOK { - successCount++ - } - - } - - if successCount == len(options.VarnishInstances) { - logger.Info("All BAN requests succeeded on Varnish pods!", zap.Int("successCount", successCount)) - w.WriteHeader(http.StatusOK) - } else { - logger.Warn("One or more Varnish BAN requests failed", zap.Int("successCount", successCount), - zap.Int("failureCount", len(options.VarnishInstances)-successCount)) - response = fmt.Sprintf("One or more Varnish BAN requests failed, check the logs!\nSucceeded request = %d\n"+ - "Failed request = %d", successCount, len(options.VarnishInstances)-successCount) - w.WriteHeader(http.StatusBadRequest) - } - - writeResponse(w, response) -} - func purgeHandler(w http.ResponseWriter, r *http.Request) { var successCount int var response string diff --git a/internal/web/utils.go b/internal/web/utils.go index e1f0f31..711ee05 100644 --- a/internal/web/utils.go +++ b/internal/web/utils.go @@ -11,7 +11,6 @@ import ( // registerHandlers registers the handlers of the web server func registerHandlers(router *mux.Router) { - router.HandleFunc("/ban", banHandler).Methods("BAN").Schemes("http").Name("ban") router.HandleFunc("/purge", purgeHandler).Methods("PURGE").Schemes("http").Name("purge") }