From 0bf814b7036f989f28cea2ebcad386b66b3a6c64 Mon Sep 17 00:00:00 2001 From: Fifciuu Date: Sun, 9 Feb 2020 15:42:57 +0100 Subject: [PATCH 1/8] MVP --- config/default.json | 6 ++ docker-compose.nodejs.yml | 6 ++ docker-compose.varnish.yml | 16 +++ docker-compose.yml | 10 ++ docker/varnish/Dockerfile | 19 ++++ docker/varnish/README.md | 199 +++++++++++++++++++++++++++++++++++++ docker/varnish/config.vcl | 130 ++++++++++++++++++++++++ 7 files changed, 386 insertions(+) create mode 100644 docker-compose.varnish.yml create mode 100644 docker/varnish/Dockerfile create mode 100644 docker/varnish/README.md create mode 100644 docker/varnish/config.vcl diff --git a/config/default.json b/config/default.json index 32f934f6..6397f8c0 100644 --- a/config/default.json +++ b/config/default.json @@ -116,6 +116,12 @@ ] } }, + "varnish": { + "host": "185.246.52.88", + "port": 80, + "method": "BAN", + "enabled": false + }, "redis": { "host": "localhost", "port": 6379, diff --git a/docker-compose.nodejs.yml b/docker-compose.nodejs.yml index 10fb5a4c..a658535b 100644 --- a/docker-compose.nodejs.yml +++ b/docker-compose.nodejs.yml @@ -26,3 +26,9 @@ services: - /var/www/dist ports: - '8080:8080' + networks: + - some-net + +networks: + some-net: + driver: bridge \ No newline at end of file diff --git a/docker-compose.varnish.yml b/docker-compose.varnish.yml new file mode 100644 index 00000000..71c93665 --- /dev/null +++ b/docker-compose.varnish.yml @@ -0,0 +1,16 @@ +version: '3.0' +services: + varnish: + build: + context: . + dockerfile: varnish/Dockerfile + volumes: + - ./docker/varnish/config.vcl:/usr/local/etc/varnish/default.vcl + ports: + - '1234:80' + networks: + - vuestorefrontapi_some-net + +networks: + vuestorefrontapi_some-net: + external: true \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 6b42d798..b13fc05d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,8 @@ services: - cluster.name=docker-cluster - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xmx512m -Xms512m" + networks: + - vuestorefrontapi_some-net kibana: build: docker/kibana/ @@ -26,11 +28,19 @@ services: - '5601:5601' depends_on: - es1 + networks: + - vuestorefrontapi_some-net redis: image: 'redis:4-alpine' ports: - '6379:6379' + networks: + - vuestorefrontapi_some-net volumes: esdat1: + +networks: + vuestorefrontapi_some-net: + external: true \ No newline at end of file diff --git a/docker/varnish/Dockerfile b/docker/varnish/Dockerfile new file mode 100644 index 00000000..4c3ab637 --- /dev/null +++ b/docker/varnish/Dockerfile @@ -0,0 +1,19 @@ +FROM cooptilleuls/varnish:6.0-stretch + +# install varnish-modules +RUN apt-get update -y && \ + apt-get install -y build-essential automake libtool curl git python-docutils && \ + curl -s https://packagecloud.io/install/repositories/varnishcache/varnish60/script.deb.sh | bash; + +RUN apt-get install -y pkg-config libvarnishapi1 libvarnishapi-dev autotools-dev; + +RUN git clone https://github.com/varnish/varnish-modules.git /tmp/vm; +RUN cd /tmp/vm; \ + git checkout 6.0; \ + ./bootstrap && \ + ./configure; + +RUN cd /tmp/vm && \ + make && \ + make check && \ + make install; \ No newline at end of file diff --git a/docker/varnish/README.md b/docker/varnish/README.md new file mode 100644 index 00000000..e1d1fc2c --- /dev/null +++ b/docker/varnish/README.md @@ -0,0 +1,199 @@ +### Tutorial +1. Create network with `docker network create ` +2. Use `docker network ls` and find your network. It should have prefix! +E.g. when I used `docker network create some-net`, I have network with name `vuestorefrontapi_some-net` +3. Open docker-compose.yml: +At the end: +``` +networks: + vuestorefrontapi_some-net: + external: true +``` +Set vuestorefrontapi_some-net to your network name + +4. Check each `docker-compose` file and set proper network name. +5. In the docker-compose.nodejs.yml it should not have a prefix, e.g: +``` + networks: + - some-net + +networks: + some-net: + driver: bridge +``` + +### How does it work? +1. I add output tags to the VSF-API response: +``` +const tagsHeader = output.tags.join(' ') +res.setHeader('X-VS-Cache-Tag', tagsHeader) +``` + +2. After it invalidates cache in the Redis. I forward request to the: +``` +http://${config.varnish.host}:${config.varnish.port}/ +``` +With invalidate tag in headers: +``` +headers: { + "X-VS-Cache-Tag": tag +} +``` + +I set Varnish invalidate method to `BAN` but you can change it in your config + varnish's config. + +3. Configuration of BANning we have inside `docker/varnish/config.vcl` in `vcl_recv`. +It tries to BAN resource which has `X-VS-Cache-Tag` header: +``` +# Logic for the ban, using the X-Cache-Tag header. +if (req.http.X-VS-Cache-Tag) { + ban("obj.http.X-VS-Cache-Tag ~ " + req.http.X-VS-Cache-Tag); +} +``` + +Below under BANning logic. I have to tell Varnish what to cache. +``` +if (req.url ~ "^\/api\/catalog\/") { + if (req.method == "POST") { + # It will allow me to cache by req body in the vcl_hash + std.cache_req_body(500KB); + set req.http.X-Body-Len = bodyaccess.len_req_body(); + } + + if ((req.method == "POST" || req.method == "GET")) { + return (hash); + } +} +``` + +I am caching request that starts with `/api/catalog/`. As you can see I cache both POST and GET. +This is because in my project I use huge ES requests to compute Facted Filters. I would exceed HTTP GET limit. + +Thanks to this line and `bodyaccess`, I can distinguish requests to the same URL by their body! +``` +std.cache_req_body(500KB); +``` + +Then in `vcl_hash` I create hash for POST requests with `bodyaccess.hash_req_body()`: +``` +sub vcl_hash { + # To cache POST and PUT requests + if (req.http.X-Body-Len) { + bodyaccess.hash_req_body(); + } else { + hash_data(""); + } +} +``` + +By default, Varnish change each request to HTTP GET. We need to tell him to send POST requests to the VSF-API as POST - not GET. +We will do it like that: +``` +sub vcl_backend_fetch { + if (bereq.http.X-Body-Len) { + set bereq.method = "POST"; + } +} +``` + + +### Caching Stock +It might be a good idea to cache stock requests if you check it lifetime in VSF-PWA in visiblityChanged hook (product listing). +In one project when I have slow Magento - it reduced Time-To-Response from ~2s to ~70ms. + +``` +if (req.url ~ "^\/api\/stock\/") { + if (req.method == "GET") { + # M2 Stock + return (hash); + } +} +``` + +Then in `vcl_backend_response` you should set safe TTL (Time to live) for your stock cache. I've set 15 minutes (900 seconds) +``` +sub vcl_backend_response { + # Set ban-lurker friendly custom headers. + if (beresp.http.X-VS-Cache && beresp.http.X-VS-Cache ~ "Miss") { + set beresp.ttl = 0s; + } + if (bereq.url ~ "^\/api\/stock\/") { + set beresp.ttl = 900s; // 15 minutes + } + set beresp.http.X-Url = bereq.url; + set beresp.http.X-Host = bereq.http.host; +} +``` + +For X-VS-Cache, I set TTL 0s so it is permanent. Because it will be automaticly invalidated when needed. + +### Caching Extensions +You might want to cache response from various extensions. +E.g. I am fetching Menus, Available Countries (for checkout) from M2 by VSF-API proxy. +As in this project Magento is pretty slow. By caching responses I've changed response time from ~2s +to around ~50ms. + +How to do that? +Inside `vcl_recv` add: +``` +// As in my case I want to cache only GET requests +if (req.method == "GET") { + # Countries for storecode GET - M2 - /directory/countries + if (req.url ~ "^\/api\/ext\/directory\/") { + return (hash); + } + + # Menus GET - M2 - /menus & /nodes + if (req.url ~ "^\/api\/ext\/menus\/") { + return (hash); + } +} +``` + +How to invalidate extension's tag? +You can do it by sending request with `X-VS-Cache-Ext` header. +If value of this header is part of any cached URL - it will be invalidated. +E.g. for menus extension: +``` +/api/ext/menus +``` +You could send: +BAN `http://${config.varnish.host}:${config.varnish.port}/` +headers: { + "X-VS-Cache-Ext": "menus" +} + +But sending HTTP requests is not so handy. So I've extended Invalidate endpoint. To the same you could just open: +``` +http://localhost:8080/invalidate?key=aeSu7aip&ext=menus +``` + +As value of the `ext` will be searched inside `Cached URL`. +If you would provide here `product` it would cache product's catalog. You should have it in mind. + +### Banning permissions +It will be allowed only from certain IPs. In my case I put here only VSF-API IP. But here we have `app` as Docker will resolve it as VSF-API IP: +``` +acl purge { + "app"; // IP which can BAN cache - it should be VSF-API's IP +} +``` + +### What to cache +We should provide to Varnish - IP & Port to cache, there we have it: +``` +backend default { + .host = "app"; + .port = "8080"; +} +``` + +### URL +Varnish by default using port `80` but by Docker's port mapping we are using `1234` + +### How to install on VPS +1. Install Varnish +2. Install Varnish Modules +3. By using Reverse Proxy output `/api` from Varnish, to the world + +I'll try to prepare more detailed tutorial (with commands) as I will probably do it again in the following month. \ No newline at end of file diff --git a/docker/varnish/config.vcl b/docker/varnish/config.vcl new file mode 100644 index 00000000..774b42ba --- /dev/null +++ b/docker/varnish/config.vcl @@ -0,0 +1,130 @@ + + +vcl 4.0; + +import std; +import bodyaccess; + +acl purge { + "app"; // IP which can BAN cache - it should be VSF-API's IP +} + + +backend default { + .host = "app"; + .port = "8080"; +} + +sub vcl_recv { + unset req.http.X-Body-Len; + # Only allow BAN requests from IP addresses in the 'purge' ACL. + if (req.method == "BAN") { + # Same ACL check as above: + if (!client.ip ~ purge) { + return (synth(403, "Not allowed.")); + } + + # Logic for the ban, using the X-Cache-Tags header. + if (req.http.X-VS-Cache-Tag) { + ban("obj.http.X-VS-Cache-Tag ~ " + req.http.X-VS-Cache-Tag); + } + if (req.http.X-VS-Cache-Ext) { + ban("req.url ~ " + req.http.X-VS-Cache-Ext); + } + if (!req.http.X-VS-Cache-Tag && !req.http.X-VS-Cache-Ext) { + return (synth(403, "X-VS-Cache-Tag or X-VS-Cache-Ext header missing.")); + } + + # Throw a synthetic page so the request won't go to the backend. + return (synth(200, "Ban added.")); + } + + if (req.url ~ "^\/api\/catalog\/") { + if (req.method == "POST") { + # It will allow me to cache by req body in the vcl_hash + std.cache_req_body(500KB); + set req.http.X-Body-Len = bodyaccess.len_req_body(); + } + + if ((req.method == "POST" || req.method == "GET")) { + return (hash); + } + } + + if (req.url ~ "^\/api\/ext\/") { + if (req.method == "GET") { + # Custom packs GET - M2 - /jimmylion/pack/${req.params.packId} + if (req.url ~ "^\/api\/ext\/custom-packs\/") { + return (hash); + } + + # Countries for storecode GET - M2 - /directory/countries + if (req.url ~ "^\/api\/ext\/directory\/") { + return (hash); + } + + # Menus GET - M2 - /menus & /nodes + if (req.url ~ "^\/api\/ext\/menus\/") { + return (hash); + } + } + } + + if (req.url ~ "^\/api\/stock\/") { + if (req.method == "GET") { + # M2 Stock + return (hash); + } + } + + return (pipe); + +} + +sub vcl_hash { + # To cache POST and PUT requests + if (req.http.X-Body-Len) { + bodyaccess.hash_req_body(); + } else { + hash_data(""); + } +} + +sub vcl_backend_fetch { + if (bereq.http.X-Body-Len) { + set bereq.method = "POST"; + } +} + +sub vcl_backend_response { + # Set ban-lurker friendly custom headers. + if (beresp.http.X-VS-Cache && beresp.http.X-VS-Cache ~ "Miss") { + set beresp.ttl = 0s; + } + if (bereq.url ~ "^\/api\/stock\/") { + set beresp.ttl = 900s; // 15 minutes + } + set beresp.http.X-Url = bereq.url; + set beresp.http.X-Host = bereq.http.host; +} + +sub vcl_deliver { + if (obj.hits > 0) { + set resp.http.X-Cache = "HIT_1"; + set resp.http.X-Cache-Hits = obj.hits; + } else { + set resp.http.X-Cache = "MISS_1"; + } + set resp.http.X-Cache-Expires = resp.http.Expires; + unset resp.http.X-Varnish; + unset resp.http.Via; + unset resp.http.Age; + unset resp.http.X-Purge-URL; + unset resp.http.X-Purge-Host; + # Remove ban-lurker friendly custom headers when delivering to client. + unset resp.http.X-Url; + unset resp.http.X-Host; + # Comment these for easier Drupal cache tag debugging in development. + unset resp.http.X-Cache-Tags; + unset resp.http.X-Cache-Contexts; +} \ No newline at end of file From 5774173e5993fd447927935b026e0816839f8b70 Mon Sep 17 00:00:00 2001 From: Fifciuu Date: Sun, 9 Feb 2020 15:47:29 +0100 Subject: [PATCH 2/8] Readme update --- docker/varnish/README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docker/varnish/README.md b/docker/varnish/README.md index e1d1fc2c..92b23269 100644 --- a/docker/varnish/README.md +++ b/docker/varnish/README.md @@ -4,7 +4,7 @@ E.g. when I used `docker network create some-net`, I have network with name `vuestorefrontapi_some-net` 3. Open docker-compose.yml: At the end: -``` +```yaml networks: vuestorefrontapi_some-net: external: true @@ -13,7 +13,7 @@ Set vuestorefrontapi_some-net to your network name 4. Check each `docker-compose` file and set proper network name. 5. In the docker-compose.nodejs.yml it should not have a prefix, e.g: -``` +```yaml networks: - some-net @@ -24,17 +24,17 @@ networks: ### How does it work? 1. I add output tags to the VSF-API response: -``` +```js const tagsHeader = output.tags.join(' ') res.setHeader('X-VS-Cache-Tag', tagsHeader) ``` 2. After it invalidates cache in the Redis. I forward request to the: -``` +```js http://${config.varnish.host}:${config.varnish.port}/ ``` With invalidate tag in headers: -``` +```js headers: { "X-VS-Cache-Tag": tag } @@ -44,7 +44,7 @@ I set Varnish invalidate method to `BAN` but you can change it in your config + 3. Configuration of BANning we have inside `docker/varnish/config.vcl` in `vcl_recv`. It tries to BAN resource which has `X-VS-Cache-Tag` header: -``` +```vcl # Logic for the ban, using the X-Cache-Tag header. if (req.http.X-VS-Cache-Tag) { ban("obj.http.X-VS-Cache-Tag ~ " + req.http.X-VS-Cache-Tag); @@ -52,7 +52,7 @@ if (req.http.X-VS-Cache-Tag) { ``` Below under BANning logic. I have to tell Varnish what to cache. -``` +```vcl if (req.url ~ "^\/api\/catalog\/") { if (req.method == "POST") { # It will allow me to cache by req body in the vcl_hash @@ -70,12 +70,12 @@ I am caching request that starts with `/api/catalog/`. As you can see I cache bo This is because in my project I use huge ES requests to compute Facted Filters. I would exceed HTTP GET limit. Thanks to this line and `bodyaccess`, I can distinguish requests to the same URL by their body! -``` +```vcl std.cache_req_body(500KB); ``` Then in `vcl_hash` I create hash for POST requests with `bodyaccess.hash_req_body()`: -``` +```vcl sub vcl_hash { # To cache POST and PUT requests if (req.http.X-Body-Len) { @@ -88,7 +88,7 @@ sub vcl_hash { By default, Varnish change each request to HTTP GET. We need to tell him to send POST requests to the VSF-API as POST - not GET. We will do it like that: -``` +```vcl sub vcl_backend_fetch { if (bereq.http.X-Body-Len) { set bereq.method = "POST"; @@ -101,7 +101,7 @@ sub vcl_backend_fetch { It might be a good idea to cache stock requests if you check it lifetime in VSF-PWA in visiblityChanged hook (product listing). In one project when I have slow Magento - it reduced Time-To-Response from ~2s to ~70ms. -``` +```vcl if (req.url ~ "^\/api\/stock\/") { if (req.method == "GET") { # M2 Stock @@ -111,7 +111,7 @@ if (req.url ~ "^\/api\/stock\/") { ``` Then in `vcl_backend_response` you should set safe TTL (Time to live) for your stock cache. I've set 15 minutes (900 seconds) -``` +```vcl sub vcl_backend_response { # Set ban-lurker friendly custom headers. if (beresp.http.X-VS-Cache && beresp.http.X-VS-Cache ~ "Miss") { @@ -135,7 +135,7 @@ to around ~50ms. How to do that? Inside `vcl_recv` add: -``` +```vcl // As in my case I want to cache only GET requests if (req.method == "GET") { # Countries for storecode GET - M2 - /directory/countries @@ -173,7 +173,7 @@ If you would provide here `product` it would cache product's catalog. You should ### Banning permissions It will be allowed only from certain IPs. In my case I put here only VSF-API IP. But here we have `app` as Docker will resolve it as VSF-API IP: -``` +```vcl acl purge { "app"; // IP which can BAN cache - it should be VSF-API's IP } @@ -181,7 +181,7 @@ acl purge { ### What to cache We should provide to Varnish - IP & Port to cache, there we have it: -``` +```vcl backend default { .host = "app"; .port = "8080"; From f916cc7f96f1f398d1a22fb9d9b1f2a1517ce788 Mon Sep 17 00:00:00 2001 From: Fifciuu Date: Sun, 9 Feb 2020 16:10:53 +0100 Subject: [PATCH 3/8] Readme fixes --- docker/varnish/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/varnish/README.md b/docker/varnish/README.md index 92b23269..b87adc58 100644 --- a/docker/varnish/README.md +++ b/docker/varnish/README.md @@ -67,7 +67,7 @@ if (req.url ~ "^\/api\/catalog\/") { ``` I am caching request that starts with `/api/catalog/`. As you can see I cache both POST and GET. -This is because in my project I use huge ES requests to compute Facted Filters. I would exceed HTTP GET limit. +This is because in my project I use huge ES requests to compute Faceted Filters. I would exceed HTTP GET limit. Thanks to this line and `bodyaccess`, I can distinguish requests to the same URL by their body! ```vcl @@ -98,7 +98,7 @@ sub vcl_backend_fetch { ### Caching Stock -It might be a good idea to cache stock requests if you check it lifetime in VSF-PWA in visiblityChanged hook (product listing). +It might be a good idea to cache stock requests if you check it often (filterUnavailableVariants, configurableChildrenStockPrefetchDynamic) in VSF-PWA in visiblityChanged hook (product listing). In one project when I have slow Magento - it reduced Time-To-Response from ~2s to ~70ms. ```vcl @@ -136,7 +136,7 @@ to around ~50ms. How to do that? Inside `vcl_recv` add: ```vcl -// As in my case I want to cache only GET requests +# As in my case I want to cache only GET requests if (req.method == "GET") { # Countries for storecode GET - M2 - /directory/countries if (req.url ~ "^\/api\/ext\/directory\/") { From 833158d114aefc8833c947e874733f9eb4f2b554 Mon Sep 17 00:00:00 2001 From: Fifciuu Date: Sat, 15 Feb 2020 15:28:26 +0100 Subject: [PATCH 4/8] Catalog tags header if varnish.enabled --- src/api/catalog.ts | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/api/catalog.ts b/src/api/catalog.ts index 355e835c..50a67b56 100755 --- a/src/api/catalog.ts +++ b/src/api/catalog.ts @@ -129,7 +129,15 @@ export default ({config, db}) => async function (req, res, body) { if (entityType === 'product') { resultProcessor.process(_resBody.hits.hits, groupId).then(async (result) => { _resBody.hits.hits = result - _cacheStorageHandler(config, _resBody, reqHash, tagsArray) + if (config.get('varnish.enabled')) { + // Add tags to cache, so we can display them in response headers then + _cacheStorageHandler(config, { + ..._resBody, + tags: tagsArray + }, reqHash, tagsArray) + } else { + _cacheStorageHandler(config, _resBody, reqHash, tagsArray) + } if (_resBody.aggregations && config.entities.attribute.loadByAttributeMetadata) { const attributeListParam = AttributeService.transformAggsToAttributeListParam(_resBody.aggregations) // find attribute list @@ -143,7 +151,15 @@ export default ({config, db}) => async function (req, res, body) { } else { resultProcessor.process(_resBody.hits.hits).then((result) => { _resBody.hits.hits = result - _cacheStorageHandler(config, _resBody, reqHash, tagsArray) + if (config.get('varnish.enabled')) { + // Add tags to cache, so we can display them in response headers then + _cacheStorageHandler(config, { + ..._resBody, + tags: tagsArray + }, reqHash, tagsArray) + } else { + _cacheStorageHandler(config, _resBody, reqHash, tagsArray) + } res.json(_outputFormatter(_resBody, responseFormat)); }).catch((err) => { console.error(err) @@ -161,6 +177,11 @@ export default ({config, db}) => async function (req, res, body) { ).then(output => { if (output !== null) { res.setHeader('X-VS-Cache', 'Hit') + if (config.get('varnish.enabled')) { + const tagsHeader = output.tags.join(' ') + res.setHeader('X-VS-Cache-Tags', tagsHeader) + delete output.tags + } res.json(output) console.log(`cache hit [${req.url}], cached request: ${Date.now() - s}ms`) } else { From d5a7c3ca001c0549f3606dcdbda596059af5ff10 Mon Sep 17 00:00:00 2001 From: Fifciuu Date: Sat, 15 Feb 2020 15:37:23 +0100 Subject: [PATCH 5/8] Invalidate on only if varnish.enabled --- src/api/invalidate.ts | 62 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/src/api/invalidate.ts b/src/api/invalidate.ts index 1d7bf5aa..1c856daf 100644 --- a/src/api/invalidate.ts +++ b/src/api/invalidate.ts @@ -5,12 +5,14 @@ import request from 'request' function invalidateCache (req, res) { if (config.get('server.useOutputCache')) { - if (req.query.tag && req.query.key) { // clear cache pages for specific query tag - if (req.query.key !== config.get('server.invalidateCacheKey')) { - console.error('Invalid cache invalidation key') - apiStatus(res, 'Invalid cache invalidation key', 500) - return - } + if (!req.query.key || req.query.key !== config.get('server.invalidateCacheKey')) { + console.error('Invalid cache invalidation key') + apiStatus(res, 'Invalid cache invalidation key', 500) + return + } + + if (req.query.tag) { // clear cache pages for specific query tag + console.log(`Clear cache request for [${req.query.tag}]`) let tags = [] if (req.query.tag === '*') { @@ -25,6 +27,27 @@ function invalidateCache (req, res) { })) { subPromises.push(cache.invalidate(tag).then(() => { console.log(`Tags invalidated successfully for [${tag}]`) + if (config.get('varnish.enabled')) { + request( + { + uri: `http://${config.get('varnish.host')}:${config.get('varnish.port')}/`, + method: "BAN", + headers: { + // I should change Tags -> tag + "X-VS-Cache-Tags": tag + } + }, + (err, res, body) => { + if (body && body.includes("200 Ban added")) { + console.log( + `Tags invalidated successfully for [${tag}] in the Varnish` + ); + } else { + console.error(`Couldn't ban tag: ${tag} in the Varnish`); + } + } + ); + } })) } else { console.error(`Invalid tag name ${tag}`) @@ -48,6 +71,33 @@ function invalidateCache (req, res) { }); } } + } else if (config.get('varnish.enabled') && req.query.ext) { + const exts = req.query.ext.split(',') + for (let ext of exts) { + request( + { + uri: `http://${config.get('varnish.host')}:${config.get('varnish.port')}/`, + method: "BAN", + headers: { + "X-VS-Cache-Ext": ext + } + }, + (err, res, body) => { + if (body && body.includes("200 Ban added")) { + console.log( + `Cache invalidated successfully for [${ext}] in the Varnish` + ); + } else { + console.error(`Couldn't ban extension: ${ext} in the Varnish`); + } + } + ); + } + apiStatus( + res, + "Cache invalidation succeed", + 200 + ); } else { apiStatus(res, 'Invalid parameters for Clear cache request', 500) console.error('Invalid parameters for Clear cache request') From f5fe6cf5f6b5fa29fcedec13461d346015617cbc Mon Sep 17 00:00:00 2001 From: Fifciuu Date: Sat, 15 Feb 2020 15:57:49 +0100 Subject: [PATCH 6/8] Typo fix --- src/api/catalog.ts | 2 +- src/api/invalidate.ts | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/api/catalog.ts b/src/api/catalog.ts index 50a67b56..d489c2d2 100755 --- a/src/api/catalog.ts +++ b/src/api/catalog.ts @@ -179,7 +179,7 @@ export default ({config, db}) => async function (req, res, body) { res.setHeader('X-VS-Cache', 'Hit') if (config.get('varnish.enabled')) { const tagsHeader = output.tags.join(' ') - res.setHeader('X-VS-Cache-Tags', tagsHeader) + res.setHeader('X-VS-Cache-Tag', tagsHeader) delete output.tags } res.json(output) diff --git a/src/api/invalidate.ts b/src/api/invalidate.ts index 1c856daf..f7223fa8 100644 --- a/src/api/invalidate.ts +++ b/src/api/invalidate.ts @@ -12,7 +12,6 @@ function invalidateCache (req, res) { } if (req.query.tag) { // clear cache pages for specific query tag - console.log(`Clear cache request for [${req.query.tag}]`) let tags = [] if (req.query.tag === '*') { @@ -31,18 +30,19 @@ function invalidateCache (req, res) { request( { uri: `http://${config.get('varnish.host')}:${config.get('varnish.port')}/`, - method: "BAN", + method: 'BAN', headers: { // I should change Tags -> tag - "X-VS-Cache-Tags": tag + 'X-VS-Cache-Tag': tag } }, (err, res, body) => { - if (body && body.includes("200 Ban added")) { + if (body && body.includes('200 Ban added')) { console.log( `Tags invalidated successfully for [${tag}] in the Varnish` ); } else { + console.log(body) console.error(`Couldn't ban tag: ${tag} in the Varnish`); } } @@ -77,13 +77,13 @@ function invalidateCache (req, res) { request( { uri: `http://${config.get('varnish.host')}:${config.get('varnish.port')}/`, - method: "BAN", + method: 'BAN', headers: { - "X-VS-Cache-Ext": ext + 'X-VS-Cache-Ext': ext } }, (err, res, body) => { - if (body && body.includes("200 Ban added")) { + if (body && body.includes('200 Ban added')) { console.log( `Cache invalidated successfully for [${ext}] in the Varnish` ); @@ -95,7 +95,7 @@ function invalidateCache (req, res) { } apiStatus( res, - "Cache invalidation succeed", + 'Cache invalidation succeed', 200 ); } else { From 3af5292bb915955ef26874d3be8b27a95ad7d47b Mon Sep 17 00:00:00 2001 From: Fifciuu Date: Sat, 15 Feb 2020 16:03:51 +0100 Subject: [PATCH 7/8] Update --- docker-compose.nodejs.yml | 7 +-- docker-compose.varnish.yml | 6 --- docker-compose.yml | 10 ---- docker/varnish/README.md | 1 + .../docker-compose.nodejs.yml | 34 ++++++++++++++ .../docker-compose.varnish.yml | 16 +++++++ .../docker-compose copy/docker-compose.yml | 46 +++++++++++++++++++ .../docker-compose/docker-compose.nodejs.yml | 34 ++++++++++++++ .../docker-compose/docker-compose.varnish.yml | 16 +++++++ .../varnish/docker-compose/docker-compose.yml | 46 +++++++++++++++++++ 10 files changed, 194 insertions(+), 22 deletions(-) create mode 100644 docker/varnish/docker-compose copy/docker-compose.nodejs.yml create mode 100644 docker/varnish/docker-compose copy/docker-compose.varnish.yml create mode 100644 docker/varnish/docker-compose copy/docker-compose.yml create mode 100644 docker/varnish/docker-compose/docker-compose.nodejs.yml create mode 100644 docker/varnish/docker-compose/docker-compose.varnish.yml create mode 100644 docker/varnish/docker-compose/docker-compose.yml diff --git a/docker-compose.nodejs.yml b/docker-compose.nodejs.yml index a658535b..4d046685 100644 --- a/docker-compose.nodejs.yml +++ b/docker-compose.nodejs.yml @@ -26,9 +26,4 @@ services: - /var/www/dist ports: - '8080:8080' - networks: - - some-net - -networks: - some-net: - driver: bridge \ No newline at end of file + \ No newline at end of file diff --git a/docker-compose.varnish.yml b/docker-compose.varnish.yml index 71c93665..f2a5de94 100644 --- a/docker-compose.varnish.yml +++ b/docker-compose.varnish.yml @@ -8,9 +8,3 @@ services: - ./docker/varnish/config.vcl:/usr/local/etc/varnish/default.vcl ports: - '1234:80' - networks: - - vuestorefrontapi_some-net - -networks: - vuestorefrontapi_some-net: - external: true \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b13fc05d..6b42d798 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,6 @@ services: - cluster.name=docker-cluster - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xmx512m -Xms512m" - networks: - - vuestorefrontapi_some-net kibana: build: docker/kibana/ @@ -28,19 +26,11 @@ services: - '5601:5601' depends_on: - es1 - networks: - - vuestorefrontapi_some-net redis: image: 'redis:4-alpine' ports: - '6379:6379' - networks: - - vuestorefrontapi_some-net volumes: esdat1: - -networks: - vuestorefrontapi_some-net: - external: true \ No newline at end of file diff --git a/docker/varnish/README.md b/docker/varnish/README.md index b87adc58..629095db 100644 --- a/docker/varnish/README.md +++ b/docker/varnish/README.md @@ -21,6 +21,7 @@ networks: some-net: driver: bridge ``` +You can find Docker Compose files with applied network settings inside docker/varnish/docker-compose ### How does it work? 1. I add output tags to the VSF-API response: diff --git a/docker/varnish/docker-compose copy/docker-compose.nodejs.yml b/docker/varnish/docker-compose copy/docker-compose.nodejs.yml new file mode 100644 index 00000000..a658535b --- /dev/null +++ b/docker/varnish/docker-compose copy/docker-compose.nodejs.yml @@ -0,0 +1,34 @@ +version: '3.0' +services: + app: + # image: divante/vue-storefront-api:latest + build: + context: . + dockerfile: docker/vue-storefront-api/Dockerfile + depends_on: + - es1 + - redis + env_file: docker/vue-storefront-api/default.env + environment: + VS_ENV: dev + volumes: + - './config:/var/www/config' + - './ecosystem.json:/var/www/ecosystem.json' + - './migrations:/var/www/migrations' + - './package.json:/var/www/package.json' + - './babel.config.js:/var/www/babel.config.js' + - './tsconfig.json:/var/www/tsconfig.json' + - './nodemon.json:/var/www/nodemon.json' + - './scripts:/var/www/scripts' + - './src:/var/www/src' + - './var:/var/www/var' + tmpfs: + - /var/www/dist + ports: + - '8080:8080' + networks: + - some-net + +networks: + some-net: + driver: bridge \ No newline at end of file diff --git a/docker/varnish/docker-compose copy/docker-compose.varnish.yml b/docker/varnish/docker-compose copy/docker-compose.varnish.yml new file mode 100644 index 00000000..71c93665 --- /dev/null +++ b/docker/varnish/docker-compose copy/docker-compose.varnish.yml @@ -0,0 +1,16 @@ +version: '3.0' +services: + varnish: + build: + context: . + dockerfile: varnish/Dockerfile + volumes: + - ./docker/varnish/config.vcl:/usr/local/etc/varnish/default.vcl + ports: + - '1234:80' + networks: + - vuestorefrontapi_some-net + +networks: + vuestorefrontapi_some-net: + external: true \ No newline at end of file diff --git a/docker/varnish/docker-compose copy/docker-compose.yml b/docker/varnish/docker-compose copy/docker-compose.yml new file mode 100644 index 00000000..b13fc05d --- /dev/null +++ b/docker/varnish/docker-compose copy/docker-compose.yml @@ -0,0 +1,46 @@ +version: '3.0' +services: + es1: + container_name: elasticsearch + build: docker/elasticsearch/ + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - ./docker/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + ports: + - '9200:9200' + - '9300:9300' + environment: + - discovery.type=single-node + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xmx512m -Xms512m" + networks: + - vuestorefrontapi_some-net + + kibana: + build: docker/kibana/ + volumes: + - ./docker/kibana/config/:/usr/share/kibana/config:ro + ports: + - '5601:5601' + depends_on: + - es1 + networks: + - vuestorefrontapi_some-net + + redis: + image: 'redis:4-alpine' + ports: + - '6379:6379' + networks: + - vuestorefrontapi_some-net + +volumes: + esdat1: + +networks: + vuestorefrontapi_some-net: + external: true \ No newline at end of file diff --git a/docker/varnish/docker-compose/docker-compose.nodejs.yml b/docker/varnish/docker-compose/docker-compose.nodejs.yml new file mode 100644 index 00000000..a658535b --- /dev/null +++ b/docker/varnish/docker-compose/docker-compose.nodejs.yml @@ -0,0 +1,34 @@ +version: '3.0' +services: + app: + # image: divante/vue-storefront-api:latest + build: + context: . + dockerfile: docker/vue-storefront-api/Dockerfile + depends_on: + - es1 + - redis + env_file: docker/vue-storefront-api/default.env + environment: + VS_ENV: dev + volumes: + - './config:/var/www/config' + - './ecosystem.json:/var/www/ecosystem.json' + - './migrations:/var/www/migrations' + - './package.json:/var/www/package.json' + - './babel.config.js:/var/www/babel.config.js' + - './tsconfig.json:/var/www/tsconfig.json' + - './nodemon.json:/var/www/nodemon.json' + - './scripts:/var/www/scripts' + - './src:/var/www/src' + - './var:/var/www/var' + tmpfs: + - /var/www/dist + ports: + - '8080:8080' + networks: + - some-net + +networks: + some-net: + driver: bridge \ No newline at end of file diff --git a/docker/varnish/docker-compose/docker-compose.varnish.yml b/docker/varnish/docker-compose/docker-compose.varnish.yml new file mode 100644 index 00000000..71c93665 --- /dev/null +++ b/docker/varnish/docker-compose/docker-compose.varnish.yml @@ -0,0 +1,16 @@ +version: '3.0' +services: + varnish: + build: + context: . + dockerfile: varnish/Dockerfile + volumes: + - ./docker/varnish/config.vcl:/usr/local/etc/varnish/default.vcl + ports: + - '1234:80' + networks: + - vuestorefrontapi_some-net + +networks: + vuestorefrontapi_some-net: + external: true \ No newline at end of file diff --git a/docker/varnish/docker-compose/docker-compose.yml b/docker/varnish/docker-compose/docker-compose.yml new file mode 100644 index 00000000..b13fc05d --- /dev/null +++ b/docker/varnish/docker-compose/docker-compose.yml @@ -0,0 +1,46 @@ +version: '3.0' +services: + es1: + container_name: elasticsearch + build: docker/elasticsearch/ + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - ./docker/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + ports: + - '9200:9200' + - '9300:9300' + environment: + - discovery.type=single-node + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xmx512m -Xms512m" + networks: + - vuestorefrontapi_some-net + + kibana: + build: docker/kibana/ + volumes: + - ./docker/kibana/config/:/usr/share/kibana/config:ro + ports: + - '5601:5601' + depends_on: + - es1 + networks: + - vuestorefrontapi_some-net + + redis: + image: 'redis:4-alpine' + ports: + - '6379:6379' + networks: + - vuestorefrontapi_some-net + +volumes: + esdat1: + +networks: + vuestorefrontapi_some-net: + external: true \ No newline at end of file From e8870f999599a778a37c57307081837db944778c Mon Sep 17 00:00:00 2001 From: Fifciuu Date: Mon, 17 Feb 2020 10:15:09 +0100 Subject: [PATCH 8/8] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 328c9283..483d6f74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Create attribute service that allows to fetch attributes with specific options - used for products aggregates - @gibkigonzo (https://github.com/DivanteLtd/vue-storefront/pull/4001, https://github.com/DivanteLtd/mage2vuestorefront/pull/99) - Add ElasticSearch client support for HTTP authentication - @cewald (#397) - Endpoint for reset password with reset token. Only for Magento 2 - @Fifciu +- Varnish Cache with autoinvalidation by Cache tags as addon - @Fifciu ### Fixed - add es7 support for map url module and fixed default index for es config - @gibkigonzo