From 08d2e946f8d1299bffbf90e711d900c180ac7d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20R?= Date: Thu, 6 Jul 2017 10:24:49 +0200 Subject: [PATCH] EZP-27542: Add support for using Varnish with Docker (#188) * [Docker] Add docker support for using varnish * [Travis] Enable testing with Varnish * [Doc] Remove here invalid leftover from other branch [skip ci] * Add curl to varnsih image * Change to debian:jessie * [Travis] Run platformui behat profile with redis Both since it randomly crashes with Stash error when directories are attempted to be deleted between tests, and to get coverage with redis. * [.env] Simplify env variables for web and selenium host * [VCL] Change to use include for paramters to avoid duplicating vcl --- .env | 5 +++ .travis.yml | 6 +-- app/config/default_parameters.yml | 4 ++ app/config/env/docker.php | 6 +++ app/config/ezplatform.yml | 7 ++++ doc/docker-compose/Dockerfile-varnish | 19 +++++++++ doc/docker-compose/README.md | 1 + .../entrypoint/varnish/parameters.vcl | 19 +++++++++ doc/docker-compose/redis.yml | 4 ++ doc/docker-compose/selenium.yml | 8 ++-- doc/docker-compose/solr.yml | 4 ++ doc/docker-compose/varnish.yml | 40 +++++++++++++++++++ doc/varnish/vcl/parameters.vcl | 21 ++++++++++ doc/varnish/vcl/varnish3.vcl | 20 +--------- doc/varnish/vcl/varnish4.vcl | 20 +--------- 15 files changed, 140 insertions(+), 44 deletions(-) create mode 100644 doc/docker-compose/Dockerfile-varnish create mode 100644 doc/docker-compose/entrypoint/varnish/parameters.vcl create mode 100644 doc/docker-compose/varnish.yml create mode 100644 doc/varnish/vcl/parameters.vcl diff --git a/.env b/.env index 20198edd2..32848aef7 100644 --- a/.env +++ b/.env @@ -23,3 +23,8 @@ APP_DOCKER_FILE=Dockerfile # Install config, used by .platform.app.yaml among others INSTALL_EZ_INSTALL_TYPE=clean + +# Behat / Selenium config +## web host refer to the tip of the setup, so varnish if that is used. +WEB_HOST=web +SELENIUM_HOST=selenium diff --git a/.travis.yml b/.travis.yml index 4eb5ba390..62ef13a52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,13 +16,11 @@ env: - SYMFONY_DEBUG=1 # list of behat arguments to test matrix: - # Disabled until failures on BDD tests with redis is solved / figured out - #- TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" COMPOSE_FILE="doc/docker-compose/base-prod.yml:doc/docker-compose/redis.yml:doc/docker-compose/selenium.yml" - - TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" COMPOSE_FILE="doc/docker-compose/base-prod.yml:doc/docker-compose/selenium.yml" + - TEST_CMD="bin/behat -vv --profile=rest --suite=fullJson --tags=~@broken" COMPOSE_FILE="doc/docker-compose/base-prod.yml:doc/docker-compose/varnish.yml:doc/docker-compose/selenium.yml" WEB_HOST="varnish" - TEST_CMD="bin/behat -vv --profile=rest --suite=fullXml --tags=~@broken" - TEST_CMD="bin/behat -vv --profile=core --tags=~@broken" - TEST_CMD="bin/phpunit -v vendor/ezsystems/ezpublish-kernel/eZ/Bundle/EzPublishRestBundle/Tests/Functional" - - TEST_CMD="bin/behat -vv --profile=platformui --tags='@common'" + - TEST_CMD="bin/behat -vv --profile=platformui --tags='@common'" COMPOSE_FILE="doc/docker-compose/base-dev.yml:doc/docker-compose/redis.yml:doc/docker-compose/selenium.yml" # test only master (+ Pull requests) branches: diff --git a/app/config/default_parameters.yml b/app/config/default_parameters.yml index 671040161..85bfeb1d9 100644 --- a/app/config/default_parameters.yml +++ b/app/config/default_parameters.yml @@ -28,3 +28,7 @@ parameters: cache_memcached_port: 11211 cache_redis_port: 6379 + + # Settings for HttpCache + purge_type: "local" + purge_server: "http://my.varnish.server:80" diff --git a/app/config/env/docker.php b/app/config/env/docker.php index 79ca6d6cb..6eb6833f8 100644 --- a/app/config/env/docker.php +++ b/app/config/env/docker.php @@ -94,3 +94,9 @@ $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__ . '/../cache_pool')); $loader->load($pool . '.yml'); } + +// HttpCache setting (for configuring Varnish purging) +if ($purgeServer = getenv('HTTPCACHE_PURGE_SERVER')) { + $container->setParameter('purge_type', 'http'); + $container->setParameter('purge_server', $purgeServer); +} diff --git a/app/config/ezplatform.yml b/app/config/ezplatform.yml index edc4d2c47..42ebffd37 100644 --- a/app/config/ezplatform.yml +++ b/app/config/ezplatform.yml @@ -1,4 +1,8 @@ ezpublish: + # HttpCache settings, By default 'local' (Symfony HttpCache Proxy), by setting it to 'http' you can point it to Varnish + http_cache: + purge_type: %purge_type% + # Repositories configuration, setup default repository to support solr if enabled repositories: default: @@ -29,3 +33,6 @@ ezpublish: # so removing eng-GB from this list may lead to errors or content not being shown, unless you change # all eng-GB data to other locales first. languages: [eng-GB] + # HttpCache purge server(s) setting, eg Varnish, for when ezpublish.http_cache.purge_type is set to 'http'. + http_cache: + purge_servers: ["%purge_server%"] diff --git a/doc/docker-compose/Dockerfile-varnish b/doc/docker-compose/Dockerfile-varnish new file mode 100644 index 000000000..77499dd20 --- /dev/null +++ b/doc/docker-compose/Dockerfile-varnish @@ -0,0 +1,19 @@ +FROM debian:jessie + +ENV DEBIAN_FRONTEND noninteractive + +# Use packages from Varnish to get Varnsih 5.1, currently does not work on debian:stretch +RUN apt-get update -q -y && \ + apt-get install -q -y --force-yes --no-install-recommends ca-certificates curl && \ + curl -s https://packagecloud.io/install/repositories/varnishcache/varnish5/script.deb.sh | bash && \ + apt-get install -q -y --force-yes --no-install-recommends varnish + +# This will need debian:stretch, just here for referance, had segmentation faults so rather using varnsih 5.1 above +#RUN \ +# apt-get update -q -y && \ +# apt-get install -q -y --force-yes --no-install-recommends ca-certificates varnish-modules varnish + + +EXPOSE 80 6082 + +CMD ["varnishd", "-F", "-a", ":80", "-T", ":6082", "-f", "/etc/varnish/default.vcl", "-s", "malloc,256M"] diff --git a/doc/docker-compose/README.md b/doc/docker-compose/README.md index bfc31c0bd..477148b5a 100644 --- a/doc/docker-compose/README.md +++ b/doc/docker-compose/README.md @@ -32,6 +32,7 @@ The current Docker Compose files are made to be mixed and matched togtehr as you In addition the following exists but are work in progress, thus not tested yet and are known to be somewhat broken: - redis.yml _(optional, adds redis service and appends config to app)_ +- varnish.yml _(optional, adds varnish service and appends config to app)_ - solr.yml _(optional, add solr service and configure app for it)_ diff --git a/doc/docker-compose/entrypoint/varnish/parameters.vcl b/doc/docker-compose/entrypoint/varnish/parameters.vcl new file mode 100644 index 000000000..14f4111c8 --- /dev/null +++ b/doc/docker-compose/entrypoint/varnish/parameters.vcl @@ -0,0 +1,19 @@ +// Custom parameters.vcl for docker use + +backend ezplatform { + .host = "web"; + .port = "80"; +} + +// ACL for invalidators IP +acl invalidators { + "127.0.0.1"; + "172.16.0.0"/20; + "app"; +} + +// ACL for debuggers IP +acl debuggers { + "127.0.0.1"; + "172.16.0.0"/20; +} diff --git a/doc/docker-compose/redis.yml b/doc/docker-compose/redis.yml index a069d6179..c7276a54d 100644 --- a/doc/docker-compose/redis.yml +++ b/doc/docker-compose/redis.yml @@ -1,6 +1,10 @@ version: '2.1' # Redis config, to be appended after base-prod or base-dev, ..., but before selenium.yml +## WARNING: +# This service is currently work in progress, is not tested by CI, and thus not guaranteed to work. +# You are however more then welcome to try it out and help make it stable. + services: app: depends_on: diff --git a/doc/docker-compose/selenium.yml b/doc/docker-compose/selenium.yml index cd11971db..db74dc05b 100644 --- a/doc/docker-compose/selenium.yml +++ b/doc/docker-compose/selenium.yml @@ -1,5 +1,5 @@ version: '2.1' -# Appends services to prod.yml, prod+dev, prod+redis, ..., always latests +# Appends services to prod.yml, prod+dev, prod+redis, ..., always latest services: selenium: @@ -16,6 +16,6 @@ services: depends_on: - selenium environment: - - EZP_TEST_REST_HOST=web - - BEHAT_SELENIUM_HOST=selenium - - BEHAT_WEB_HOST=web + - EZP_TEST_REST_HOST=$WEB_HOST + - BEHAT_SELENIUM_HOST=$SELENIUM_HOST + - BEHAT_WEB_HOST=$WEB_HOST diff --git a/doc/docker-compose/solr.yml b/doc/docker-compose/solr.yml index 4c45f3710..b4669e232 100644 --- a/doc/docker-compose/solr.yml +++ b/doc/docker-compose/solr.yml @@ -4,6 +4,10 @@ version: '2.1' # NOTE: You'll need to manually reindex the solr index when booting this as we don't have entrypoint for solr yet. # (Unless you use ezplatform:install command which indexes for you) +## WARNING! +# This service is currently work in progress, is not tested by CI, and thus not guaranteed to work. +# You are however more then welcome to try it out and help make it stable. + services: app: depends_on: diff --git a/doc/docker-compose/varnish.yml b/doc/docker-compose/varnish.yml new file mode 100644 index 000000000..c2c7a364b --- /dev/null +++ b/doc/docker-compose/varnish.yml @@ -0,0 +1,40 @@ +version: '2.1' +# Simple single server setup for prod + +## WARNING! +# This service is currently work in progress, is not tested by CI, and thus not guaranteed to work. +# You are however more then welcome to try it out and help make it stable. + +services: + app: + environment: + - SYMFONY_HTTP_CACHE=0 + - SYMFONY_TRUSTED_PROXIES=varnish + - HTTPCACHE_PURGE_SERVER=http://varnish + + varnish: + build: + context: . + dockerfile: Dockerfile-varnish + image: my_varnish + mem_limit: 384M + memswap_limit: 512M + ports: + - "8081:80" + depends_on: + - web + volumes: + - ${COMPOSE_DIR}/../varnish/vcl/varnish4.vcl:/etc/varnish/default.vcl:ro + - ${COMPOSE_DIR}/entrypoint/varnish/parameters.vcl:/etc/varnish/parameters.vcl:ro + + +## DEBUG?? +# In need of debugging all request going to Varnish, use varnishlog, example: +# docker-compose exec varnish varnishlog -c -i ReqURL,ReqMethod -I ReqHeader:X-Location-Id +# Or more relevant only BAN's with all info: +# docker-compose exec varnish varnishlog -g request -q "ReqMethod eq 'BAN'" +# +# But before doing that check that http and not local purge client is set: +# docker-compose exec app app/console --env=dev debug:container ezpublish.http_cache.purge_client +# +# And if in prod make sure you have rebuilt app container on code changes ;) diff --git a/doc/varnish/vcl/parameters.vcl b/doc/varnish/vcl/parameters.vcl new file mode 100644 index 000000000..b988fee6c --- /dev/null +++ b/doc/varnish/vcl/parameters.vcl @@ -0,0 +1,21 @@ +// Our Backend - Assuming that web server is listening on port 80 +// Replace the host to fit your setup +// +// For additional example see: doc/docker-compose/entrypoint/varnish/parameters.vcl + +backend ezplatform { + .host = "127.0.0.1"; + .port = "80"; +} + +// ACL for invalidators IP +acl invalidators { + "127.0.0.1"; + "192.168.0.0"/16; +} + +// ACL for debuggers IP +acl debuggers { + "127.0.0.1"; + "192.168.0.0"/16; +} diff --git a/doc/varnish/vcl/varnish3.vcl b/doc/varnish/vcl/varnish3.vcl index 718c26ea7..fa1485999 100644 --- a/doc/varnish/vcl/varnish3.vcl +++ b/doc/varnish/vcl/varnish3.vcl @@ -1,24 +1,8 @@ // Varnish 3 style - eZ 5.4+ / 2014.09+ // Complete VCL example -// Our Backend - Assuming that web server is listening on port 80 -// Replace the host to fit your setup -backend ezplatform { - .host = "127.0.0.1"; - .port = "80"; -} - -// ACL for invalidators IP -acl invalidators { - "127.0.0.1"; - "192.168.0.0"/16; -} - -// ACL for debuggers IP -acl debuggers { - "127.0.0.1"; - "192.168.0.0"/16; -} +// For customizing your backend and acl rules see parameters.yml +include "parameters.vcl"; // Called at the beginning of a request, after the complete request has been received sub vcl_recv { diff --git a/doc/varnish/vcl/varnish4.vcl b/doc/varnish/vcl/varnish4.vcl index 8d743b656..7bbf42163 100644 --- a/doc/varnish/vcl/varnish4.vcl +++ b/doc/varnish/vcl/varnish4.vcl @@ -3,24 +3,8 @@ vcl 4.0; -// Our Backend - Assuming that web server is listening on port 80 -// Replace the host to fit your setup -backend ezplatform { - .host = "127.0.0.1"; - .port = "80"; -} - -// ACL for invalidators IP -acl invalidators { - "127.0.0.1"; - "192.168.0.0"/16; -} - -// ACL for debuggers IP -acl debuggers { - "127.0.0.1"; - "192.168.0.0"/16; -} +// For customizing your backend and acl rules see parameters.yml +include "parameters.vcl"; // Called at the beginning of a request, after the complete request has been received sub vcl_recv {