EZP-28043: As a Developer I want a HttpCache system by default that is cable of reliably clear affected caches on changes

app/AppCache.php

@@ -6,14 +6,14 @@
  * @license For full copyright and license information view LICENSE file distributed with this source code.
  * @version //autogentag//
  */
-use eZ\Bundle\EzPublishCoreBundle\HttpCache;
+use EzSystems\PlatformHttpCacheBundle\AppCache as PlatformHttpCacheBundleAppCache;
 
 /**
  * Class AppCache.
  *
  * For easier upgrade do not change this file, as of 2015.01 possible to extend
  * cleanly via SYMFONY_HTTP_CACHE_CLASS & SYMFONY_CLASSLOADER_FILE env variables!
  */
-class AppCache extends HttpCache
+class AppCache extends PlatformHttpCacheBundleAppCache
 {
 }

app/AppKernel.php

@@ -33,6 +33,7 @@ public function registerBundles()
             new Hautelook\TemplatedUriBundle\HautelookTemplatedUriBundle(),
             new Liip\ImagineBundle\LiipImagineBundle(),
             new FOS\HttpCacheBundle\FOSHttpCacheBundle(),
+            new EzSystems\PlatformHttpCacheBundle\EzSystemsPlatformHttpCacheBundle(),
             new eZ\Bundle\EzPublishCoreBundle\EzPublishCoreBundle(),
             new eZ\Bundle\EzPublishLegacySearchEngineBundle\EzPublishLegacySearchEngineBundle(),
             new eZ\Bundle\EzPublishIOBundle\EzPublishIOBundle(),

app/config/config.yml

@@ -109,3 +109,26 @@ swiftmailer:
     username: '%mailer_user%'
     password:  '%mailer_password%'
     spool: { type: memory }
+
+# FOSHttpCache Configuration
+fos_http_cache:
+    cache_control:
+        rules:
+            # Make sure already cacheable (fresh) responses from eZ Platform which are errors/redirect gets lower ttl (then default_ttl)
+            -
+                match:
+                    match_response: "response.isFresh() && ( response.isServerError() || response.isClientError() || response.isRedirect() )"
+                headers:
+                    overwrite: true
+                    cache_control:
+                        max_age: 10
+            # Example of performance tuning, force TTL on 404 pages to avoid crawlers / ... taking to much load
+            # Should not be set to high, as cached 404's can cause issues for future routes, url aliases, wildcards, ..
+            -
+                match:
+                    match_response: "!response.isFresh() && response.isNotFound()"
+                headers:
+                    overwrite: true
+                    cache_control:
+                        public: true
+                        max_age: 30

app/config/default_parameters.yml

@@ -33,3 +33,7 @@ parameters:
     # Settings for HttpCache
     purge_type: "local"
     purge_server: "http://my.varnish.server:80"
+
+    ## By default cache ttl is set to 24h, when using Varnish you can set a much higher value. High values depends on
+    ## using EzSystemsPlatformHttpCacheBundle (default as of v1.12) which by design expires affected cache on changes
+    httpcache_default_ttl: 86400

app/config/env/docker.php

@@ -100,3 +100,7 @@
     $container->setParameter('purge_type', 'http');
     $container->setParameter('purge_server', $purgeServer);
 }
+
+if ($value = getenv('HTTPCACHE_DEFAULT_TTL')) {
+    $container->setParameter('httpcache_default_ttl', $value);
+}

app/config/ezplatform.yml

@@ -33,6 +33,10 @@ ezpublish:
             # so removing eng-GB from this list may lead to errors or content not being shown, unless you change
             # all eng-GB data to other locales first.
             languages: [eng-GB]
+            content:
+              # As we by default enable EzSystemsPlatformHttpCacheBundle which is designed to expire all affected cache
+              # on changes, and as error / redirects now have separate ttl, we easier allow ttl to be greatly increased
+              default_ttl: '%httpcache_default_ttl%'
             # HttpCache purge server(s) setting, eg Varnish, for when ezpublish.http_cache.purge_type is set to 'http'.
             http_cache:
                 purge_servers: ['%purge_server%']

doc/docker/Dockerfile-varnish

@@ -45,7 +45,7 @@ RUN set -xe \
         && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false $buildDeps \
         && rm -rf /var/lib/apt/lists/*
 
-COPY doc/varnish/vcl/varnish4.vcl /etc/varnish/default.vcl
+COPY doc/varnish/vcl/varnish4_xkey.vcl /etc/varnish/default.vcl
 COPY doc/docker/entrypoint/varnish/parameters.vcl /etc/varnish/parameters.vcl
 COPY doc/docker/entrypoint/varnish/entrypoint.sh /entrypoint.sh
 

doc/varnish/varnish.md

@@ -7,10 +7,12 @@ Prerequisites
 
 Recommended VCL base files
 --------------------------
-For Varnish to work properly with eZ, you'll need to use one of the provided files as a basis:
+For Varnish to work properly with eZ, you'll need to use the following VCL as starting point:
+
+* [eZ Platform 1.7+ with Varnish 4.x/5.x with xkey VMOD](vcl/varnish4_xkey.vcl)
+* [eZ 5.4+ / 2014.09+ with Varnish 4/5 using builtin BAN](vcl/varnish4_ban.vcl) _(Deprecated in favour of xkey version)_
+* [eZ 5.4+ / 2014.09+ with Varnish 3 using builtin BAN](vcl/varnish3_ban.vcl) _(Deprecated in favour of xkey version)_
 
-* [eZ 5.4+ / 2014.09+ with Varnish 3](vcl/varnish3.vcl)
-* [eZ 5.4+ / 2014.09+ with Varnish 4](vcl/varnish4.vcl)
 
 > **Note:** Http cache management is done with the help of [FOSHttpCacheBundle](http://foshttpcachebundle.readthedocs.org/).
   One may need to tweak their VCL further on according to [FOSHttpCache documentation](http://foshttpcache.readthedocs.org/en/latest/varnish-configuration.html)

doc/varnish/vcl/varnish3.vcl → doc/varnish/vcl/varnish3_ban.vcl

@@ -1,5 +1,6 @@
 // Varnish 3 style - eZ 5.4+ / 2014.09+
 // Complete VCL example
+// DEPRECATED: For faster, and over time more stable setup look to ezplatform-http-cache and Varnish xkey use.
 
 // For customizing your backend and acl rules see parameters.yml
 include "parameters.vcl";

doc/varnish/vcl/varnish4.vcl → doc/varnish/vcl/varnish4_ban.vcl

@@ -1,5 +1,6 @@
 // Varnish 4 style - eZ 5.4+ / 2014.09+
 // Complete VCL example
+// DEPRECATED: For faster, and over time more stable setup look to ezplatform-http-cache and Varnish xkey use.
 
 vcl 4.0;
 

doc/varnish/vcl/varnish4_xkey.vcl

@@ -0,0 +1,227 @@
+// Varnish VCL for:
+// - Varnish 4.1 or higher with xkey vmod (via varnish-modules package, or via Varnish Plus)
+// - eZ Platform 1.8 or higher with ezplatform-http-cache bundle
+//
+// Complete VCL example, further reading on:
+// - https://symfony.com/doc/current/http_cache/varnish.html
+// - https://foshttpcache.readthedocs.io/en/stable/varnish-configuration.html
+// - https://github.com/varnish/varnish-modules/blob/master/docs/vmod_xkey.rst
+// - https://www.varnish-cache.org/docs/trunk/users-guide/vcl.html
+//
+// Make sure to at least adjust default parameters.yml, defaults there reflect our testing needs with docker.
+
+vcl 4.0;
+import std;
+import xkey;
+
+// For customizing your backend and acl rules see parameters.yml
+include "parameters.vcl";
+
+// Called at the beginning of a request, after the complete request has been received
+sub vcl_recv {
+
+    // Set the backend
+    set req.backend_hint = ezplatform;
+
+    // Advertise Symfony for ESI support
+    set req.http.Surrogate-Capability = "abc=ESI/1.0";
+
+    // Varnish, in its default configuration, sends the X-Forwarded-For header but does not filter out Forwarded header
+    unset req.http.Forwarded;
+
+    // Trigger cache purge if needed
+    call ez_purge;
+
+    // Don't cache requests other than GET and HEAD.
+    if (req.method != "GET" && req.method != "HEAD") {
+        return (pass);
+    }
+
+    // Don't cache Authenticate & Authorization
+    // You may remove this when using REST API with basic auth.
+    if (req.http.Authenticate || req.http.Authorization) {
+        if (client.ip ~ debuggers) {
+            set req.http.X-Debug = "Not Cached according to configuration (Authorization)";
+        }
+        return (hash);
+    }
+
+    // Remove all cookies besides Session ID, as JS tracker cookies and so will make the responses effectively un-cached
+    if (req.http.cookie) {
+        set req.http.cookie = ";" + req.http.cookie;
+        set req.http.cookie = regsuball(req.http.cookie, "; +", ";");
+        set req.http.cookie = regsuball(req.http.cookie, ";(eZSESSID[^=]*)=", "; \1=");
+        set req.http.cookie = regsuball(req.http.cookie, ";[^ ][^;]*", "");
+        set req.http.cookie = regsuball(req.http.cookie, "^[; ]+|[; ]+$", "");
+
+        if (req.http.cookie == "") {
+            // If there are no more cookies, remove the header to get page cached.
+            unset req.http.cookie;
+        }
+    }
+
+    // Do a standard lookup on assets (these don't vary by user context hash)
+    // Note that file extension list below is not extensive, so consider completing it to fit your needs.
+    if (req.url ~ "\.(css|js|gif|jpe?g|bmp|png|tiff?|ico|img|tga|wmf|svg|swf|ico|mp3|mp4|m4a|ogg|mov|avi|wmv|zip|gz|pdf|ttf|eot|wof)$") {
+        return (hash);
+    }
+
+    // Retrieve client user context hash and add it to the forwarded request.
+    call ez_user_context_hash;
+
+    // If it passes all these tests, do a lookup anyway.
+    return (hash);
+}
+
+// Called when a cache lookup is successful. The object being hit may be stale: It can have a zero or negative ttl with only grace or keep time left.
+sub vcl_hit {
+   if (obj.ttl >= 0s) {
+       // A pure unadultered hit, deliver it
+       return (deliver);
+   }
+
+   if (obj.ttl + obj.grace > 0s) {
+       // Object is in grace, logic below in this block is what differs from default:
+       // https://varnish-cache.org/docs/5.0/users-guide/vcl-grace.html#grace-mode
+       if (!std.healthy(req.backend_hint)) {
+           // Service is unhealthy, deliver from cache
+           return (deliver);
+       } else if (req.url ~ "^/api/ezp/v2" && req.http.referer ~ "/ez$") {
+           // Request is for Platform UI for REST API, fetch it as 1.x UI does not handle stale data to well
+           return (miss);
+       }
+
+       // By default deliver cache, automatically triggers a background fetch
+       return (deliver);
+   }
+
+   // fetch & deliver once we get the result
+   return (miss);
+}
+
+// Called when the requested object has been retrieved from the backend
+sub vcl_backend_response {
+
+    if (bereq.http.accept ~ "application/vnd.fos.user-context-hash"
+        && beresp.status >= 500
+    ) {
+        return (abandon);
+    }
+
+    // Check for ESI acknowledgement and remove Surrogate-Control header
+    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
+        unset beresp.http.Surrogate-Control;
+        set beresp.do_esi = true;
+    }
+
+    // Make Varnish keep all objects for up to 1 hour beyond their TTL, see vcl_hit for Request logic on this
+    set beresp.grace = 1h;
+}
+
+// Handle purge
+// You may add FOSHttpCacheBundle tagging rules
+// See http://foshttpcache.readthedocs.org/en/latest/varnish-configuration.html#id4
+sub ez_purge {
+    if (req.method == "PURGE") {
+        if (!client.ip ~ invalidators) {
+            return (synth(405, "Method not allowed"));
+        }
+
+        # If http header "key" is set, we assume purge is on key and you have Varnish xkey installed
+        if (req.http.key) {
+            # By default we recommend using soft purge to respect grace time, if you need to hard purge use:
+            # set req.http.n-gone = xkey.purge(req.http.key);
+            set req.http.n-gone = xkey.softpurge(req.http.key);
+
+            return (synth(200, "Invalidated "+req.http.n-gone+" objects"));
+        }
+
+        # if not, then this is a normal purge by url
+        return (purge);
+    }
+}
+
+// Sub-routine to get client user context hash, used to for being able to vary page cache on user rights.
+sub ez_user_context_hash {
+
+    // Prevent tampering attacks on the hash mechanism
+    if (req.restarts == 0
+        && (req.http.accept ~ "application/vnd.fos.user-context-hash"
+            || req.http.x-user-hash
+        )
+    ) {
+        return (synth(400));
+    }
+
+    if (req.restarts == 0 && (req.method == "GET" || req.method == "HEAD")) {
+        // Backup accept header, if set
+        if (req.http.accept) {
+            set req.http.x-fos-original-accept = req.http.accept;
+        }
+        set req.http.accept = "application/vnd.fos.user-context-hash";
+
+        // Backup original URL
+        set req.http.x-fos-original-url = req.url;
+        set req.url = "/_fos_user_context_hash";
+
+        // Force the lookup, the backend must tell not to cache or vary on all
+        // headers that are used to build the hash.
+        return (hash);
+    }
+
+    // Rebuild the original request which now has the hash.
+    if (req.restarts > 0
+        && req.http.accept == "application/vnd.fos.user-context-hash"
+    ) {
+        set req.url = req.http.x-fos-original-url;
+        unset req.http.x-fos-original-url;
+        if (req.http.x-fos-original-accept) {
+            set req.http.accept = req.http.x-fos-original-accept;
+            unset req.http.x-fos-original-accept;
+        } else {
+            // If accept header was not set in original request, remove the header here.
+            unset req.http.accept;
+        }
+
+        // Force the lookup, the backend must tell not to cache or vary on the
+        // user context hash to properly separate cached data.
+
+        return (hash);
+    }
+}
+
+sub vcl_deliver {
+    // On receiving the hash response, copy the hash header to the original
+    // request and restart.
+    if (req.restarts == 0
+        && resp.http.content-type ~ "application/vnd.fos.user-context-hash"
+    ) {
+        set req.http.x-user-hash = resp.http.x-user-hash;
+
+        return (restart);
+    }
+
+    // If we get here, this is a real response that gets sent to the client.
+
+    // Remove the vary on user context hash, this is nothing public. Keep all
+    // other vary headers.
+    set resp.http.Vary = regsub(resp.http.Vary, "(?i),? *X-User-Hash *", "");
+    set resp.http.Vary = regsub(resp.http.Vary, "^, *", "");
+    if (resp.http.Vary == "") {
+        unset resp.http.Vary;
+    }
+
+    // Sanity check to prevent ever exposing the hash to a client.
+    unset resp.http.x-user-hash;
+
+    if (client.ip ~ debuggers) {
+        if (resp.http.X-Varnish ~ " ") {
+            set resp.http.X-Cache = "HIT";
+        } else {
+            set resp.http.X-Cache = "MISS";
+        }
+    } else {
+        // Remove tag headers when delivering to non debug client
+        unset resp.http.xkey;
+    }
+}