working on issue magento#10087. Added a normalise parameters option t…

…o the varnish admin config. This is then exported to the varnish 4 and varnish 5 vcl files. I've also included a config parameter for default google click id, and google campaign urls.

app/code/Magento/PageCache/Model/Config.php

@@ -43,6 +43,8 @@ class Config
 
     const XML_VARNISH_PAGECACHE_DESIGN_THEME_REGEX = 'design/theme/ua_regexp';
 
+    const XML_VARNISH_PAGECACHE_NORMALIZE_PARAMS = 'system/full_page_cache/varnish/normalize_params';
+
     /**
      * @var \Magento\Framework\App\Config\ScopeConfigInterface
      */
@@ -183,7 +185,8 @@ protected function _getReplacements()
                 '-',
                 $this->_scopeConfig->getValue(\Magento\Framework\HTTP\PhpEnvironment\Request::XML_PATH_OFFLOADER_HEADER)
             ),
-            '/* {{ grace_period }} */' => $this->_scopeConfig->getValue(self::XML_VARNISH_PAGECACHE_GRACE_PERIOD)
+            '/* {{ grace_period }} */' => $this->_scopeConfig->getValue(self::XML_VARNISH_PAGECACHE_GRACE_PERIOD),
+            '/* {{ normalize_params }} */' => $this->_getNormaliseParams()
         ];
     }
 
@@ -249,6 +252,30 @@ protected function _getDesignExceptions()
         return $result;
     }
 
+   /**
+   * Get a pipe separated list of query string parameters that Varnish should strip off incoming requests
+   * when determining a match for the page. This can be used to strip tracking parameters off urls
+   * e.g. gclid, gtm_source, gtm_medium etc.
+   *
+   * @return string|null
+   */
+  protected function _getNormaliseParams()
+  {
+      $normaliseParams = $this->_scopeConfig->getValue(
+          self::XML_VARNISH_PAGECACHE_NORMALIZE_PARAMS,
+          \Magento\Store\Model\ScopeInterface::SCOPE_STORE
+      );
+
+      if (empty($normaliseParams)) {
+          return $normaliseParams;
+      }
+
+      # strip carriage returns and trailing comma if present and swap out for pipes.
+      $normaliseParams = preg_replace("/(\r|\n)*|\|$/","",str_replace(",","|",$normaliseParams));
+
+      return $normaliseParams;
+  }
+
     /**
      * Whether a cache type is enabled in Cache Management Grid
      *

app/code/Magento/PageCache/etc/adminhtml/system.xml

@@ -49,6 +49,14 @@
                             <field id="caching_application">1</field>
                         </depends>
                     </field>
+                    <field id="normalize_params" type="textarea" translate="label comment" sortOrder="30" showInDefault="1" showInWebsite="0" showInStore="0">
+                        <label>Normalize Parameters</label>
+                        <comment>Comma separated list of query string parameters that Varnish should cleanse from the request URL.</comment>
+                        <backend_model>Magento\PageCache\Model\System\Config\Backend\Varnish</backend_model>
+                        <depends>
+                            <field id="caching_application">1</field>
+                        </depends>
+                    </field>
                     <field id="export_button_version4" type="button" sortOrder="35" showInDefault="1" showInWebsite="0" showInStore="0">
                         <label>Export Configuration</label>
                         <frontend_model>Magento\PageCache\Block\System\Config\Form\Field\Export\Varnish4</frontend_model>

app/code/Magento/PageCache/etc/config.xml

@@ -28,6 +28,7 @@
                     <backend_port>8080</backend_port>
                     <ttl>86400</ttl>
                     <grace_period>300</grace_period>
+                    <normalize_params>gclid,gclsrc,utm_content,utm_term,utm_campaign,utm_medium,utm_source,_ga</normalize_params>
                 </default>
             </full_page_cache>
         </system>

app/code/Magento/PageCache/etc/varnish4.vcl

@@ -91,11 +91,12 @@ sub vcl_recv {
         }
     }
 
-    # Remove Google gclid parameters to minimize the cache objects
-    set req.url = regsuball(req.url,"\?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA"
-    set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
-    set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"
+    # normalize query string parameters that Varnish should not vary cache for
+    set req.url = regsuball(req.url, "((\?)|&)(/* {{ normalize_params }} */)=[^&]*", "\2");
 
+    # second pass to ensure after params have been stripped we're not left with ?!,?,& causing a cache miss
+    set req.url = regsub(req.url, "(\?&|\?|&)$", "");
+
     # Static files caching
     if (req.url ~ "^/(pub/)?(media|static)/.*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|html|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|tiff|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)$") {
         # Static files should not be cached by default

app/code/Magento/PageCache/etc/varnish5.vcl

@@ -92,10 +92,11 @@ sub vcl_recv {
         }
     }
 
-    # Remove Google gclid parameters to minimize the cache objects
-    set req.url = regsuball(req.url,"\?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA"
-    set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
-    set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"
+    # normalize query string parameters that Varnish should not vary cache for
+    set req.url = regsuball(req.url, "((\?)|&)(/* {{ normalize_params }} */)=[^&]*", "\2");
+
+    # second pass to ensure after params have been stripped we're not left with ?!,?,& causing a cache miss
+    set req.url = regsub(req.url, "(\?&|\?|&)$", "");
 
     # Static files caching
     if (req.url ~ "^/(pub/)?(media|static)/.*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|html|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|tiff|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)$") {

app/code/Magento/PageCache/i18n/en_US.csv

@@ -17,3 +17,5 @@
 "Public content cache lifetime in seconds. If field is empty default value 86400 will be saved. ","Public content cache lifetime in seconds. If field is empty default value 86400 will be saved. "
 "Page Cache","Page Cache"
 "Full page caching","Full page caching"
+"Normalize Parameters","Normalize Parameters"
+"Comma separated list of query string parameters that Varnish should cleanse from the request URL.","Comma separated list of query string parameters that Varnish should cleanse from the request URL."