Test purging of ipro-rewritten resources. This previously worked in (#1306)

Apache, but only today was fixed in ngx_pagespeed on 5/11/16, in
apache/incubator-pagespeed-ngx#1193

To test a fix to this bug, I need to alter files in the htdocs
directory during the test.  This is source-controlled.  We were doing
this before with mod_pagespeed_test/cache_flush but somehow getting
away with it (or just getting lucky with timing).  But this is really
a bad practice.

So in this pull-request I also change the infrastructure to simply copy
the cache_flush/ and purge/ subdirectories to htdocs/.  The rest of the
directories get symlinked.

This must be committed alongside apache/incubator-pagespeed-ngx#1197

Use full paths in ln -s and in cp -r, rather than using the cd-command-chain. (#1309)

More Makefile.tests changes that didn't make it into the last git commit.

setup the doc root before running vm_system_tests. (#1310)

Author: jmarantz

install/Makefile.tests

@@ -39,7 +39,7 @@ export APACHE_DEBUG_PAGESPEED_CONF
 export APACHE_LOG
 export MOD_PAGESPEED_CACHE
 
-apache_vm_system_tests :
+apache_vm_system_tests : setup_doc_root
 	$(MAKE) FAST_RESTART=1 apache_debug_smoke_test
 	$(MAKE) FAST_RESTART=1 apache_debug_downstream_caching_test
 	$(MAKE) FAST_RESTART=1 apache_debug_per_vhost_stats_test
@@ -454,3 +454,22 @@ apache_debug_unplugged_test : apache_install_conf apache_debug_restart
 	mv $(APACHE_DEBUG_PAGESPEED_CONF).save $(APACHE_DEBUG_PAGESPEED_CONF)
 	$(MAKE) apache_debug_stop
 	@echo PASS
+
+# Target used to populate the docroot for a webserver, including the
+# read-only mod_pagespeed_example, mod_pagespeed_test, and do_not_modify,
+# plus the read/write areas for cache purging and flush tests.  The
+# read-only directories are symlinked in for speed, and the read/write
+# directories are deep-copied, but they are small.
+setup_doc_root :
+	mkdir -p $(APACHE_DOC_ROOT)
+	ln -Tsf "$(realpath $(INSTALL_DATA_DIR))/mod_pagespeed_example" \
+		$(APACHE_DOC_ROOT)/mod_pagespeed_example
+	ln -Tsf "$(realpath $(INSTALL_DATA_DIR))/mod_pagespeed_test" \
+		$(APACHE_DOC_ROOT)/mod_pagespeed_test
+	ln -Tsf "$(realpath $(INSTALL_DATA_DIR))/do_not_modify" \
+		$(APACHE_DOC_ROOT)/do_not_modify
+	rm -rf "$(APACHE_DOC_ROOT)/cache_flush" 
+	cp -r "$(INSTALL_DATA_DIR)/mod_pagespeed_test/cache_flush" \
+		$(APACHE_DOC_ROOT)
+	rm -rf "$(APACHE_DOC_ROOT)/purge" 
+	cp -r "$(INSTALL_DATA_DIR)/mod_pagespeed_test/purge" $(APACHE_DOC_ROOT)

install/debug.conf.template

@@ -430,7 +430,7 @@ ModPagespeedCompressMetadataCache true
 
 # Make a non-empty subdirectory config to make sure that
 # cache.flush updates get transmitted to nested configurations.
-<Directory "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/cache_flush/" >
+<Directory "@@APACHE_DOC_ROOT@@/cache_flush/" >
   ModPagespeedRewriteLevel PassThrough
   ModPagespeedEnableFilters inline_css
   ModPagespeedDisableFilters add_instrumentation
@@ -479,7 +479,7 @@ NameVirtualHost localhost:@@APACHE_SECONDARY_PORT@@
 
   # Make a non-empty subdirectory config to make sure that
   # cache.flush updates get transmitted to nested configurations.
-  <Directory "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/cache_flush/" >
+  <Directory "@@APACHE_DOC_ROOT@@/cache_flush/" >
     ModPagespeedRewriteLevel PassThrough
     ModPagespeedEnableFilters inline_css
     ModPagespeedDisableFilters add_instrumentation
@@ -1284,7 +1284,7 @@ ModPagespeedCreateSharedMemoryMetadataCache "@@MOD_PAGESPEED_CACHE@@_noned_shmm"
   ModPagespeedEnableCachePurge on
 
   ModPagespeedPurgeMethod PURGE
-  DocumentRoot "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/purge"
+  DocumentRoot "@@APACHE_DOC_ROOT@@/purge"
   ModPagespeedFileCachePath "@@MOD_PAGESPEED_CACHE@@_purge"
   ModPagespeedDisableFilters add_instrumentation
   ModPagespeedRewriteLevel PassThrough
@@ -1295,15 +1295,15 @@ ModPagespeedCreateSharedMemoryMetadataCache "@@MOD_PAGESPEED_CACHE@@_noned_shmm"
 # back on.
 <VirtualHost localhost:@@APACHE_SECONDARY_PORT@@>
   ServerName psoff-dir-on.example.com
-  DocumentRoot "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/purge"
+  DocumentRoot "@@APACHE_DOC_ROOT@@/purge"
   ModPagespeedFileCachePath "@@MOD_PAGESPEED_CACHE@@_mpsoff_dir_on"
   ModPagespeed off
   ModPagespeedEnableCachePurge on
   ModPagespeedPurgeMethod PURGE
   ModPagespeedDisableFilters add_instrumentation
   ModPagespeedRewriteLevel PassThrough
   ModPagespeedEnableFilters rewrite_css
-  <Directory "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/purge">
+  <Directory "@@APACHE_DOC_ROOT@@/purge">
     ModPagespeed on
   </Directory>
 </VirtualHost>
@@ -1312,15 +1312,15 @@ ModPagespeedCreateSharedMemoryMetadataCache "@@MOD_PAGESPEED_CACHE@@_noned_shmm"
 # back on.
 <VirtualHost localhost:@@APACHE_SECONDARY_PORT@@>
   ServerName psoff-htaccess-on.example.com
-  DocumentRoot "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/purge"
+  DocumentRoot "@@APACHE_DOC_ROOT@@/purge"
   ModPagespeedFileCachePath "@@MOD_PAGESPEED_CACHE@@_mpsoff_htaccess_on"
   ModPagespeed off
   ModPagespeedEnableCachePurge on
   ModPagespeedPurgeMethod PURGE
   ModPagespeedDisableFilters add_instrumentation
   ModPagespeedRewriteLevel PassThrough
   ModPagespeedEnableFilters rewrite_css
-  <Directory "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/purge">
+  <Directory "@@APACHE_DOC_ROOT@@/purge">
     AllowOverride All
   </Directory>
 </VirtualHost>

pagespeed/apache/system_test.sh

@@ -53,7 +53,8 @@ if [ "$CACHE_FLUSH_TEST" = "on" ]; then
 
   # To fetch from the secondary test root, we must set
   # http_proxy=${SECONDARY_HOSTNAME} during fetches.
-  SECONDARY_TEST_ROOT=http://secondary.example.com/mod_pagespeed_test
+  SECONDARY_ROOT="http://secondary.example.com"
+  SECONDARY_TEST_ROOT="$SECONDARY_ROOT/mod_pagespeed_test"
 else
   # Force the variable to be set albeit blank so tests don't fail.
   : ${SECONDARY_HOSTNAME:=}
@@ -392,8 +393,8 @@ if [ "$CACHE_FLUSH_TEST" = "on" ]; then
   echo http_proxy=$SECONDARY_HOSTNAME $WGET -O $WGET_OUTPUT $URL
   http_proxy=$SECONDARY_HOSTNAME $WGET -O $WGET_OUTPUT $URL
   check [ $(grep -o "<script" $WGET_OUTPUT|wc -l) = 3 ]
-  check [ $(grep -c "pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'beforeunload', '', 'http://secondary.example.com/mod_pagespeed_test/add_instrumentation.html');" $WGET_OUTPUT) = 1 ]
-  check [ $(grep -c "pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'load', '', 'http://secondary.example.com/mod_pagespeed_test/add_instrumentation.html');" $WGET_OUTPUT) = 1 ]
+  check [ $(grep -c "pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'beforeunload', '', '$SECONDARY_TEST_ROOT/add_instrumentation.html');" $WGET_OUTPUT) = 1 ]
+  check [ $(grep -c "pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'load', '', '$SECONDARY_TEST_ROOT/add_instrumentation.html');" $WGET_OUTPUT) = 1 ]
 
   if [ "$NO_VHOST_MERGE" = "on" ]; then
     start_test When ModPagespeedMaxHtmlParseBytes is not set, we do not insert \
@@ -426,7 +427,7 @@ if [ "$CACHE_FLUSH_TEST" = "on" ]; then
   $SUDO touch ${MOD_PAGESPEED_CACHE}_ipro_for_browser/cache.flush
   sleep 1
 
-  CACHE_TESTING_DIR="$APACHE_DOC_ROOT/mod_pagespeed_test/cache_flush/"
+  CACHE_TESTING_DIR="$APACHE_DOC_ROOT/cache_flush"
   CACHE_TESTING_TMPDIR="$CACHE_TESTING_DIR/$$"
   echo $SUDO mkdir "$CACHE_TESTING_TMPDIR"
   $SUDO mkdir "$CACHE_TESTING_TMPDIR"
@@ -435,7 +436,7 @@ if [ "$CACHE_FLUSH_TEST" = "on" ]; then
   $SUDO cp "$CACHE_TESTING_DIR/cache_flush_test.html" "$CACHE_TESTING_TMPDIR/"
   CSS_FILE="$CACHE_TESTING_TMPDIR/update.css"
   URL_PATH=cache_flush/$$/cache_flush_test.html
-  URL=$TEST_ROOT/$URL_PATH
+  URL="$PRIMARY_SERVER/$URL_PATH"
   TMP_CSS_FILE=$TESTTMP/update.css
 
   # First, write color 0 into the css file and make sure it gets inlined into
@@ -448,7 +449,7 @@ if [ "$CACHE_FLUSH_TEST" = "on" ]; then
 
   # Also do the same experiment using a different VirtualHost.  It points
   # to the same htdocs, but uses a separate cache directory.
-  SECONDARY_URL=$SECONDARY_TEST_ROOT/$URL_PATH
+  SECONDARY_URL="$SECONDARY_ROOT/$URL_PATH"
   http_proxy=$SECONDARY_HOSTNAME fetch_until $SECONDARY_URL "grep -c $COLOR0" 1
 
   # Track how many flushes were noticed by Apache processes up till
@@ -1004,7 +1005,7 @@ check_from "$OUT" fgrep -q "$CONTENTS"
 
 function scrape_secondary_stat {
   http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
-    http://secondary.example.com/mod_pagespeed_statistics/ | \
+    "$SECONDARY_ROOT/mod_pagespeed_statistics/" | \
     scrape_pipe_stat "$1"
 }
 

pagespeed/system/system_test.sh

@@ -47,6 +47,8 @@ mkdir -p $OUTDIR
 # NO_VHOST_MERGE to "on".
 NO_VHOST_MERGE="${NO_VHOST_MERGE:-off}"
 
+SUDO=${SUDO:-}
+
 start_test Check for correct default pagespeed header format.
 # This will be X-Page-Speed in nginx and X-ModPagespeed in apache.  Accept both.
 OUT=$($WGET_DUMP $EXAMPLE_ROOT/combine_css.html)
@@ -2162,6 +2164,7 @@ function cache_purge_test() {
       echo $CURL --request PURGE --proxy $SECONDARY_HOSTNAME "$PURGE_URL"
       check $CURL --request PURGE --proxy $SECONDARY_HOSTNAME "$PURGE_URL"
     fi
+    echo ""
     if [ $statistics_enabled -eq "0" ]; then
       # Without statistics, we have no mechanism to transmit state-changes
       # from one Apache child process to another, and so each process must
@@ -2186,13 +2189,30 @@ function cache_purge_test() {
   yellow_css=$(grep yellow.css $FETCH_UNTIL_OUTFILE | cut -d\" -f6)
   blue_css=$(grep blue.css $FETCH_UNTIL_OUTFILE | cut -d\" -f6)
 
+  purple_path="styles/$$"
+  purple_url="$PURGE_ROOT/$purple_path/purple.css"
+  purple_dir="$APACHE_DOC_ROOT/purge/$purple_path"
+  ls -ld $APACHE_DOC_ROOT $APACHE_DOC_ROOT/purge
+  echo $SUDO mkdir -p "$purple_dir"
+  $SUDO mkdir -p "$purple_dir"
+  purple_file="$purple_dir/purple.css"
+
   for method in $CACHE_PURGE_METHODS; do
     echo Individual URL Cache Purging with $method
     check_from "$(read_metadata_cache $yellow_css)" fgrep -q cache_ok:true
     check_from "$(read_metadata_cache $blue_css)" fgrep -q cache_ok:true
+    echo 'body { background: MediumPurple; }' > "/tmp/purple.$$"
+    $SUDO mv "/tmp/purple.$$" "$purple_file"
+    http_proxy=$SECONDARY_HOSTNAME fetch_until "$purple_url" 'fgrep -c 9370db' 1
+    echo 'body { background: black; }' > "/tmp/purple.$$"
+    $SUDO mv "/tmp/purple.$$" "$purple_file"
+
     cache_purge $method "*"
+
     check_from "$(read_metadata_cache $yellow_css)" fgrep -q cache_ok:false
     check_from "$(read_metadata_cache $blue_css)" fgrep -q cache_ok:false
+    http_proxy=$SECONDARY_HOSTNAME fetch_until "$purple_url" 'fgrep -c #000' 1
+    cache_purge "$method" "$purple_path/purple.css"
 
     sleep 1
     STATS=$OUTDIR/purge.stats
@@ -2224,6 +2244,7 @@ function cache_purge_test() {
     http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $PURGE_STATS_URL > $STATS.3
     check_stat $STATS.2 $STATS.3 num_resource_fetch_successes 1
   done
+  $SUDO rm -rf "$purple_dir"
 }
 
 if [ "$CACHE_FLUSH_TEST" = "on" ]; then