Permalink
Browse files

Test purging of ipro-rewritten resources. This previously worked in (#…

…1306)

Apache, but only today was fixed in ngx_pagespeed on 5/11/16, in
apache/incubator-pagespeed-ngx#1193

To test a fix to this bug, I need to alter files in the htdocs
directory during the test.  This is source-controlled.  We were doing
this before with mod_pagespeed_test/cache_flush but somehow getting
away with it (or just getting lucky with timing).  But this is really
a bad practice.

So in this pull-request I also change the infrastructure to simply copy
the cache_flush/ and purge/ subdirectories to htdocs/.  The rest of the
directories get symlinked.

This must be committed alongside apache/incubator-pagespeed-ngx#1197

Use full paths in ln -s and in cp -r, rather than using the cd-command-chain. (#1309)

More Makefile.tests changes that didn't make it into the last git commit.

setup the doc root before running vm_system_tests. (#1310)
  • Loading branch information...
jmarantz authored and crowell committed May 13, 2016
1 parent 1ff43e1 commit d1972f66f65a20624826362d7e2838dc91f59c95
Showing with 56 additions and 15 deletions.
  1. +20 −1 install/Makefile.tests
  2. +7 −7 install/debug.conf.template
  3. +8 −7 pagespeed/apache/system_test.sh
  4. +21 −0 pagespeed/system/system_test.sh
@@ -39,7 +39,7 @@ export APACHE_DEBUG_PAGESPEED_CONF
export APACHE_LOG
export MOD_PAGESPEED_CACHE

apache_vm_system_tests :
apache_vm_system_tests : setup_doc_root
$(MAKE) FAST_RESTART=1 apache_debug_smoke_test
$(MAKE) FAST_RESTART=1 apache_debug_downstream_caching_test
$(MAKE) FAST_RESTART=1 apache_debug_per_vhost_stats_test
@@ -454,3 +454,22 @@ apache_debug_unplugged_test : apache_install_conf apache_debug_restart
mv $(APACHE_DEBUG_PAGESPEED_CONF).save $(APACHE_DEBUG_PAGESPEED_CONF)
$(MAKE) apache_debug_stop
@echo PASS

# Target used to populate the docroot for a webserver, including the
# read-only mod_pagespeed_example, mod_pagespeed_test, and do_not_modify,
# plus the read/write areas for cache purging and flush tests. The
# read-only directories are symlinked in for speed, and the read/write
# directories are deep-copied, but they are small.
setup_doc_root :
mkdir -p $(APACHE_DOC_ROOT)
ln -Tsf "$(realpath $(INSTALL_DATA_DIR))/mod_pagespeed_example" \
$(APACHE_DOC_ROOT)/mod_pagespeed_example
ln -Tsf "$(realpath $(INSTALL_DATA_DIR))/mod_pagespeed_test" \
$(APACHE_DOC_ROOT)/mod_pagespeed_test
ln -Tsf "$(realpath $(INSTALL_DATA_DIR))/do_not_modify" \
$(APACHE_DOC_ROOT)/do_not_modify
rm -rf "$(APACHE_DOC_ROOT)/cache_flush"
cp -r "$(INSTALL_DATA_DIR)/mod_pagespeed_test/cache_flush" \
$(APACHE_DOC_ROOT)
rm -rf "$(APACHE_DOC_ROOT)/purge"
cp -r "$(INSTALL_DATA_DIR)/mod_pagespeed_test/purge" $(APACHE_DOC_ROOT)
@@ -430,7 +430,7 @@ ModPagespeedCompressMetadataCache true

# Make a non-empty subdirectory config to make sure that
# cache.flush updates get transmitted to nested configurations.
<Directory "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/cache_flush/" >
<Directory "@@APACHE_DOC_ROOT@@/cache_flush/" >
ModPagespeedRewriteLevel PassThrough
ModPagespeedEnableFilters inline_css
ModPagespeedDisableFilters add_instrumentation
@@ -479,7 +479,7 @@ NameVirtualHost localhost:@@APACHE_SECONDARY_PORT@@

# Make a non-empty subdirectory config to make sure that
# cache.flush updates get transmitted to nested configurations.
<Directory "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/cache_flush/" >
<Directory "@@APACHE_DOC_ROOT@@/cache_flush/" >
ModPagespeedRewriteLevel PassThrough
ModPagespeedEnableFilters inline_css
ModPagespeedDisableFilters add_instrumentation
@@ -1284,7 +1284,7 @@ ModPagespeedCreateSharedMemoryMetadataCache "@@MOD_PAGESPEED_CACHE@@_noned_shmm"
ModPagespeedEnableCachePurge on

ModPagespeedPurgeMethod PURGE
DocumentRoot "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/purge"
DocumentRoot "@@APACHE_DOC_ROOT@@/purge"
ModPagespeedFileCachePath "@@MOD_PAGESPEED_CACHE@@_purge"
ModPagespeedDisableFilters add_instrumentation
ModPagespeedRewriteLevel PassThrough
@@ -1295,15 +1295,15 @@ ModPagespeedCreateSharedMemoryMetadataCache "@@MOD_PAGESPEED_CACHE@@_noned_shmm"
# back on.
<VirtualHost localhost:@@APACHE_SECONDARY_PORT@@>
ServerName psoff-dir-on.example.com
DocumentRoot "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/purge"
DocumentRoot "@@APACHE_DOC_ROOT@@/purge"
ModPagespeedFileCachePath "@@MOD_PAGESPEED_CACHE@@_mpsoff_dir_on"
ModPagespeed off
ModPagespeedEnableCachePurge on
ModPagespeedPurgeMethod PURGE
ModPagespeedDisableFilters add_instrumentation
ModPagespeedRewriteLevel PassThrough
ModPagespeedEnableFilters rewrite_css
<Directory "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/purge">
<Directory "@@APACHE_DOC_ROOT@@/purge">
ModPagespeed on
</Directory>
</VirtualHost>
@@ -1312,15 +1312,15 @@ ModPagespeedCreateSharedMemoryMetadataCache "@@MOD_PAGESPEED_CACHE@@_noned_shmm"
# back on.
<VirtualHost localhost:@@APACHE_SECONDARY_PORT@@>
ServerName psoff-htaccess-on.example.com
DocumentRoot "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/purge"
DocumentRoot "@@APACHE_DOC_ROOT@@/purge"
ModPagespeedFileCachePath "@@MOD_PAGESPEED_CACHE@@_mpsoff_htaccess_on"
ModPagespeed off
ModPagespeedEnableCachePurge on
ModPagespeedPurgeMethod PURGE
ModPagespeedDisableFilters add_instrumentation
ModPagespeedRewriteLevel PassThrough
ModPagespeedEnableFilters rewrite_css
<Directory "@@APACHE_DOC_ROOT@@/mod_pagespeed_test/purge">
<Directory "@@APACHE_DOC_ROOT@@/purge">
AllowOverride All
</Directory>
</VirtualHost>
@@ -53,7 +53,8 @@ if [ "$CACHE_FLUSH_TEST" = "on" ]; then

# To fetch from the secondary test root, we must set
# http_proxy=${SECONDARY_HOSTNAME} during fetches.
SECONDARY_TEST_ROOT=http://secondary.example.com/mod_pagespeed_test
SECONDARY_ROOT="http://secondary.example.com"
SECONDARY_TEST_ROOT="$SECONDARY_ROOT/mod_pagespeed_test"
else
# Force the variable to be set albeit blank so tests don't fail.
: ${SECONDARY_HOSTNAME:=}
@@ -392,8 +393,8 @@ if [ "$CACHE_FLUSH_TEST" = "on" ]; then
echo http_proxy=$SECONDARY_HOSTNAME $WGET -O $WGET_OUTPUT $URL
http_proxy=$SECONDARY_HOSTNAME $WGET -O $WGET_OUTPUT $URL
check [ $(grep -o "<script" $WGET_OUTPUT|wc -l) = 3 ]
check [ $(grep -c "pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'beforeunload', '', 'http://secondary.example.com/mod_pagespeed_test/add_instrumentation.html');" $WGET_OUTPUT) = 1 ]
check [ $(grep -c "pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'load', '', 'http://secondary.example.com/mod_pagespeed_test/add_instrumentation.html');" $WGET_OUTPUT) = 1 ]
check [ $(grep -c "pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'beforeunload', '', '$SECONDARY_TEST_ROOT/add_instrumentation.html');" $WGET_OUTPUT) = 1 ]
check [ $(grep -c "pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'load', '', '$SECONDARY_TEST_ROOT/add_instrumentation.html');" $WGET_OUTPUT) = 1 ]

if [ "$NO_VHOST_MERGE" = "on" ]; then
start_test When ModPagespeedMaxHtmlParseBytes is not set, we do not insert \
@@ -426,7 +427,7 @@ if [ "$CACHE_FLUSH_TEST" = "on" ]; then
$SUDO touch ${MOD_PAGESPEED_CACHE}_ipro_for_browser/cache.flush
sleep 1

CACHE_TESTING_DIR="$APACHE_DOC_ROOT/mod_pagespeed_test/cache_flush/"
CACHE_TESTING_DIR="$APACHE_DOC_ROOT/cache_flush"
CACHE_TESTING_TMPDIR="$CACHE_TESTING_DIR/$$"
echo $SUDO mkdir "$CACHE_TESTING_TMPDIR"
$SUDO mkdir "$CACHE_TESTING_TMPDIR"
@@ -435,7 +436,7 @@ if [ "$CACHE_FLUSH_TEST" = "on" ]; then
$SUDO cp "$CACHE_TESTING_DIR/cache_flush_test.html" "$CACHE_TESTING_TMPDIR/"
CSS_FILE="$CACHE_TESTING_TMPDIR/update.css"
URL_PATH=cache_flush/$$/cache_flush_test.html
URL=$TEST_ROOT/$URL_PATH
URL="$PRIMARY_SERVER/$URL_PATH"
TMP_CSS_FILE=$TESTTMP/update.css

# First, write color 0 into the css file and make sure it gets inlined into
@@ -448,7 +449,7 @@ if [ "$CACHE_FLUSH_TEST" = "on" ]; then

# Also do the same experiment using a different VirtualHost. It points
# to the same htdocs, but uses a separate cache directory.
SECONDARY_URL=$SECONDARY_TEST_ROOT/$URL_PATH
SECONDARY_URL="$SECONDARY_ROOT/$URL_PATH"
http_proxy=$SECONDARY_HOSTNAME fetch_until $SECONDARY_URL "grep -c $COLOR0" 1

# Track how many flushes were noticed by Apache processes up till
@@ -1004,7 +1005,7 @@ check_from "$OUT" fgrep -q "$CONTENTS"

function scrape_secondary_stat {
http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \
http://secondary.example.com/mod_pagespeed_statistics/ | \
"$SECONDARY_ROOT/mod_pagespeed_statistics/" | \
scrape_pipe_stat "$1"
}

@@ -47,6 +47,8 @@ mkdir -p $OUTDIR
# NO_VHOST_MERGE to "on".
NO_VHOST_MERGE="${NO_VHOST_MERGE:-off}"

SUDO=${SUDO:-}

start_test Check for correct default pagespeed header format.
# This will be X-Page-Speed in nginx and X-ModPagespeed in apache. Accept both.
OUT=$($WGET_DUMP $EXAMPLE_ROOT/combine_css.html)
@@ -2162,6 +2164,7 @@ function cache_purge_test() {
echo $CURL --request PURGE --proxy $SECONDARY_HOSTNAME "$PURGE_URL"
check $CURL --request PURGE --proxy $SECONDARY_HOSTNAME "$PURGE_URL"
fi
echo ""
if [ $statistics_enabled -eq "0" ]; then
# Without statistics, we have no mechanism to transmit state-changes
# from one Apache child process to another, and so each process must
@@ -2186,13 +2189,30 @@ function cache_purge_test() {
yellow_css=$(grep yellow.css $FETCH_UNTIL_OUTFILE | cut -d\" -f6)
blue_css=$(grep blue.css $FETCH_UNTIL_OUTFILE | cut -d\" -f6)

purple_path="styles/$$"
purple_url="$PURGE_ROOT/$purple_path/purple.css"
purple_dir="$APACHE_DOC_ROOT/purge/$purple_path"
ls -ld $APACHE_DOC_ROOT $APACHE_DOC_ROOT/purge
echo $SUDO mkdir -p "$purple_dir"
$SUDO mkdir -p "$purple_dir"
purple_file="$purple_dir/purple.css"

for method in $CACHE_PURGE_METHODS; do
echo Individual URL Cache Purging with $method
check_from "$(read_metadata_cache $yellow_css)" fgrep -q cache_ok:true
check_from "$(read_metadata_cache $blue_css)" fgrep -q cache_ok:true
echo 'body { background: MediumPurple; }' > "/tmp/purple.$$"
$SUDO mv "/tmp/purple.$$" "$purple_file"
http_proxy=$SECONDARY_HOSTNAME fetch_until "$purple_url" 'fgrep -c 9370db' 1
echo 'body { background: black; }' > "/tmp/purple.$$"
$SUDO mv "/tmp/purple.$$" "$purple_file"

cache_purge $method "*"

check_from "$(read_metadata_cache $yellow_css)" fgrep -q cache_ok:false
check_from "$(read_metadata_cache $blue_css)" fgrep -q cache_ok:false
http_proxy=$SECONDARY_HOSTNAME fetch_until "$purple_url" 'fgrep -c #000' 1
cache_purge "$method" "$purple_path/purple.css"

sleep 1
STATS=$OUTDIR/purge.stats
@@ -2224,6 +2244,7 @@ function cache_purge_test() {
http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $PURGE_STATS_URL > $STATS.3
check_stat $STATS.2 $STATS.3 num_resource_fetch_successes 1
done
$SUDO rm -rf "$purple_dir"
}

if [ "$CACHE_FLUSH_TEST" = "on" ]; then

0 comments on commit d1972f6

Please sign in to comment.