system-test: infrastructure, host validation

.gitignore

@@ -0,0 +1 @@
+test/pagespeed_test.conf

README.md

@@ -201,77 +201,25 @@ you can set your beacons to go to another site by specifying a full path:
 ### Testing
 
 The generic Pagespeed system test is ported, and all but three tests pass.  To
-run it you need to first build and configure nginx.  Set it up something like:
-
-    ...
-    http {
-      pagespeed on;
-
-      // TODO(jefftk): this should be the default.
-      pagespeed RewriteLevel CoreFilters;
-
-      # This can be anywhere on your filesystem.
-      pagespeed FileCachePath /path/to/ngx_pagespeed_cache;
-
-      # For testing that the Library command works.
-      pagespeed Library 43 1o978_K0_LNE5_ystNklf
-                http://www.modpagespeed.com/rewrite_javascript.js;
-
-      # These gzip options are needed for tests that assume that pagespeed
-      # always enables gzip.  Which it does in apache, but not in nginx.
-      gzip on;
-      gzip_vary on;
-
-      # Turn on gzip for all content types that should benefit from it.
-      gzip_types application/ecmascript;
-      gzip_types application/javascript;
-      gzip_types application/json;
-      gzip_types application/pdf;
-      gzip_types application/postscript;
-      gzip_types application/x-javascript;
-      gzip_types image/svg+xml;
-      gzip_types text/css;
-      gzip_types text/csv;
-      # "gzip_types text/html" is assumed.
-      gzip_types text/javascript;
-      gzip_types text/plain;
-      gzip_types text/xml;
-
-      gzip_http_version 1.0;
-
-      ...
-
-      server {
-        listen 8050;
-        server_name localhost;
-        root /path/to/mod_pagespeed/src/install;
-        index index.html;
-
-        add_header Cache-Control "public, max-age=600";
-
-        # Disable parsing if the size of the HTML exceeds 50kB.
-        pagespeed MaxHtmlParseBytes 50000;
-
-        location /mod_pagespeed_test/no_cache/ {
-          add_header Cache-Control no-cache;
-        }
-
-        location /mod_pagespeed_test/compressed/ {
-          add_header Cache-Control max-age=600;
-          add_header Content-Encoding gzip;
-          types {
-            text/javascript custom_ext;
-          }
-        }
-
-        ...
-      }
-    }
+run it you need to first build nginx.  You also need to check out mod_pagespeed,
+but we can take a shortcut and do this the easy way, without gyp, because we
+don't need any dependencies:
+
+    $ svn checkout https://modpagespeed.googlecode.com/svn/trunk/ mod_pagespeed
+
+Then run:
 
-Then run the test, using the port you set up with `listen` in the configuration
-file:
+    test/nginx_system_test.sh \
+      primary_port \
+      secondary_port \
+      mod_pagespeed_dir \
+      file_cache_path \
+      nginx_executable_path
 
-    /path/to/ngx_pagespeed/test/nginx_system_test.sh localhost:8050
+For example:
+
+    $ test/nginx_system_test.sh 8050 8051 /path/to/mod_pagespeed \
+        /path/to/ngx_pagespeed_cache /path/to/sbin/nginx
 
 This should print out a lot of lines like:
 
@@ -302,16 +250,14 @@ you to [submit a bug](https://github.com/pagespeed/ngx_pagespeed/issues/new).
 
 Start an memcached server:
 
-    memcached -p 11213
+    memcached -p 11211
 
-To the configuration above add to the main or server block:
+In `ngx_pagespeed/test/pagespeed_test.conf.template` uncomment:
 
-    pagespeed MemcachedServers "localhost:11213";
+    pagespeed MemcachedServers "localhost:11211";
     pagespeed MemcachedThreads 1;
 
-Then run the system test:
-
-    /path/to/ngx_pagespeed/test/nginx_system_test.sh localhost:8050
+Then run the system test as above.
 
 #### Testing with valgrind
 

test/nginx_system_test.sh

@@ -25,16 +25,74 @@
 # Exits with status 2 if command line args are wrong.
 #
 # Usage:
-#   Set up nginx to serve mod_pagespeed/src/install/ statically at the server
-#   root, then run:
-#     ./ngx_system_test.sh HOST:PORT
+#   ./ngx_system_test.sh primary_port secondary_port \
+#                        mod_pagespeed_dir file_cache_path
 #   for example:
-#     ./ngx_system_test.sh localhost:8050
+#     ./ngx_system_test.sh 8050 8051 \
+#         /path/to/mod_pagespeed \
+#         /path.to/ngx_pagespeed_cache
 #
 
-this_dir="$( dirname "$0" )"
+if [ "$#" -ne 5 ] ; then
+  echo "Usage: $0 primary_port secondary_port mod_pagespeed_dir"
+  echo "  file_cache_path nginx_executable"
+  exit 1
+fi
+
+PRIMARY_PORT="$1"
+SECONDARY_PORT="$2"
+MOD_PAGESPEED_DIR="$3"
+FILE_CACHE_PATH="$4"
+NGINX_EXECUTABLE="$5"
+
+PRIMARY_HOSTNAME="localhost:$PRIMARY_PORT"
+SECONDARY_HOSTNAME="localhost:$SECONDARY_PORT"
+
+SERVER_ROOT="$MOD_PAGESPEED_DIR/src/install/"
+
+# We need check and check_not before we source SYSTEM_TEST_FILE that provides
+# them.
+function handle_failure_simple() {
+  echo "FAIL"
+  exit 2
+}
+function check_simple() {
+  echo "     check" "$@"
+  "$@" || handle_failure_simple
+}
+function check_not_simple() {
+  echo "     check_not" "$@"
+  "$@" && handle_failure_simple
+}
+
+this_dir="$( cd $(dirname "$0") && pwd)"
+
+# set up the config file for the test
+PAGESPEED_CONF="$this_dir/pagespeed_test.conf"
+PAGESPEED_CONF_TEMPLATE="$PAGESPEED_CONF.template"
+# check for config file template
+check_simple test -e "$PAGESPEED_CONF_TEMPLATE"
+# create PAGESPEED_CONF by substituting on PAGESPEED_CONF_TEMPLATE
+echo > $PAGESPEED_CONF <<EOF
+This file is automatically generated from $PAGESPEED_CONF_TEMPLATE"
+by nginx_system_test.sh; don't edit here."
+EOF
+cat $PAGESPEED_CONF_TEMPLATE \
+  | sed 's#@@FILE_CACHE_PATH@@#'"$FILE_CACHE_PATH/"'#' \
+  | sed 's#@@SERVER_ROOT@@#'"$SERVER_ROOT"'#' \
+  | sed 's#@@PRIMARY_PORT@@#'"$PRIMARY_PORT"'#' \
+  | sed 's#@@SECONDARY_PORT@@#'"$SECONDARY_PORT"'#' \
+  >> $PAGESPEED_CONF
+# make sure we substituted all the variables
+check_not_simple grep @@ $PAGESPEED_CONF
 
-SYSTEM_TEST_FILE="$this_dir/../../mod_pagespeed/src/install/system_test.sh"
+# restart nginx with new config
+killall nginx
+sleep .1
+check_simple "$NGINX_EXECUTABLE" -c "$PAGESPEED_CONF"
+
+# run generic system tests
+SYSTEM_TEST_FILE="$MOD_PAGESPEED_DIR/src/install/system_test.sh"
 
 if [ ! -e "$SYSTEM_TEST_FILE" ] ; then
   echo "Not finding $SYSTEM_TEST_FILE -- is mod_pagespeed not in a parallel"
@@ -51,5 +109,31 @@ PAGESPEED_EXPECTED_FAILURES="
   ~In-place resource optimization~
 "
 
+# The existing system test takes its arguments as positional parameters, and
+# wants different ones than we want, so we need to reset our positional args.
+set -- "$PRIMARY_HOSTNAME"
 source $SYSTEM_TEST_FILE
+
+# nginx-specific system tests
+
+# When we allow ourself to fetch a resource because the Host header tells us
+# that it is one of our resources, we should be fetching it from ourself.
+start_test Loopback fetches go to local IPs without DNS lookup
+
+# If we're properly fetching from ourself we will issue loopback fetches for
+# /mod_pagespeed_example/combine_javascriptN.js, which will succeed, so
+# combining will work.  If we're taking 'Host:www.google.com' to mean that we
+# should fetch from www.google.com then those fetches will fail because
+# google.com won't have /mod_pagespeed_example/combine_javascriptN.js and so
+# we'll not rewrite any resources.
+
+URL="$HOSTNAME/mod_pagespeed_example/combine_javascript.html"
+URL+="?ModPagespeed=on&ModPagespeedFilters=combine_javascript"
+fetch_until "$URL" "fgrep -c .pagespeed." 1 --header=Host:www.google.com
+
+# If this accepts the Host header and fetches from google.com it will fail with
+# a 404.  Instead it should use a loopback fetch and succeed.
+URL="$HOSTNAME/mod_pagespeed_example/.pagespeed.ce.8CfGBvwDhH.css"
+check wget -O /dev/null --header=Host:www.google.com "$URL"
+
 check_failures_and_exit