[WIP] Left-overs and open questions from old infrastructure

Makefile

@@ -9,7 +9,7 @@
 #
 
 ifndef MAKEFLAGS
-MAKEFLAGS += -j 4
+MAKEFLAGS += -j 8
 endif
 
 export DEBUG=0
@@ -21,7 +21,7 @@ all: lint
 lint:
 	puppet-lint --fail-on-warnings .
 
-test: test-codeorigin-stage test-codeorigin-prod-http test-codeorigin-prod-https test-codeorigin-next-http test-codeorigin-next-https test-contentorigin-prod test-miscweb test-wpdocs
+test: test-codeorigin-prod-http test-codeorigin-prod-https test-codeorigin-next-http test-codeorigin-next-https test-codeorigin-stage test-contentorigin-prod test-miscweb test-wpdocs
 
 test-codeorigin-prod-http:
 	@ php test/CodeoriginTest.php "http://code.jquery.com"
@@ -39,6 +39,10 @@ test-codeorigin-next-https:
 	@ php test/CodeoriginTest.php "https://codeorigin.jquery.com"
 	@ echo "✅ $@"
 
+test-codeorigin-stage:
+	@ php test/CodeoriginTest.php "https://codeorigin-02.stage.ops.jquery.net"
+	@ echo "✅ $@"
+
 test-contentorigin-prod:
 	@ php test/ContentoriginTest.php
 	@ echo "✅ $@"
@@ -50,7 +54,3 @@ test-miscweb:
 test-wpdocs:
 	@ php test/WpdocsTest.php
 	@ echo "✅ $@"
-
-test-codeorigin-stage:
-	@ php test/CodeoriginTest.php "https://codeorigin-02.stage.ops.jquery.net"
-	@ echo "✅ $@"

doc/monitoring.md

@@ -18,7 +18,7 @@ HTTP/HTTPS monitoring every 10-15 minutes, with alerts sent to `site-monitor@js.
 
 Monitoring includes:
 
-* jQuery Content: jquery.com, jqueryui.com, jquerymobile.com.
 * jQuery CDN: code.jquery.com, content.jquery.com.
-* jQuery Blogs: blog.jquery.com, blog.jqueryui.com.
+* jQuery Doc sites: jquery.com, api.jquery.com, jqueryui.com, jquerymobile.com.
+* jQuery Blogs: blog.jquery.com, blog.jqueryui.com, blog.jquerymobile.com.
 * jQuery Foundation projects: qunitjs.com, gruntjs.com mochajs.org, eslint.org, lodash.com, etc.

doc/runbook-fastly-log.md

@@ -0,0 +1,61 @@
+# Runbook: Real-time logging at Fastly
+
+Aggregate statistics are available under the "Edge Observer" tab
+of the [Observability](https://docs.fastly.com/en/guides/about-the-observability-page)
+page in Fastly, this includes for example requests by cache response
+type (hit, miss, pass, synthetic, error), status code, object size,
+and HTTP version.
+
+
+There isn't a built-in way to review, e.g. a sampled web request log,
+but you can debug actual requests by setting up a logging endpoint
+under one of the "services" in your account. This endpiont then
+receives a stream of real-time log events.
+
+Documentation:
+* https://docs.fastly.com/en/guides/setting-up-remote-log-streaming
+* https://docs.fastly.com/en/guides/log-streaming-https
+* https://docs.fastly.com/en/guides/useful-conditions-for-logging
+* https://docs.fastly.com/en/guides/useful-variables-to-log
+
+## Example: Log "pass" requests
+
+* https://docs.fastly.com/en/guides/using-conditions
+* https://developer.fastly.com/reference/vcl/variables/
+* https://developer.fastly.com/reference/http/http-headers/
+
+Log the request URLs of requests that are considered uncachable
+(as opposed to a cache "miss").
+
+Condition:
+```
+req.http.Fastly-Cachetype == "PASS"
+```
+
+Logging endpoint:
+
+* Condition: "If pass"
+* Placement: "Format Version Default"
+* Log format: (simplified from the default to not contain any obvious PII)
+  ```json
+  {
+    "host": "%{if(req.http.Fastly-Orig-Host, req.http.Fastly-Orig-Host, req.http.Host)}V",
+    "url": "%{json.escape(req.url)}V",
+    "request_method": "%{json.escape(req.method)}V",
+    "response_status": %{resp.status}V
+  }
+  ```
+* URL: A newly generated URL from https://log-bin.fastly.dev/
+* Advanced
+  * Content type: `text/plain`
+  * Method: `POST`
+  * JSON log entry format: "Newline delimited"
+  * Select a log line format: "Blank"
+
+## Example: Sampled "miss" log
+
+* https://developer.fastly.com/reference/vcl/functions/randomness/randombool/
+
+```
+req.http.Fastly-Cachetype == "PASS"
+```

hieradata/common.yaml

@@ -150,18 +150,18 @@ docs_sites:
     active_theme: contribute.jquery.org
     enable_static_index: true
     redirects:
-      # E.g. http://contribute.jquery.org/CLA/status/?owner=jquery&repo=jquery&sha=XYZ
+      # E.g. https://contribute.jquery.org/CLA/status/?owner=jquery&repo=jquery&sha=XYZ
       - match:
           path: /CLA
           mode: prefix
         target:
-          url: https://cla.openjsf.org
+          url: https://cla.openjsf.org/
           mode: exact
       - match:
           path: /cla
           mode: prefix
         target:
-          url: https://cla.openjsf.org
+          url: https://cla.openjsf.org/
           mode: exact
   jquery_com:
     host: jquery.com
@@ -180,7 +180,7 @@ docs_sites:
         target:
           url: https://api.jquery.com
           mode: exact
-        permanent: false
+        permanent: true
       - match:
           path: /blog
           mode: prefix
@@ -195,6 +195,13 @@ docs_sites:
           url: https://bugs.jquery.com
           mode: exact
         permanent: false
+      - match:
+          path: /docs
+          mode: prefix
+        target:
+          url: https://api.jquery.com
+          mode: exact
+        permanent: true
       - match:
           path: /join
           mode: prefix
@@ -400,6 +407,11 @@ docs_sites:
     certificate: wordpress
     active_theme: jqueryui.com
     enable_static_index: true
+    extra_config: |
+      rewrite ^/docs/Changelog(.*)$ /changelog$1 permanent;
+      rewrite ^/docs/Theming(.*)$ https://learn.jquery.com/jquery-ui/theming/ permanent;
+      rewrite ^/docs(.*)$ $1 permanent;
+      rewrite ^/bugs/(.*)$ https://bugs.jqueryui.com/$1 permanent;
     redirects:
       - match:
           path: /docs/Changelog

modules/mariadb/templates/server/mariadb.cnf.erb

@@ -32,3 +32,13 @@ log_bin
 binlog-format           = ROW
 expire_logs_days        = 14
 max_binlog_size         = 100M
+
+
+# old/mysql/[mysqld_safe]/nice=0
+#skip-character-set-client-handshake
+#key_buffer    = 16M
+#max_allowed_packet  = 16M
+#thread_stack    = 128K
+#thread_cache_size = 8
+#query_cache_limit       = 1M
+#query_cache_size        = 16M

modules/php/manifests/fpm.pp

@@ -6,6 +6,11 @@
 
   $version = $::php::version
   $ini_values = merge({
+    # TODO: Subst Debian default, or lower based on WordPress.org docs, or by testing
+    # TODO: add memory_used_max to jquery-wp-content footer?
+    # TODO: Beware this affects miscweb as well. If we need different defaults, move to miscweb and wordpress-blogs separately.
+    'memory_limit' => '64M',
+    # TODO: https://github.com/jquery/infrastructure/issues/437
     'expose_php'      => 'On',
     # Enable deprecation warnings
     # Will be redundant on PHP 8.0+

modules/php/templates/fpm/pool.ini.erb

@@ -7,6 +7,20 @@ listen.owner = www-data
 listen.group = www-data
 
 pm = dynamic
+# TODO: Raise this?
+# Test concurrent load on doc site, e.g. 10 /?s= on learn.jq.c
+#
+# Test concurrent load on blog site, e.g. 10 /?s= on blog.jq.c
+#
+# Optimize for largest VM, fine to overload CPU on smaller ones?
+# Hard limit on memory though, ensure (php mem * max children) < mem - 1GB (0.5G for DB, 0.5G for misc including puppet and node-notifier)
+# Vary by hardware if needed. Or maybe still fine because th 256M limit
+# is mainly a ceiling for XML-RPC requests during builder. Web traffic uses
+# far less, doesn't reach limit.
+# Test :)
+#
+# The above is a reason to use a much lower mem limit on blog.jq.c.
+#
 pm.max_children = 5
 pm.start_servers = 2
 pm.min_spare_servers = 1

modules/profile/manifests/base.pp

@@ -30,7 +30,7 @@
     'curl',
     'git',
     'jq',
-    'molly-guard',
+    'molly-guard', #TODO: What is this again?
     'tmux',
   ])
 }

modules/profile/manifests/puppet/server.pp

@@ -83,6 +83,16 @@
     umask   => '002',
   }
 
+  # TODO: From old infra, needed?
+  # Disk usage going up? Files no longer in new puppet?
+  # managed by g10k? logrotated by Debian?
+  # cron { "delete old puppet reports":
+  #   command => 'find /var/lib/puppet/reports/ -type f -name \*.yaml -mtime +30 -exec rm -f {} \;',
+  #   weekday => 6,
+  #   hour => 4,
+  #   minute => 0,
+  # }
+
   file { [
     "${private_repo_dir}/hieradata/",
     "${private_repo_dir}/files/",

modules/profile/manifests/wordpress/base.pp

@@ -27,21 +27,18 @@
     extensions => [
       # https://make.wordpress.org/hosting/handbook/server-environment/#php-extensions
       'curl',
+      # php5-gd
+      # php-gettext
       { package => 'php-imagick' },
       'intl',
       'mbstring',
       'mysql',
+      # php5-mcrypt
       'xml',
       'zip',
     ],
   }
 
-  class { 'php::fpm':
-    ini_values_extra => {
-      'memory_limit' => '512M',
-    },
-  }
-
   class { 'wordpress::cli':
     version => $wordpress_cli_version,
   }
@@ -55,4 +52,15 @@
       content => template('profile/wordpress/base/default-tls.nginx.erb'),
     }
   }
+
+  # TODO: As static file?
+  # Or let Nginx rewrite to PHP and let WP-CLI set search option off for stage?
+  # Or let Nginx rewrite and do it within a jquery-wp-content hook?
+  #
+  # if ( $::environment == "stage" ) {
+  #  file { "${wp_docroot}/robots.txt":
+  #    content => "User-Agent: *
+  # Disallow: /"
+  #  }
+  # }
 }

modules/profile/manifests/wordpress/docs.pp

@@ -13,6 +13,16 @@
 ) {
   include profile::wordpress::base
 
+  class { 'php::fpm':
+    ini_values_extra => {
+      # Raised to accomodate uploads via XML-RPC from builder host.
+      # TODO: if runtime PHP is allowed to raise this, then remove this
+      # in favour of an emperically-found default in php::fpm and an overrride
+      # in jquery-wp-content specific to XML-RPL requests.
+      'memory_limit' => '256M',
+    },
+  }
+
   git::clone { 'jquery-wp-content':
     path   => '/srv/wordpress/jquery-wp-content',
     remote => 'https://github.com/jquery/jquery-wp-content',

modules/profile/templates/filestash/vhost.nginx.erb

@@ -16,6 +16,9 @@ server {
 
   location / {
     root /srv/filestash/data;
+
+    # Probably here, or maybe as extra_config under releases_jquery_com
+    # add_header Access-Control-Allow-Origin *;
   }
 
   location /.well-known/acme-challenge {

modules/wordpress/templates/site/site.nginx.erb

@@ -39,6 +39,7 @@ server {
   location = /robots.txt {
     access_log off;
     log_not_found off;
+   #try_files $uri /wp-content/themes/:host/robots.txt /index.php$is_args$args;
     try_files $uri /wp-content/themes/<%= @active_theme %>/robots.txt;
   }
 
@@ -54,8 +55,45 @@ server {
   location ~ \.php$ {
     client_max_body_size 25M;
     try_files            $uri =404;
+    #fastcgi_buffers 16 16k;
+    #fastcgi_buffer_size 32k;
     fastcgi_pass         unix:/run/php/php<%= @php_fpm_version %>-fpm.sock;
     fastcgi_param        SCRIPT_FILENAME $request_filename;
     include              /etc/nginx/fastcgi_params;
   }
+
+  # TODO:
+  # Inside wp-content, upload or write a small plugin that
+  # emits non-zero Cache-Control: public, max-age=.. for
+  # for simple page views, i.e. GET, no cookies being emitted, and no other
+  # cache-control set already.
+}
+<!--
+    # Deny WordPress pingbacks, this was the cause of a ddos for us on 5/14/2014.
+    # But allow WordPress POST's to itself
+    # TODO: Block POST from Fastly/Cloudflare, once cron is moved to CLI.
+
+    # git blame - which site uses/used this?
+    location ~ ^/(?<pre>.+/)?resources/(?<foo>.+\.php)$ {
+        alias /var/www/wordpress/gw-resources/$host/$pre$foo;
+        client_max_body_size 25M;
+        fastcgi_buffers 16 16k;
+        fastcgi_buffer_size 32k;
+        fastcgi_pass   unix:/var/run/php5-fpm.sock;
+        fastcgi_index  index.php;
+        include        /etc/nginx/fastcgi_params;
+    }
+    location ~ ^/(?<pre>.+/)?resources/(?<foo>.*) {
+        alias /var/www/wordpress/gw-resources/$host/$pre$foo;
+    }
+
+    # TODO: Why is this here?
+    # Pass uploaded files to wp-includes/ms-files.php.
+    rewrite /files/$ /index.php last;
+
+    # TODO: Why is this here?
+    if ($uri !~ wp-content/plugins) {
+        rewrite /files/(.+)$ /wordpress/wp-includes/ms-files.php?file=$1 last;
+    }
 }
+ -->

test/ContentoriginTest.php

@@ -9,8 +9,8 @@
 
 foreach ( [
 	'http://content.jquery.com',
+	'https://content.jquery.com',
 	'http://content.origin.jquery.com',
-	'https://content.jquery.com'
 ] as $server ) {
 	Unit::testHttp( $server, '/podcast/wp-content/uploads/2010/09/jquerypodcast.png', [], [
 		'status' => '200',
@@ -32,8 +32,8 @@
 
 foreach ( [
 	'http://static.jquery.com',
+	'https://static.jquery.com',
 	'http://static.origin.jquery.com',
-	'https://static.jquery.com'
 ] as $server ) {
 	Unit::testHttp( $server, '/podcast/wp-content/uploads/2010/09/jquerypodcast.png', [], [
 		'status' => '200',

test/MiscwebTest.php

@@ -13,7 +13,7 @@
   'http://learn.jqueryui.com/something' => 'https://learn.jquery.com/jquery-ui/',
   'http://view.jquery.com/something' => 'https://releases.jquery.com/jquery/',
   'http://ui.jquery.com/about/' => 'https://jqueryui.com/about/',
-  'http://ui.jquery.com/bugs/ticket/3484' => 'https://jqueryui.com/bugs/ticket/3484',
+  'http://ui.jquery.com/bugs/ticket/3484' => 'https://bugs.jqueryui.com/ticket/3484',
   'http://wiki.jqueryui.com/Droppable' => 'https://jqueryui.pbworks.com/Droppable',
   'http://docs.jquery.com/UI/API/1.7.1/Datepicker' => 'https://api.jqueryui.com/datepicker/',
   'http://docs.jquery.com/QUnit/deepEqual' => 'https://api.qunitjs.com/deepEqual/',