ansible/hg-web: configure response caching in S3 (bug 1507221); r=she…

…ehan

With the wireprotocache extension landing, we are now ready to use
it.

This commit teaches Ansible to configure the wireprotocache
extension for caching in S3.

We currently only have a single S3 bucket in us-west-1. This is fine
for our immediate testing needs. But we'll probably want to do
something more complicated later. We'll also presumably want to run
moto in the Docker environment and point the test environment at it.
This work is deferred for now.

Differential Revision: https://phabricator.services.mozilla.com/D11923

--HG--
extra : rebase_source : e832f0fb27dccf40b19c832139a3d2f7553630a6
extra : amend_source : e55594bc6f95a122e897ee1e8306bca89da5e380

ansible/deploy-hgmo.yml

@@ -14,7 +14,7 @@
     - name: capture host key
       slurp: src=/etc/mercurial/ssh/ssh_host_rsa_key.pub
       register: mirror_host_key
-      
+
     - name: capture hg-deploy-bot password
       slurp: src=/repo/hg/hg-deploy-bot.pw
       register: hg_deploy_bot_pw
@@ -55,6 +55,10 @@
       slurp: src=/etc/mercurial/bloxtool.json
       register: bloxtool_config
 
+    - name: capture aws s3 cache key
+      slurp: src=/etc/mercurial/s3cacher.json
+      register: s3cacher_config
+
   roles:
     - { role: hg-web,
         # We have to use hostvars to reference variables on other hosts.
@@ -69,6 +73,8 @@
         bloxtool_user: "{{ (bloxtool_config.content | b64decode | from_json).user }}",
         bloxtool_password: "{{ (bloxtool_config.content | b64decode | from_json).password }}",
         bloxtool_url: "{{ (bloxtool_config.content | b64decode | from_json).url }}",
+        s3_cache_access_key_id: "{{ (s3cacher_config.content | b64decode | from_json).access_key_id }}",
+        s3_cache_secret_key: "{{ s3cacher_config.content | b64decode | from_json).secret_access_key }}",
       }
 
 - hosts: hgssh-prod

ansible/roles/hg-web/defaults/main.yml

@@ -30,3 +30,9 @@ chroot_mozbuild_exists: False
 bloxtool_user: bloxtool_user
 bloxtool_password: bloxtool_password
 bloxtool_url: https://bloxtool
+
+s3_cache_access_key_id: dummy_aws_access_key_id
+s3_cache_secret_key: dummy_aws_secret_key
+s3_cache_bucket: moz-hg-wireproto-cache-testing
+s3_cache_region: us-west-1
+s3_cache_redirect_targets: https://s3-us-west-1.amazonaws.com/moz-hg-wireproto-cache-testing/

ansible/roles/hg-web/templates/hgrc.j2

@@ -77,6 +77,7 @@ replicateowner = /var/hg/version-control-tools/hgext/replicateowner
 serverlog = /var/hg/version-control-tools/hgext/serverlog
 hgmo = /var/hg/version-control-tools/hgext/hgmo
 vcsreplicator-hgweb = /var/hg/version-control-tools/pylib/vcsreplicator/vcsreplicator/hgwebext.py
+wireprotocache = /var/hg/version-control-tools/hgext/wireprotocache
 
 [interhg]
 a_httplinks = s|(https?://\S+)|<a href="\1">\1</a>|i
@@ -108,3 +109,23 @@ pullclonebundlesmanifest = true
 
 [mozilla]
 firefoxreleasesdb = /var/hg/fxreleases/db.db
+
+[wireprotocache]
+plugin = s3
+
+[wireprotocache.s3]
+access_key_id = {{ s3_cache_access_key_id | mandatory }}
+secret_access_key = {{ s3_cache_secret_key | mandatory }}
+
+bucket = {{ s3_cache_bucket | mandatory }}
+region = {{ s3_cache_region | mandatory }}
+
+# Private by default. Bucket IAM policy can override.
+cacheacl = private
+
+redirecttargets = {{ s3_cache_redirect_targets | mandatory }}
+
+# Caching small responses doesn't make much sense, as the overhead
+# from having the client make a new HTTP request will often be
+# higher than transferring data inline.
+minimumobjectsize = 65536