Add support for Ubuntu Xenial (16.04) on staging

.gitignore

@@ -90,7 +90,7 @@ _SUCCESS
 .settings
 .history
 .idea
-
+.vscode
 
 # Emacs
 \#*#

deployment/ansible/roles/model-my-watershed.app/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
-- { include: configuration.yml }
 - { include: app.yml, when: "['packer'] | is_in(group_names)" }
+- { include: configuration.yml }
 - { include: dependencies.yml }
 - { include: dev-and-test-dependencies.yml, when: "['development', 'test'] | some_are_in(group_names)" }
 - { include: static-files.yml }

deployment/ansible/roles/model-my-watershed.app/templates/gunicorn.py.j2

@@ -12,7 +12,7 @@ preload_app = False
 reload = True
 {% else %}
 accesslog = None
-errorlog = "{{ app_gunicorn_log }}"
+errorlog = "-"
 loglevel = 'info'
 preload_app = True
 reload = False

deployment/ansible/roles/model-my-watershed.app/templates/nginx-app.conf.j2

@@ -5,15 +5,18 @@ resolver_timeout 10s;
 
 proxy_cache_path {{ nginx_cache_dir }} levels=1:2 keys_zone=OBSERVATION:10m max_size=100m;
 
+set_real_ip_from 10.0.0.0/8;
+real_ip_header X-Forwarded-For;
+
 server {
   listen *:80;
   server_name _;
   client_max_body_size 5M;
 
   root {{ app_home }}/static;
 
-  error_log syslog:server=unix:/dev/log,facility=local7,tag=nginx_app,severity=error;
-  access_log syslog:server=unix:/dev/log,facility=local7,tag=nginx_app,severity=info combined;
+  error_log syslog:server=unix:/dev/log,nohostname,facility=local7,tag=nginx_app,severity=error;
+  access_log syslog:server=unix:/dev/log,nohostname,facility=local7,tag=nginx_app,severity=info combined;
 
   {% if ['packer'] | is_in(group_names) -%}
   location = /version.txt {

deployment/ansible/roles/model-my-watershed.base/tasks/dependencies.yml

@@ -14,4 +14,3 @@
 
 - name: Install PostgreSQL client
   apt: pkg=postgresql-client-{{ postgresql_version }}={{ postgresql_package_version }}
-

deployment/ansible/roles/model-my-watershed.base/tasks/main.yml

@@ -1,3 +1,7 @@
 ---
-- { include: configuration.yml }
-- { include: dependencies.yml }
+- import_tasks: configuration.yml
+
+- import_tasks: dependencies.yml
+
+- import_tasks: papertrail.yml
+  when: "['packer'] | is_in(group_names)"

deployment/ansible/roles/model-my-watershed.base/tasks/papertrail.yml

@@ -0,0 +1,8 @@
+- name: Download Papertrail root certificate
+  get_url:
+    url: https://papertrailapp.com/tools/papertrail-bundle.pem
+    dest: /etc/papertrail-bundle.pem
+    checksum: sha256:79ea479e9f329de7075c40154c591b51eb056d458bc4dff76d9a4b9c6c4f6d0b
+
+- name: Install rsyslog TLS utils
+  apt: name=rsyslog-gnutls

deployment/ansible/roles/model-my-watershed.celery-worker/handlers/main.yml

@@ -1,3 +1,6 @@
 ---
 - name: Restart Celery
   service: name=celeryd state=restarted
+
+- name: Restart Syslog
+  service: name=rsyslog state=restarted

deployment/ansible/roles/model-my-watershed.celery-worker/tasks/configuration.yml

@@ -8,3 +8,7 @@
   with_dict: "{{ app_config }}"
   notify:
     - Restart Celery
+
+- name: Send all logs to syslog
+  template: src=celery-syslog.j2 dest=/etc/rsyslog.d/23-celery.conf
+  notify: Restart Syslog

deployment/ansible/roles/model-my-watershed.celery-worker/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
-- { include: configuration.yml }
 - { include: app.yml, when: "['packer'] | is_in(group_names)" }
+- { include: configuration.yml }
 - { include: dependencies.yml }
 - { include: celery-user.yml }
 - { include: celery-service.yml }

deployment/ansible/roles/model-my-watershed.celery-worker/templates/celery-syslog.j2

@@ -0,0 +1,7 @@
+module(load="imfile")
+
+{% for id in range(0,celery_number_of_workers) %}
+input(type="imfile"
+      File="/var/log/celery/*-worker{{ id }}.log"
+      Tag="celery-worker-{{ id }}")
+{% endfor %}

deployment/ansible/roles/model-my-watershed.geoprocessing/templates/nginx-geoprocessing.conf.j2

@@ -1,9 +1,12 @@
+set_real_ip_from 10.0.0.0/8;
+real_ip_header X-Forwarded-For;
+
 server {
   listen *:80;
   server_name _;
 
-  error_log syslog:server=unix:/dev/log,facility=local7,tag=nginx_app,severity=error;
-  access_log syslog:server=unix:/dev/log,facility=local7,tag=nginx_geop,severity=info combined;
+  error_log syslog:server=unix:/dev/log,nohostname,facility=local7,tag=nginx_app,severity=error;
+  access_log syslog:server=unix:/dev/log,nohostname,facility=local7,tag=nginx_geop,severity=info combined;
 
   {% if ['packer'] | is_in(group_names) -%}
   location /version.txt {

deployment/ansible/roles/model-my-watershed.tiler/tasks/main.yml

@@ -1,5 +1,5 @@
 ---
-- { include: configuration.yml }
 - { include: app.yml, when: "['packer'] | is_in(group_names)" }
+- { include: configuration.yml }
 - { include: dependencies.yml }
 - { include: reverse-proxy.yml }

deployment/ansible/roles/model-my-watershed.tiler/templates/nginx-tiler.conf.j2

@@ -1,9 +1,12 @@
+set_real_ip_from 10.0.0.0/8;
+real_ip_header X-Forwarded-For;
+
 server {
   listen *:80;
   server_name _;
 
-  error_log syslog:server=unix:/dev/log,facility=local7,tag=nginx_tiler,severity=error;
-  access_log syslog:server=unix:/dev/log,facility=local7,tag=nginx_tiler,severity=info combined;
+  error_log syslog:server=unix:/dev/log,nohostname,facility=local7,tag=nginx_tiler,severity=error;
+  access_log syslog:server=unix:/dev/log,nohostname,facility=local7,tag=nginx_tiler,severity=info combined;
 
   {% if ['packer'] | is_in(group_names) -%}
   location = /version.txt {

deployment/cfn/application.py

@@ -71,6 +71,8 @@ class Application(StackNode):
         'RollbarServerSideAccessToken':
         ['global:RollbarServerSideAccessToken'],
         'ClientAppUserPassword': ['global:ClientAppUserPassword'],
+        'PapertrailHost': ['global:PapertrailHost'],
+        'PapertrailPort': ['global:PapertrailPort'],
     }
 
     DEFAULTS = {
@@ -94,10 +96,7 @@ class Application(StackNode):
     def set_up_stack(self):
         super(Application, self).set_up_stack()
 
-        tags = self.get_input('Tags').copy()
-        tags.update({'StackType': 'Application'})
-
-        self.default_tags = tags
+        self.default_tags = self.get_input('Tags').copy()
         self.region = self.get_input('Region')
 
         self.add_description('Application server stack for MMW')
@@ -266,6 +265,16 @@ def set_up_stack(self):
             Description='Password for the client apps django account',
         ), 'ClientAppUserPassword')
 
+        self.papertrail_host = self.add_parameter(Parameter(
+            'PapertrailHost', Type='String',
+            Description='Hostname for Papertrail log destination',
+        ), 'PapertrailHost')
+
+        self.papertrail_port = self.add_parameter(Parameter(
+            'PapertrailPort', Type='String',
+            Description='Port for Papertrail log destination',
+        ), 'PapertrailPort')
+
         app_server_lb_security_group, \
             app_server_security_group = self.create_security_groups()
         app_server_lb, \
@@ -293,7 +302,10 @@ def get_recent_app_server_ami(self):
         try:
             app_server_ami_id = self.get_input('AppServerAMI')
         except MKUnresolvableInputError:
-            app_server_ami_id = get_recent_ami(self.aws_profile, 'mmw-app-*')
+            filters = {'name': 'mmw-app-*'}
+
+            app_server_ami_id = get_recent_ami(self.aws_profile, filters=filters,
+                                               region=self.region)
 
         return app_server_ami_id
 
@@ -350,7 +362,7 @@ def create_security_groups(self):
                     IpProtocol='tcp', CidrIp=ALLOW_ALL_CIDR, FromPort=p,
                     ToPort=p
                 )
-                for p in [HTTP, HTTPS]
+                for p in [HTTP, HTTPS, self.get_input('PapertrailPort')]
             ],
             Tags=self.get_tags(Name=app_server_security_group_name)
         ))
@@ -621,7 +633,30 @@ def get_cloud_config(self, tile_distribution_endpoint):
                 '  - path: /etc/mmw.d/env/MMW_CLIENT_APP_USER_PASSWORD\n',
                 '    permissions: 0750\n',
                 '    owner: root:mmw\n',
-                '    content: ', Ref(self.client_app_user_password)]
+                '    content: ', Ref(self.client_app_user_password), '\n',
+                '\n',
+                'rsyslog:\n',
+                '  - $DefaultNetstreamDriverCAFile /etc/papertrail-bundle.pem # trust these CAs\n',
+                '  - $PreserveFQDN off\n',
+                '  - $ActionSendStreamDriver gtls # use gtls netstream driver\n',
+                '  - $ActionSendStreamDriverMode 1 # require TLS\n',
+                '  - $ActionSendStreamDriverAuthMode x509/name # authenticate by hostname\n',
+                '  - $ActionSendStreamDriverPermittedPeer *.papertrailapp.com\n',
+                '  - $ActionResumeInterval 10\n',
+                '  - $ActionQueueSize 100000\n',
+                '  - $ActionQueueDiscardMark 97500\n',
+                '  - $ActionQueueHighWaterMark 80000\n',
+                '  - $ActionQueueType LinkedList\n',
+                '  - $ActionQueueFileName papertrailqueue\n',
+                '  - $ActionQueueCheckpointInterval 100\n',
+                '  - $ActionQueueMaxDiskSpace 2g\n',
+                '  - $ActionResumeRetryCount -1\n',
+                '  - $ActionQueueSaveOnShutdown on\n',
+                '  - $ActionQueueTimeoutEnqueue 2\n',
+                '  - $ActionQueueDiscardSeverity 0\n',
+                '  - "*.*  @@', Ref(self.papertrail_host), ':', Ref(
+                    self.papertrail_port), '"\n',
+                'rsyslog_filename: 22-mmw-papertrail.conf\n']
 
     def get_tags(self, **kwargs):
         """Helper method to return Troposphere tags + default tags