split log-cache from doppler, use syslog ingress

Making this change for a few reasons:
- The scaling needs of dopplers and log-cache are often different, so
  grouping them together can be problematic. Dopplers are limited to ~40
  instances and some high traffic foundations need larger log-cache
  instance groups.
- Syslog ingress eliminates the load on dopplers and traffic controllers
  to get envelopes to log-cache. This increases the load slightly on
  diego cells, and eliminates significant load on dopplers/tc's.

It's recomended after deploying this change to evaluate the memory
allocated doppler nodes and switch them to compute heavy instances and
deploy log-cache to high memory intances.

cf-deployment.yml

@@ -83,6 +83,7 @@ addons:
         cert: "((syslog_agent_metrics_tls.certificate))"
         key: "((syslog_agent_metrics_tls.private_key))"
         server_name: syslog_agent_metrics
+      drain_ca_cert: "((log_cache_syslog_tls.ca))"
 
 - name: prom_scraper
   include:
@@ -338,7 +339,13 @@ addons:
           deployment: cf
           network: default
           domain: bosh
-
+      - domain: log-cache.service.cf.internal
+        targets:
+        - deployment: cf
+          domain: bosh
+          instance_group: log-cache
+          network: default
+          query: '*'
 
 instance_groups:
 - name: smoke-tests
@@ -1317,6 +1324,7 @@ instance_groups:
         cert: "((loggr_syslog_binding_cache_metrics_tls.certificate))"
         key: "((loggr_syslog_binding_cache_metrics_tls.private_key))"
         server_name: loggr_syslog_binding_cache_metrics
+      aggregate_drains: "syslog-tls://log-cache.service.cf.internal:6067?include-metrics-deprecated=true&ssl-strict-internal=true"
   - name: loggr-udp-forwarder
     release: loggregator-agent
     properties:
@@ -1426,27 +1434,16 @@ instance_groups:
         cert: "((loggr_udp_forwarder_tls.certificate))"
         key: "((loggr_udp_forwarder_tls.private_key))"
         server_name: loggr_udp_forwarder_metrics
-- name: doppler
+- name: log-cache
   azs:
   - z1
   - z2
-  instances: 4
-  vm_type: minimal
+  instances: 1
+  vm_type: small-highmem
   stemcell: default
   networks:
   - name: default
   jobs:
-  - name: doppler
-    release: loggregator
-    provides:
-      doppler: {as: doppler, shared: true}
-    properties:
-      loggregator:
-        tls:
-          ca_cert: "((loggregator_tls_doppler.ca))"
-          doppler:
-            cert: "((loggregator_tls_doppler.certificate))"
-            key: "((loggregator_tls_doppler.private_key))"
   - name: log-cache
     provides:
       log-cache: {shared: true}
@@ -1473,21 +1470,17 @@ instance_groups:
         key: "((log_cache_gateway_metrics_tls.private_key))"
         server_name: log_cache_gateway_metrics
     release: log-cache
-  - consumes:
-      reverse_log_proxy: {from: reverse_log_proxy}
-    name: log-cache-nozzle
+  - name: log-cache-syslog-server
+    release: log-cache
     properties:
+      tls:
+        cert: "((log_cache_syslog_tls.certificate))"
+        key: "((log_cache_syslog_tls.private_key))"
       metrics:
-        ca_cert: ((log_cache_nozzle_metrics_tls.ca))
-        cert: ((log_cache_nozzle_metrics_tls.certificate))
-        key: ((log_cache_nozzle_metrics_tls.private_key))
-        server_name: log_cache_nozzle_metrics
-      logs_provider:
-        tls:
-          ca_cert: ((logs_provider.ca))
-          cert: ((logs_provider.certificate))
-          key: ((logs_provider.private_key))
-    release: log-cache
+        ca_cert: "((log_cache_syslog_server_metrics_tls.ca))"
+        cert: "((log_cache_syslog_server_metrics_tls.certificate))"
+        key: "((log_cache_syslog_server_metrics_tls.private_key))"
+        server_name: log_cache_syslog_server_metrics
   - name: route_registrar
     properties:
       nats:
@@ -1526,6 +1519,27 @@ instance_groups:
         client_secret: ((uaa_clients_doppler_secret))
         internal_addr: https://uaa.service.cf.internal:8443
     release: log-cache
+- name: doppler
+  azs:
+  - z1
+  - z2
+  instances: 4
+  vm_type: minimal
+  stemcell: default
+  networks:
+  - name: default
+  jobs:
+  - name: doppler
+    release: loggregator
+    provides:
+      doppler: {as: doppler, shared: true}
+    properties:
+      loggregator:
+        tls:
+          ca_cert: "((loggregator_tls_doppler.ca))"
+          doppler:
+            cert: "((loggregator_tls_doppler.certificate))"
+            key: "((loggregator_tls_doppler.private_key))"
 - name: diego-cell
   azs:
   - z1
@@ -2244,6 +2258,16 @@ variables:
     common_name: localhost
     alternative_names:
     - localhost
+- name: log_cache_syslog_tls
+  type: certificate
+  options:
+    ca: loggregator_ca
+    common_name: log-cache.service.cf.internal
+    alternative_names:
+      - "q-s3.log-cache.default.cf.bosh"
+      - "log-cache.service.cf.internal"
+    extended_key_usage:
+      - server_auth
 - name: router_ca
   type: certificate
   options:
@@ -2471,6 +2495,16 @@ variables:
     common_name: metricScraperCA
     is_ca: true
 
+- name: log_cache_syslog_server_metrics_tls
+  type: certificate
+  update_mode: converge
+  options:
+    ca: metric_scraper_ca
+    common_name: log_cache_syslog_server_metrics
+    alternative_names:
+    - log_cache_syslog_server_metrics
+    extended_key_usage:
+    - server_auth
 - name: metrics_agent_tls
   type: certificate
   update_mode: converge

operations/experimental/use-logcache-syslog-ingress-windows2019.yml

@@ -1,4 +1,4 @@
+# Has been integrated into cf-deployment.yml
+#
+# Please delete this file in the future
 ---
-- type: replace
-  path: /instance_groups/name=windows2019-cell/jobs/name=loggr-syslog-agent-windows/properties/drain_ca_cert?
-  value: "((log_cache_syslog_tls.ca))"

operations/experimental/use-logcache-syslog-ingress.yml

@@ -1,54 +1,4 @@
+# Has been integrated into cf-deployment.yml
+#
+# Please delete this file in the future
 ---
-- type: replace
-  path: /instance_groups/name=doppler/jobs/name=log-cache-syslog-server?
-  value:
-    release: log-cache
-    name: log-cache-syslog-server
-    properties:
-      tls:
-        cert: "((log_cache_syslog_tls.certificate))"
-        key: "((log_cache_syslog_tls.private_key))"
-      metrics:
-        ca_cert: "((log_cache_syslog_server_metrics_tls.ca))"
-        cert: "((log_cache_syslog_server_metrics_tls.certificate))"
-        key: "((log_cache_syslog_server_metrics_tls.private_key))"
-        server_name: log_cache_syslog_server_metrics
-
-- type: replace
-  path: /variables/name=log_cache_syslog_tls?
-  value:
-    name: log_cache_syslog_tls
-    type: certificate
-    options:
-      ca: loggregator_ca
-      common_name: doppler.service.cf.internal
-      alternative_names:
-        - "q-s3.doppler.default.cf.bosh"
-        - "doppler.service.cf.internal"
-      extended_key_usage:
-        - server_auth
-
-- type: remove
-  path: /instance_groups/name=doppler/jobs/name=log-cache-nozzle?
-
-- type: replace
-  path: /instance_groups/name=scheduler/jobs/name=loggr-syslog-binding-cache/properties/aggregate_drains?
-  value: "syslog-tls://doppler.service.cf.internal:6067?include-metrics-deprecated=true&ssl-strict-internal=true"
-
-- type: replace
-  path: /addons/name=loggr-syslog-agent/jobs/name=loggr-syslog-agent/properties/drain_ca_cert?
-  value: "((log_cache_syslog_tls.ca))"
-
-- type: replace
-  path: /variables/name=log_cache_syslog_server_metrics_tls?
-  value:
-    name: log_cache_syslog_server_metrics_tls
-    type: certificate
-    update_mode: converge
-    options:
-      ca: metric_scraper_ca
-      common_name: log_cache_syslog_server_metrics
-      alternative_names:
-      - log_cache_syslog_server_metrics
-      extended_key_usage:
-      - server_auth

operations/rename-network-and-deployment.yml

@@ -12,6 +12,10 @@
   path: /instance_groups/name=doppler/networks/name=default/name
   value: ((network_name))
 
+- type: replace
+  path: /instance_groups/name=log-cache/networks/name=default/name
+  value: ((network_name))
+
 - type: replace
   path: /instance_groups/name=database/networks/name=default/name
   value: ((network_name))
@@ -134,6 +138,13 @@
       deployment: ((deployment_name))
       network: ((network_name))
       domain: bosh
+  - domain: log-cache.service.cf.internal
+    targets:
+    - query: '*'
+      instance_group: log-cache
+      deployment: ((deployment_name))
+      network: ((network_name))
+      domain: bosh
   - domain: file-server.service.cf.internal
     targets:
     - query: '*'

operations/scale-to-one-az.yml

@@ -65,6 +65,9 @@
 - type: replace
   path: /instance_groups/name=doppler/azs
   value: [ z1 ]
+- type: replace
+  path: /instance_groups/name=log-cache/azs
+  value: [ z1 ]
 - type: replace
   path: /instance_groups/name=log-api/azs
   value: [ z1 ]

operations/windows2019-cell.yml

@@ -164,6 +164,7 @@
           ca_cert: ((loggregator_tls_agent.ca))
           cert: ((loggregator_tls_agent.certificate))
           key: ((loggregator_tls_agent.private_key))
+        drain_ca_cert: ((log_cache_syslog_tls.ca))
       release: loggregator-agent
     - name: loggr-forwarder-agent-windows
       properties: