split log-cache from doppler, use syslog ingress

cf-deployment.yml

@@ -83,6 +83,7 @@ addons:
         cert: "((syslog_agent_metrics_tls.certificate))"
         key: "((syslog_agent_metrics_tls.private_key))"
         server_name: syslog_agent_metrics
+      drain_ca_cert: "((log_cache_syslog_tls.ca))"
 
 - name: prom_scraper
   include:
@@ -338,7 +339,13 @@ addons:
           deployment: cf
           network: default
           domain: bosh
-
+      - domain: log-cache.service.cf.internal
+        targets:
+        - deployment: cf
+          domain: bosh
+          instance_group: log-cache
+          network: default
+          query: '*'
 
 instance_groups:
 - name: smoke-tests
@@ -1317,6 +1324,7 @@ instance_groups:
         cert: "((loggr_syslog_binding_cache_metrics_tls.certificate))"
         key: "((loggr_syslog_binding_cache_metrics_tls.private_key))"
         server_name: loggr_syslog_binding_cache_metrics
+      aggregate_drains: "syslog-tls://log-cache.service.cf.internal:6067?include-metrics-deprecated=true&ssl-strict-internal=true"
   - name: loggr-udp-forwarder
     release: loggregator-agent
     properties:
@@ -1426,27 +1434,16 @@ instance_groups:
         cert: "((loggr_udp_forwarder_tls.certificate))"
         key: "((loggr_udp_forwarder_tls.private_key))"
         server_name: loggr_udp_forwarder_metrics
-- name: doppler
+- name: log-cache
   azs:
   - z1
   - z2
-  instances: 4
-  vm_type: minimal
+  instances: 1
+  vm_type: small-highmem
   stemcell: default
   networks:
   - name: default
   jobs:
-  - name: doppler
-    release: loggregator
-    provides:
-      doppler: {as: doppler, shared: true}
-    properties:
-      loggregator:
-        tls:
-          ca_cert: "((loggregator_tls_doppler.ca))"
-          doppler:
-            cert: "((loggregator_tls_doppler.certificate))"
-            key: "((loggregator_tls_doppler.private_key))"
   - name: log-cache
     provides:
       log-cache: {shared: true}
@@ -1473,21 +1470,17 @@ instance_groups:
         key: "((log_cache_gateway_metrics_tls.private_key))"
         server_name: log_cache_gateway_metrics
     release: log-cache
-  - consumes:
-      reverse_log_proxy: {from: reverse_log_proxy}
-    name: log-cache-nozzle
+  - name: log-cache-syslog-server
+    release: log-cache
     properties:
+      tls:
+        cert: "((log_cache_syslog_tls.certificate))"
+        key: "((log_cache_syslog_tls.private_key))"
       metrics:
-        ca_cert: ((log_cache_nozzle_metrics_tls.ca))
-        cert: ((log_cache_nozzle_metrics_tls.certificate))
-        key: ((log_cache_nozzle_metrics_tls.private_key))
-        server_name: log_cache_nozzle_metrics
-      logs_provider:
-        tls:
-          ca_cert: ((logs_provider.ca))
-          cert: ((logs_provider.certificate))
-          key: ((logs_provider.private_key))
-    release: log-cache
+        ca_cert: "((log_cache_syslog_server_metrics_tls.ca))"
+        cert: "((log_cache_syslog_server_metrics_tls.certificate))"
+        key: "((log_cache_syslog_server_metrics_tls.private_key))"
+        server_name: log_cache_syslog_server_metrics
   - name: route_registrar
     properties:
       nats:
@@ -1526,6 +1519,27 @@ instance_groups:
         client_secret: ((uaa_clients_doppler_secret))
         internal_addr: https://uaa.service.cf.internal:8443
     release: log-cache
+- name: doppler
+  azs:
+  - z1
+  - z2
+  instances: 4
+  vm_type: minimal
+  stemcell: default
+  networks:
+  - name: default
+  jobs:
+  - name: doppler
+    release: loggregator
+    provides:
+      doppler: {as: doppler, shared: true}
+    properties:
+      loggregator:
+        tls:
+          ca_cert: "((loggregator_tls_doppler.ca))"
+          doppler:
+            cert: "((loggregator_tls_doppler.certificate))"
+            key: "((loggregator_tls_doppler.private_key))"
 - name: diego-cell
   azs:
   - z1
@@ -2244,6 +2258,16 @@ variables:
     common_name: localhost
     alternative_names:
     - localhost
+- name: log_cache_syslog_tls
+  type: certificate
+  options:
+    ca: loggregator_ca
+    common_name: log-cache.service.cf.internal
+    alternative_names:
+      - "q-s3.log-cache.default.cf.bosh"
+      - "log-cache.service.cf.internal"
+    extended_key_usage:
+      - server_auth
 - name: router_ca
   type: certificate
   options:
@@ -2471,6 +2495,16 @@ variables:
     common_name: metricScraperCA
     is_ca: true
 
+- name: log_cache_syslog_server_metrics_tls
+  type: certificate
+  update_mode: converge
+  options:
+    ca: metric_scraper_ca
+    common_name: log_cache_syslog_server_metrics
+    alternative_names:
+    - log_cache_syslog_server_metrics
+    extended_key_usage:
+    - server_auth
 - name: metrics_agent_tls
   type: certificate
   update_mode: converge
@@ -2513,17 +2547,6 @@ variables:
     extended_key_usage:
     - server_auth
 
-- name: log_cache_nozzle_metrics_tls
-  type: certificate
-  update_mode: converge
-  options:
-    ca: metric_scraper_ca
-    common_name: log_cache_nozzle_metrics
-    alternative_names:
-    - log_cache_nozzle_metrics
-    extended_key_usage:
-    - server_auth
-
 - name: log_cache_cf_auth_proxy_metrics_tls
   type: certificate
   update_mode: converge

operations/experimental/use-logcache-syslog-ingress-windows2019.yml

@@ -1,4 +1,4 @@
+# Has been integrated into cf-deployment.yml
+#
+# Please delete this file in the future
 ---
-- type: replace
-  path: /instance_groups/name=windows2019-cell/jobs/name=loggr-syslog-agent-windows/properties/drain_ca_cert?
-  value: "((log_cache_syslog_tls.ca))"

operations/experimental/use-logcache-syslog-ingress.yml

@@ -1,54 +1,4 @@
+# Has been integrated into cf-deployment.yml
+#
+# Please delete this file in the future
 ---
-- type: replace
-  path: /instance_groups/name=doppler/jobs/name=log-cache-syslog-server?
-  value:
-    release: log-cache
-    name: log-cache-syslog-server
-    properties:
-      tls:
-        cert: "((log_cache_syslog_tls.certificate))"
-        key: "((log_cache_syslog_tls.private_key))"
-      metrics:
-        ca_cert: "((log_cache_syslog_server_metrics_tls.ca))"
-        cert: "((log_cache_syslog_server_metrics_tls.certificate))"
-        key: "((log_cache_syslog_server_metrics_tls.private_key))"
-        server_name: log_cache_syslog_server_metrics
-
-- type: replace
-  path: /variables/name=log_cache_syslog_tls?
-  value:
-    name: log_cache_syslog_tls
-    type: certificate
-    options:
-      ca: loggregator_ca
-      common_name: doppler.service.cf.internal
-      alternative_names:
-        - "q-s3.doppler.default.cf.bosh"
-        - "doppler.service.cf.internal"
-      extended_key_usage:
-        - server_auth
-
-- type: remove
-  path: /instance_groups/name=doppler/jobs/name=log-cache-nozzle?
-
-- type: replace
-  path: /instance_groups/name=scheduler/jobs/name=loggr-syslog-binding-cache/properties/aggregate_drains?
-  value: "syslog-tls://doppler.service.cf.internal:6067?include-metrics-deprecated=true&ssl-strict-internal=true"
-
-- type: replace
-  path: /addons/name=loggr-syslog-agent/jobs/name=loggr-syslog-agent/properties/drain_ca_cert?
-  value: "((log_cache_syslog_tls.ca))"
-
-- type: replace
-  path: /variables/name=log_cache_syslog_server_metrics_tls?
-  value:
-    name: log_cache_syslog_server_metrics_tls
-    type: certificate
-    update_mode: converge
-    options:
-      ca: metric_scraper_ca
-      common_name: log_cache_syslog_server_metrics
-      alternative_names:
-      - log_cache_syslog_server_metrics
-      extended_key_usage:
-      - server_auth

operations/rename-network-and-deployment.yml

@@ -12,6 +12,10 @@
   path: /instance_groups/name=doppler/networks/name=default/name
   value: ((network_name))
 
+- type: replace
+  path: /instance_groups/name=log-cache/networks/name=default/name
+  value: ((network_name))
+
 - type: replace
   path: /instance_groups/name=database/networks/name=default/name
   value: ((network_name))
@@ -134,6 +138,13 @@
       deployment: ((deployment_name))
       network: ((network_name))
       domain: bosh
+  - domain: log-cache.service.cf.internal
+    targets:
+    - query: '*'
+      instance_group: log-cache
+      deployment: ((deployment_name))
+      network: ((network_name))
+      domain: bosh
   - domain: file-server.service.cf.internal
     targets:
     - query: '*'

operations/scale-to-one-az.yml

@@ -65,6 +65,9 @@
 - type: replace
   path: /instance_groups/name=doppler/azs
   value: [ z1 ]
+- type: replace
+  path: /instance_groups/name=log-cache/azs
+  value: [ z1 ]
 - type: replace
   path: /instance_groups/name=log-api/azs
   value: [ z1 ]

operations/test/README.md

@@ -18,5 +18,3 @@ They may change without notice.
 | [`enable-nfs-test-server.yml`](enable-nfs-test-server.yml) | adds an NFS server to the deployment | nfstestserver can be reached at nfstestserver.service.cf.internal for acceptance testing purposes |
 | [`enable-nfs-test-ldapserver.yml`](enable-nfs-test-ldapserver.yml) | Adds an LDAP server to the deployment to allow testing of NFS volume services configured with LDAP authentication | Requires enable-nfs-volume-service.yml and enable-nfs-test-server.yml. nfstestldapserver can be reached at nfstestldapserver.service.cf.internal |
 | [`enable-smb-test-server.yml`](enable-smb-test-server.yml) | adds an SMB server to the deployment | smbtestserver can be reached at smbtestserver.service.cf.internal for acceptance testing purposes |
-| [`remove-logging-pipeline-with-danger.yml`](remove-logging-pipeline-with-danger.yml) | Remove logging pipeline v2 jobs. | |
-| [`remove-logging-pipeline-with-danger-windows2019.yml`](remove-logging-pipeline-with-danger-windows2019.yml) | Remove logging pipeline v2 jobs from the Windows 2019 Diego Cell. | Requires `remove-logging-pipeline-with-danger.yml` |