split log-cache from doppler, use syslog ingress

cf-deployment.yml

@@ -83,6 +83,7 @@ addons:
         cert: "((syslog_agent_metrics_tls.certificate))"
         key: "((syslog_agent_metrics_tls.private_key))"
         server_name: syslog_agent_metrics
+      drain_ca_cert: "((log_cache_syslog_tls.ca))"
 
 - name: prom_scraper
   include:
@@ -338,7 +339,13 @@ addons:
           deployment: cf
           network: default
           domain: bosh
-
+      - domain: log-cache.service.cf.internal
+        targets:
+        - query: '*'
+          instance_group: log-cache
+          deployment: cf
+          network: default
+          domain: bosh
 
 instance_groups:
 - name: smoke-tests
@@ -942,6 +949,8 @@ instance_groups:
         staging_upload_user: staging_user
         staging_upload_password: "((cc_staging_upload_password))"
         temporary_use_logcache: true
+        logcache:
+          host: log-cache.service.cf.internal
         logcache_tls:
           private_key: "((cc_logcache_tls.private_key))"
           certificate: "((cc_logcache_tls.certificate))"
@@ -1317,6 +1326,7 @@ instance_groups:
         cert: "((loggr_syslog_binding_cache_metrics_tls.certificate))"
         key: "((loggr_syslog_binding_cache_metrics_tls.private_key))"
         server_name: loggr_syslog_binding_cache_metrics
+      aggregate_drains: "syslog-tls://log-cache.service.cf.internal:6067?include-metrics-deprecated=true&ssl-strict-internal=true"
   - name: loggr-udp-forwarder
     release: loggregator-agent
     properties:
@@ -1426,27 +1436,16 @@ instance_groups:
         cert: "((loggr_udp_forwarder_tls.certificate))"
         key: "((loggr_udp_forwarder_tls.private_key))"
         server_name: loggr_udp_forwarder_metrics
-- name: doppler
+- name: log-cache
   azs:
   - z1
   - z2
-  instances: 4
-  vm_type: minimal
+  instances: 1
+  vm_type: small-highmem
   stemcell: default
   networks:
   - name: default
   jobs:
-  - name: doppler
-    release: loggregator
-    provides:
-      doppler: {as: doppler, shared: true}
-    properties:
-      loggregator:
-        tls:
-          ca_cert: "((loggregator_tls_doppler.ca))"
-          doppler:
-            cert: "((loggregator_tls_doppler.certificate))"
-            key: "((loggregator_tls_doppler.private_key))"
   - name: log-cache
     provides:
       log-cache: {shared: true}
@@ -1473,21 +1472,17 @@ instance_groups:
         key: "((log_cache_gateway_metrics_tls.private_key))"
         server_name: log_cache_gateway_metrics
     release: log-cache
-  - consumes:
-      reverse_log_proxy: {from: reverse_log_proxy}
-    name: log-cache-nozzle
+  - name: log-cache-syslog-server
+    release: log-cache
     properties:
+      tls:
+        cert: "((log_cache_syslog_tls.certificate))"
+        key: "((log_cache_syslog_tls.private_key))"
       metrics:
-        ca_cert: ((log_cache_nozzle_metrics_tls.ca))
-        cert: ((log_cache_nozzle_metrics_tls.certificate))
-        key: ((log_cache_nozzle_metrics_tls.private_key))
-        server_name: log_cache_nozzle_metrics
-      logs_provider:
-        tls:
-          ca_cert: ((logs_provider.ca))
-          cert: ((logs_provider.certificate))
-          key: ((logs_provider.private_key))
-    release: log-cache
+        ca_cert: "((log_cache_syslog_server_metrics_tls.ca))"
+        cert: "((log_cache_syslog_server_metrics_tls.certificate))"
+        key: "((log_cache_syslog_server_metrics_tls.private_key))"
+        server_name: log_cache_syslog_server_metrics
   - name: route_registrar
     properties:
       nats:
@@ -1526,6 +1521,27 @@ instance_groups:
         client_secret: ((uaa_clients_doppler_secret))
         internal_addr: https://uaa.service.cf.internal:8443
     release: log-cache
+- name: doppler
+  azs:
+  - z1
+  - z2
+  instances: 3
+  vm_type: minimal
+  stemcell: default
+  networks:
+  - name: default
+  jobs:
+  - name: doppler
+    release: loggregator
+    provides:
+      doppler: {as: doppler, shared: true}
+    properties:
+      loggregator:
+        tls:
+          ca_cert: "((loggregator_tls_doppler.ca))"
+          doppler:
+            cert: "((loggregator_tls_doppler.certificate))"
+            key: "((loggregator_tls_doppler.private_key))"
 - name: diego-cell
   azs:
   - z1
@@ -2178,17 +2194,6 @@ variables:
     - rlp_gateway
     extended_key_usage:
     - client_auth
-- name: logs_provider
-  type: certificate
-  update_mode: converge
-  options:
-    ca: loggregator_ca
-    common_name: log-cache
-    alternative_names:
-    - log-cache
-    extended_key_usage:
-    - client_auth
-    - server_auth
 - name: log_cache_ca
   type: certificate
   options:
@@ -2244,6 +2249,16 @@ variables:
     common_name: localhost
     alternative_names:
     - localhost
+- name: log_cache_syslog_tls
+  type: certificate
+  options:
+    ca: loggregator_ca
+    common_name: log-cache.service.cf.internal
+    alternative_names:
+      - "q-s3.log-cache.default.cf.bosh"
+      - "log-cache.service.cf.internal"
+    extended_key_usage:
+      - server_auth
 - name: router_ca
   type: certificate
   options:
@@ -2471,6 +2486,16 @@ variables:
     common_name: metricScraperCA
     is_ca: true
 
+- name: log_cache_syslog_server_metrics_tls
+  type: certificate
+  update_mode: converge
+  options:
+    ca: metric_scraper_ca
+    common_name: log_cache_syslog_server_metrics
+    alternative_names:
+    - log_cache_syslog_server_metrics
+    extended_key_usage:
+    - server_auth
 - name: metrics_agent_tls
   type: certificate
   update_mode: converge
@@ -2513,17 +2538,6 @@ variables:
     extended_key_usage:
     - server_auth
 
-- name: log_cache_nozzle_metrics_tls
-  type: certificate
-  update_mode: converge
-  options:
-    ca: metric_scraper_ca
-    common_name: log_cache_nozzle_metrics
-    alternative_names:
-    - log_cache_nozzle_metrics
-    extended_key_usage:
-    - server_auth
-
 - name: log_cache_cf_auth_proxy_metrics_tls
   type: certificate
   update_mode: converge

ci/pipelines/cf-deployment.yml

@@ -1079,7 +1079,6 @@ jobs:
         operations/experimental/set-cpu-weight.yml
         operations/experimental/enable-cpu-throttling.yml
         operations/experimental/enable-containerd-for-processes.yml
-        operations/experimental/use-logcache-syslog-ingress.yml
         operations/increase-doppler-vm-type-from-minimal-to-small.yml
       VARS_FILES: |
         environments/test/hermione/bbl-state/vars/director-vars-file.yml
@@ -1133,7 +1132,6 @@ jobs:
         operations/experimental/enable-cpu-throttling.yml
         operations/experimental/enable-oci-phase-1.yml
         operations/experimental/enable-containerd-for-processes.yml
-        operations/experimental/use-logcache-syslog-ingress.yml
         operations/increase-doppler-vm-type-from-minimal-to-small.yml
       VARS_FILES: |
         environments/test/hermione/bbl-state/vars/director-vars-file.yml

operations/README.md

@@ -61,6 +61,8 @@ This is the README for Ops-files. To learn more about `cf-deployment`, go to the
 | [`use-internal-lookup-for-route-services.yml`](use-internal-lookup-for-route-services.yml) | Configure the gorouter to prefer internal lookup of route services. **Warning**: This enables a potential exploit detailed under [CVE-2019-3789](https://www.cloudfoundry.org/blog/cve-2019-3789/) |  | **NO** |
 | [`use-latest-stemcell.yml`](use-latest-stemcell.yml) | Use the latest stemcell available on your BOSH director instead of the one in `cf-deployment.yml`. **Caution**: This ops-file should not be used in conjunction with `use-compiled-releases.yml`, since the latter relies on a specific stemcell version being used. |  | **NO** |
 | [`use-latest-windows2019-stemcell.yml`](use-latest-windows2019-stemcell.yml) | Use the latest `windows2019` stemcell available on your BOSH director instead of the one in `windows2019-cell.yml` | Requires `windows2019-cell.yml` | **NO** |
+| [`use-log-cache-nozzle.yml`](use-log-cache-nozzle.yml) | Use RLP ingress for Log Cache in place of syslog | | **NO** |
+| [`use-log-cache-nozzle-windows2019.yml`](use-log-cache-nozzle-windows2019.yml) | Use RLP ingress for Log Cache in place of syslog for Windows cells | Requires `windows2019-cell.yml` | **NO** |
 | [`use-metric-store.yml`](use-metric-store.yml) | Adds a single-node metric store. | | **NO** |
 | [`use-operator-provided-router-tls-certificates.yml`](use-operator-provided-router-tls-certificates.yml) | Allows an operator to provide their own certificates for the gorouter by providing variables [`router_ssl_pem`](example-vars-files/vars-use-operator-provided-router-tls-certificates.yml) | This is required if using AWS Network Load Balancers. | **YES** |
 | [`use-postgres.yml`](use-postgres.yml) | Replaces the MySQL instance group with a postgres instance group. **Warning**: this will lead to total data loss if applied to an existing deployment with MySQL or removed from an existing deployment with postgres. |  | **YES** |

operations/experimental/README.md

@@ -38,8 +38,6 @@ This is the README for Experimental Ops-files. To learn more about `cf-deploymen
 | [`set-cpu-weight-windows2019.yml`](set-cpu-weight-windows2019.yml) | CPU shares for each garden container are proportional to its memory limits. | Requires `../windows2019-cell.yml` and `../use-online-windows2019fs.yml` | **NO** |
 | [`use-compiled-releases-windows.yml`](use-compiled-releases-windows.yml) | Reverts to source version of releases required for Windows cells | Intended for use with `use-compiled-releases.yml` and any of `windows*-cell.yml` | **YES** |
 | [`use-create-swap-delete-vm-strategy.yml`](use-create-swap-delete-vm-strategy.yml) | Configures the default [`vm_strategy`](https://bosh.io/docs/changing-deployment-vm-strategy/) to be `create-swap-delete`. | Requires BOSH director `v267.7+` | **NO** |
-| [`use-logcache-syslog-ingress.yml`](use-logcache-syslog-ingress.yml) | Uses syslog ingress for Log Cache in place of Loggregator | | **YES** |
-| [`use-logcache-syslog-ingress-windows2019.yml`](use-logcache-syslog-ingress-windows2019.yml) | Uses syslog ingress for Log Cache in place of Loggregator for Windows cells | Requires `use-logcache-syslog-ingress.yml` | **NO** |
 | [`disable-v2-api.yml`](disable-v2-api.yml) | Disable v2 Cloud Controller API endpoints | | **NO** |
 | [`disable-logs-in-firehose.yml`](disable-logs-in-firehose.yml) | Logs are not sent to dopplers, only metrics | | **NO** |
 | [`disable-logs-in-firehose-windows2019.yml`](disable-logs-in-firehose-windows-2019.yml) | Logs are not sent to dopplers, only metrics | | **NO** |

operations/rename-network-and-deployment.yml

@@ -12,6 +12,10 @@
   path: /instance_groups/name=doppler/networks/name=default/name
   value: ((network_name))
 
+- type: replace
+  path: /instance_groups/name=log-cache/networks/name=default/name
+  value: ((network_name))
+
 - type: replace
   path: /instance_groups/name=database/networks/name=default/name
   value: ((network_name))
@@ -134,6 +138,13 @@
       deployment: ((deployment_name))
       network: ((network_name))
       domain: bosh
+  - domain: log-cache.service.cf.internal
+    targets:
+    - query: '*'
+      instance_group: log-cache
+      deployment: ((deployment_name))
+      network: ((network_name))
+      domain: bosh
   - domain: file-server.service.cf.internal
     targets:
     - query: '*'

operations/scale-to-one-az.yml

@@ -65,6 +65,9 @@
 - type: replace
   path: /instance_groups/name=doppler/azs
   value: [ z1 ]
+- type: replace
+  path: /instance_groups/name=log-cache/azs
+  value: [ z1 ]
 - type: replace
   path: /instance_groups/name=log-api/azs
   value: [ z1 ]

operations/test/README.md

@@ -18,5 +18,3 @@ They may change without notice.
 | [`enable-nfs-test-server.yml`](enable-nfs-test-server.yml) | adds an NFS server to the deployment | nfstestserver can be reached at nfstestserver.service.cf.internal for acceptance testing purposes |
 | [`enable-nfs-test-ldapserver.yml`](enable-nfs-test-ldapserver.yml) | Adds an LDAP server to the deployment to allow testing of NFS volume services configured with LDAP authentication | Requires enable-nfs-volume-service.yml and enable-nfs-test-server.yml. nfstestldapserver can be reached at nfstestldapserver.service.cf.internal |
 | [`enable-smb-test-server.yml`](enable-smb-test-server.yml) | adds an SMB server to the deployment | smbtestserver can be reached at smbtestserver.service.cf.internal for acceptance testing purposes |
-| [`remove-logging-pipeline-with-danger.yml`](remove-logging-pipeline-with-danger.yml) | Remove logging pipeline v2 jobs. | |
-| [`remove-logging-pipeline-with-danger-windows2019.yml`](remove-logging-pipeline-with-danger-windows2019.yml) | Remove logging pipeline v2 jobs from the Windows 2019 Diego Cell. | Requires `remove-logging-pipeline-with-danger.yml` |