add msk bridge and remove hardcoded kafka url

helm/coco-kafka-bridge/app-configs/kafka-bridge_eks_delivery_dev.yaml

@@ -5,11 +5,10 @@ target_env: dev
 pubAauth:
   pub-k8s-auth: "AgAgqqd2kv5LlPTe1w8s48ygmWksU7W+Pzp+y1V60uMzDn6/OWW7EaHqDWWgKGnZJ3Fru+wswE8ZFlrevBgAfOjsr0NeZmSdHK90ZBS19udtbPUo2mfzca5qN7oQuBGghp/ggpvPKQeTXrZl/mNyg3qTwAAe6e2drC1vadgqYy4XsbNmJG9oA58PIFXg/hu7ZcSESYWTN6vmdPFJfuG+7eivAq1a6owDrc9ILIFVbdPUv8DhPt9EWuMiMi7jvJurqn/ioONbU9NP5cI8nZe1C2moESjP1gxjVF7eTmlIWb0UOUmqqwHvTVqzASOUQxy9EfuT+536MYhxoxl/GNgWwezeJHK+qW4XgcM4UK5EXUkPEN5SOiwTKRMY+v6E2LpL0jUD1R+Xn2ZKYSFqhvWPufhZhF9c2Za9HTVf8DJlkzs3ANqipPQHFP/lDVQvc3qtrai2FVLbY7XX7Uh59oEJPeS2Hht07hQ/osSrgn04uS56QonWyqgPzhiuzAI8JqtdnIpmNzfZQewG4e6/tu0pxZOJHRF436m3yclZgTnenQvx5d5GkBnEoFNJ3elALVAAJaxFHfnS3kDfcpwgs0dCWh7199U1ettedDqNixCctIdwyZur38EueBMpLK+TZMC5eC1L0lAALfWeJ5otJbR2w3/k8DdkH6IItH44GmffkGYJisnlxhPefRssvYv2WJ51/JYmZEJqEIL4jRJVKz9ZrI+Hp2BrC/aQVxOpkHBkke/IQVC5eSBk7jYMcvyw8js9KpRyCwOWU3TpnvUQscV5fK521JPPxmU1eDnAy+1urqM="
   pub-pre-prod-auth: "AgAtrOQeKhxdTalE1V09mNlX4oIswE7lau/ZlFn3i2SWNP+U+AcuMmE0xLNkpd1bc6EW4sbyDeNiEMG7VHJ3g4TRmilXAH1szmYteqvthVpUku9FHd8opAq0WOIQEEFZnebIMKmsl98NCCvr/Yt6syVkvbhncAuErlKo8iaKFidszEBxcMnUjhaswnOJIG677CXcwDK7XYDzGgyyfQl6hBQPKV43WQpiPmiR6qDybmgFKyp3/bFqA20zNbSf3U+t8UZZGYuCCxP+cx4c8IBtWg/pTsXYoRNqIzksCHGuJ0nQ8efqaHOX72tPaKXw1ynGxBIuxSxemrXUO0+atqDX94asoHdz5Yc/ktT5ws2PmE++vnLdKSmS7cnWQXPytn3drK1cYU+wiLQWmn4Q+w5DHuYr8q739FNnUb+jFV1UEAx4i/+j5QNp6yldgs96E0I3hjKELDeELENGu9XLIpjfdlOk9lnTfqGql3h8HPdJDOta6qLoaj63tICJxr+h8xFuaYm3K3uSjWVnpsXR3yjxiZjajFBe6TU5LA0jFDmxzOKFOlukkbHUmivQZkOZMGJUNtI3myBKTPpBHd+dRyoJELATW1uZ+ZGPzGaWgH4gGOObQI4KO3CFQ3DYjaDVFHbs7fSZY1u3zapcIHUMvQHhTbj1ij3m7j7X+u4H4pRLHreyxVuA9E6/ZlE8ps91LOAHZ8Fo1ksxmWUWbObdE0aJX9KKoTxhTG7eEi1JsKOr58yN3nwrEjbi2CZRGMsNlNKMWslTIN4d8tcXFTymvCqaaA=="
-
 bridges:
   # Cms bridge to pub cluster
   - name: cms-kafka-bridge-pub
-    sourceKafkaProxyUrl: "https://upp-k8s-dev-publish-eu.upp.ft.com" # use the pub cluster url
+    sourceKafkaProxyUrl: "https://upp-k8s-dev-publish-eu.upp.ft.com/__kafka-rest-proxy" # use the pub cluster url
     groupIdPrefix: kafka-bridge-pub
     topic: NativeCmsPublicationEvents
     producer: "http://cms-notifier:8080"
@@ -19,9 +18,29 @@ bridges:
     replicas: 2
   # Cms metadata bridge to pub cluster
   - name: cms-metadata-kafka-bridge-pub
-    sourceKafkaProxyUrl: "https://upp-k8s-dev-publish-eu.upp.ft.com" # use the pub cluster url
+    sourceKafkaProxyUrl: "https://upp-k8s-dev-publish-eu.upp.ft.com/__kafka-rest-proxy" # use the pub cluster url
     groupIdPrefix: metadata-kafka-bridge-pub
     topic: NativeCmsMetadataPublicationEvents
+    producer: "http://kafka-rest-proxy:8080"
+    authSecretName: pub-auth
+    authSecretKey: pub-k8s-auth
+    type: "proxy"
+    replicas: 2
+  # Bridge to MSK
+  - name: kafka-to-msk-bridge
+    sourceKafkaProxyUrl: "http://kafka-rest-proxy:8080"
+    groupIdPrefix: kafka-msk-bridge
+    topic: CombinedPostPublicationEvents
+    producer: "http://kafka-rest-proxy-msk:8080"
+    authSecretName: pub-auth
+    authSecretKey: pub-k8s-auth
+    type: "proxy"
+    replicas: 2
+  - name: msk-to-kafka-bridge
+    sourceKafkaProxyUrl: "http://kafka-rest-proxy-msk:8080"
+    groupIdPrefix: kafka-msk-bridge
+    topic: NativeCmsPublicationEvents
+    producer: "http://kafka-rest-proxy:8080"
     authSecretName: pub-auth
     authSecretKey: pub-k8s-auth
     type: "proxy"

helm/coco-kafka-bridge/app-configs/kafka-bridge_eks_delivery_prod_eu.yaml

@@ -13,7 +13,7 @@ bridges:
 ## Begin prod bridges
 #CMS
 - name: cms-kafka-bridge-pub-prod-eu
-  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-prod-eu
   topic: NativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -22,7 +22,7 @@ bridges:
   type: "plainHTTP"
   replicas: 2
 - name: cms-kafka-bridge-pub-prod-us
-  sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-prod-us
   topic: NativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -32,15 +32,15 @@ bridges:
   replicas: 2
 #CMS metadata
 - name: cms-metadata-kafka-bridge-pub-prod-eu
-  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-prod-eu
   topic: NativeCmsMetadataPublicationEvents
   authSecretName: pub-auth
   authSecretKey: pub-prod-auth
   type: "proxy"
   replicas: 2
 - name: cms-metadata-kafka-bridge-pub-prod-us
-  sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-prod-us
   topic: NativeCmsMetadataPublicationEvents
   authSecretName: pub-auth

helm/coco-kafka-bridge/app-configs/kafka-bridge_eks_delivery_prod_us.yaml

@@ -13,7 +13,7 @@ bridges:
 ## Begin prod bridges
 #CMS
 - name: cms-kafka-bridge-pub-prod-eu
-  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-prod-eu
   topic: NativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -22,7 +22,7 @@ bridges:
   type: "plainHTTP"
   replicas: 2
 - name: cms-kafka-bridge-pub-prod-us
-  sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-prod-us
   topic: NativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -32,15 +32,15 @@ bridges:
   replicas: 2
 #CMS metadata
 - name: cms-metadata-kafka-bridge-pub-prod-eu
-  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-prod-eu
   topic: NativeCmsMetadataPublicationEvents
   authSecretName: pub-auth
   authSecretKey: pub-prod-auth
   type: "proxy"
   replicas: 2
 - name: cms-metadata-kafka-bridge-pub-prod-us
-  sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-prod-us
   topic: NativeCmsMetadataPublicationEvents
   authSecretName: pub-auth

helm/coco-kafka-bridge/app-configs/kafka-bridge_eks_delivery_staging_eu.yaml

@@ -13,7 +13,7 @@ bridges:
 ## Start staging bridges
 ##CMS
 - name: cms-kafka-bridge-pub-staging-eu
-  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-staging-eu
   topic: NativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -22,7 +22,7 @@ bridges:
   type: "plainHTTP"
   replicas: 2
 - name: cms-kafka-bridge-pub-staging-us
-  sourceKafkaProxyUrl: "https://upp-staging-publish-us.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-us.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-staging-us
   topic: NativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -32,15 +32,15 @@ bridges:
   replicas: 2
 #CMS metadata
 - name: cms-metadata-kafka-bridge-pub-staging-eu
-  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-staging-eu
   topic: NativeCmsMetadataPublicationEvents
   authSecretName: pub-auth
   authSecretKey: pub-staging-auth
   type: "proxy"
   replicas: 2
 - name: cms-metadata-kafka-bridge-pub-staging-us
-  sourceKafkaProxyUrl: "https://upp-staging-publish-us.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-us.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-staging-us
   topic: NativeCmsMetadataPublicationEvents
   authSecretName: pub-auth

helm/coco-kafka-bridge/app-configs/kafka-bridge_eks_delivery_staging_us.yaml

@@ -13,7 +13,7 @@ bridges:
 ## Start staging bridges
 ##CMS
 - name: cms-kafka-bridge-pub-staging-eu
-  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-staging-eu
   topic: NativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -22,7 +22,7 @@ bridges:
   type: "plainHTTP"
   replicas: 2
 - name: cms-kafka-bridge-pub-staging-us
-  sourceKafkaProxyUrl: "https://upp-staging-publish-us.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-us.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-staging-us
   topic: NativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -32,15 +32,15 @@ bridges:
   replicas: 2
 #CMS metadata
 - name: cms-metadata-kafka-bridge-pub-staging-eu
-  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-staging-eu
   topic: NativeCmsMetadataPublicationEvents
   authSecretName: pub-auth
   authSecretKey: pub-staging-auth
   type: "proxy"
   replicas: 2
 - name: cms-metadata-kafka-bridge-pub-staging-us
-  sourceKafkaProxyUrl: "https://upp-staging-publish-us.upp.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-us.upp.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-staging-us
   topic: NativeCmsMetadataPublicationEvents
   authSecretName: pub-auth

helm/coco-kafka-bridge/app-configs/kafka-bridge_eks_publish_dev.yaml

@@ -10,7 +10,7 @@ pubAauth:
 bridges:
 ## Start staging bridges
 - name: cms-kafka-bridge-pub-staging-eu
-  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-staging-eu
   topic: PreNativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -19,7 +19,7 @@ bridges:
   type: "plainHTTP"
   replicas: 2
 - name: cms-kafka-bridge-pub-staging-us
-  sourceKafkaProxyUrl: "https://upp-staging-publish-us.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-us.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-staging-us
   topic: PreNativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -29,7 +29,7 @@ bridges:
   replicas: 2
 #CMS metadata
 - name: cms-metadata-kafka-bridge-pub-staging-eu
-  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-eu.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-staging-eu
   topic: PreNativeCmsMetadataPublicationEvents
   producer: "http://cms-metadata-notifier:8080"
@@ -38,7 +38,7 @@ bridges:
   type: "plainHTTP"
   replicas: 2
 - name: cms-metadata-kafka-bridge-pub-staging-us
-  sourceKafkaProxyUrl: "https://upp-staging-publish-us.ft.com"
+  sourceKafkaProxyUrl: "https://upp-staging-publish-us.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-staging-us
   topic: PreNativeCmsMetadataPublicationEvents
   producer: "http://cms-metadata-notifier:8080"
@@ -50,7 +50,7 @@ bridges:
 ## Begin prod bridges
 #CMS
 - name: cms-kafka-bridge-pub-prod-eu
-  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-prod-eu
   topic: PreNativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -59,7 +59,7 @@ bridges:
   type: "plainHTTP"
   replicas: 2
 - name: cms-kafka-bridge-pub-prod-us
-  sourceKafkaProxyUrl: "https://upp-prod-publish-us.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-us.ft.com/__kafka-rest-proxy"
   groupIdPrefix: kafka-bridge-pub-prod-us
   topic: PreNativeCmsPublicationEvents
   producer: "http://cms-notifier:8080"
@@ -69,7 +69,7 @@ bridges:
   replicas: 2
 #CMS metadata
 - name: cms-metadata-kafka-bridge-pub-prod-eu
-  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-eu.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-prod-eu
   topic: PreNativeCmsMetadataPublicationEvents
   producer: "http://cms-metadata-notifier:8080"
@@ -78,7 +78,7 @@ bridges:
   type: "plainHTTP"
   replicas: 2
 - name: cms-metadata-kafka-bridge-pub-prod-us
-  sourceKafkaProxyUrl: "https://upp-prod-publish-us.ft.com"
+  sourceKafkaProxyUrl: "https://upp-prod-publish-us.ft.com/__kafka-rest-proxy"
   groupIdPrefix: metadata-kafka-bridge-pub-prod-us
   topic: PreNativeCmsMetadataPublicationEvents
   producer: "http://cms-metadata-notifier:8080"

helm/coco-kafka-bridge/app-configs/kafka-bridge_eks_publish_staging_eu.yaml

@@ -13,7 +13,7 @@ bridges:
   # todo remove all bridges once we are 100% live and methode publishes directly to this staging.
   ##CMS
   - name: cms-kafka-bridge-pub-prod-eu
-    sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com"
+    sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com/__kafka-rest-proxy"
     groupIdPrefix: kafka-bridge-pub-prod-eu
     topic: PreNativeCmsPublicationEvents
     producer: "http://cms-notifier:8080"
@@ -23,7 +23,7 @@ bridges:
     replicas: 2
   #CMS metadata
   - name: cms-metadata-kafka-bridge-pub-prod-eu
-    sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com"
+    sourceKafkaProxyUrl: "https://upp-prod-publish-eu.upp.ft.com/__kafka-rest-proxy"
     groupIdPrefix: metadata-kafka-bridge-pub-prod-eu
     topic: PreNativeCmsMetadataPublicationEvents
     producer: "http://cms-metadata-notifier:8080"

helm/coco-kafka-bridge/app-configs/kafka-bridge_eks_publish_staging_us.yaml

@@ -13,7 +13,7 @@ bridges:
   # todo remove all bridges once we are 100% live and methode publishes directly to this staging.
   ##CMS
   - name: cms-kafka-bridge-pub-prod-us
-    sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com"
+    sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com/__kafka-rest-proxy"
     groupIdPrefix: kafka-bridge-pub-prod-us
     topic: PreNativeCmsPublicationEvents
     producer: "http://cms-notifier:8080"
@@ -23,7 +23,7 @@ bridges:
     replicas: 2
   #CMS metadata
   - name: cms-metadata-kafka-bridge-pub-prod-us
-    sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com"
+    sourceKafkaProxyUrl: "https://upp-prod-publish-us.upp.ft.com/__kafka-rest-proxy"
     groupIdPrefix: metadata-kafka-bridge-pub-prod-us
     topic: PreNativeCmsMetadataPublicationEvents
     producer: "http://cms-metadata-notifier:8080"
@@ -33,4 +33,3 @@ bridges:
     replicas: 2
 
   ## END staging bridges
-

helm/coco-kafka-bridge/templates/deployment.yaml

@@ -43,25 +43,19 @@ spec:
         env:
         - name: SERVICE_NAME
           value: "{{ $bridge.name }}"
+        - name: PRODUCER_ADDRESS
+          value: "{{ $bridge.producer }}"
 {{- if eq $bridge.type "proxy" }}
         - name: CONSUMER_AUTOCOMMIT_ENABLE
           value: "true"
-        - name: PRODUCER_ADDRESS
-          valueFrom:
-            configMapKeyRef:
-              name: global-config
-              key: kafka.proxy.url.with.protocol
 {{- else }}
-        - name: PRODUCER_ADDRESS
-          value: "{{ $bridge.producer }}"
         - name: CONSUMER_AUTOCOMMIT_ENABLE
           value: "false"
 {{- end }}
         - name: PRODUCER_TYPE
           value: "{{ $bridge.type }}"
-{{- $proxyUrlValue := $bridge.sourceKafkaProxyUrl }}
         - name: QUEUE_PROXY_ADDRS
-          value: "{{ $proxyUrlValue }}/__kafka-rest-proxy"
+          value: "{{ $bridge.sourceKafkaProxyUrl }}"
         - name: GROUP_ID
           value: {{ $bridge.groupIdPrefix }}-{{ template "env-full-name" $global }}
         - name: TOPIC