apache · dockerzhang · Feb 28, 2023 · Feb 27, 2023
diff --git a/docker/kubernetes/README.md b/docker/kubernetes/README.md
@@ -4,10 +4,16 @@
 
 - Kubernetes 1.10+
 - Helm 3.0+
+- [Flink 1.13.5](https://nightlies.apache.org/flink/flink-docs-release-1.13/docs/try-flink/local_installation/)
+- [Apache Pulsar 2.8.x](https://pulsar.apache.org/docs/en/2.8.1/standalone/)
 - A dynamic provisioner for the PersistentVolumes(`production environment`)
 
 ## Usage
 
+### Configure
+- configure the `external.pulsar` for Apache Pulsar info in `values.yaml`
+- configure the `external.flink` for Apache Flink info in `values.yaml`
+
 ### Install
 
 If the namespace named `inlong` does not exist, create it first by running:
@@ -98,7 +104,6 @@ The configuration file is [values.yaml](values.yaml), and the following tables l
 |                              `external.mysql.port`                               |      `3306`      |                                                                     External MySQL port                                                                      |
 |                            `external.mysql.username`                             |      `root`      |                                                                   External MySQL username                                                                    |
 |                            `external.mysql.password`                             |    `password`    |                                                                   External MySQL password                                                                    |
-|                            `external.pulsar.enabled`                             |     `false`      |                                        If not exists external Pulsar, InLong will use the internal TubeMQ by default                                         |
 |                           `external.pulsar.serviceUrl`                           | `localhost:6650` |                                                                 External Pulsar service URL                                                                  |
 |                            `external.pulsar.adminUrl`                            | `localhost:8080` |                                                                  External Pulsar admin URL                                                                   |
 

diff --git a/docker/kubernetes/templates/audit-statefulset.yaml b/docker/kubernetes/templates/audit-statefulset.yaml
@@ -116,14 +116,24 @@ spec:
                 secretKeyRef:
                   name: {{ template "inlong.fullname" . }}-{{ .Values.mysql.component }}
                   key: mysql-password
+            {{ if eq .Values.dataproxy.mqType "pulsar" }}
+            - name: MQ_TYPE
+              value: "pulsar"
+            - name: PULSAR_BROKER_LIST
+              value: {{ .Values.external.pulsar.serviceUrl }}
+            - name: PULSAR_AUDIT_TOPIC
+              value: "{{ .Values.external.pulsar.auditTopicName }}"
+            {{- end }}
+            {{ if eq .Values.dataproxy.mqType "tubemq" }}
+            - name: MQ_TYPE
+              value: "tubemq"
+            - name: TUBE_MASTER_LIST
+              value: "{{ template "inlong.tubemqMaster.hostname" . }}:{{ .Values.tubemqMaster.ports.rpcPort }}"
+            {{- end }}
             - name: MANAGER_OPENAPI_IP
               value: {{ include "inlong.manager.hostname" . | quote }}
             - name: MANAGER_OPENAPI_PORT
               value: {{ .Values.manager.port | quote }}
-            - name: PULSAR_BROKER_LIST
-              value: {{ .Values.external.pulsar.serviceUrl | quote }}
-            - name: TUBE_MASTER_LIST
-              value: "{{ template "inlong.tubemqMaster.hostname" . }}:{{ .Values.tubemqMaster.ports.rpcPort }}"
             {{- range $key, $value := .Values.audit.env }}
             - name: {{ $key }}
               value: {{ $value | quote }}

diff --git a/docker/kubernetes/templates/tubemq-broker-configmap.yaml b/docker/kubernetes/templates/tubemq-broker-configmap.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

diff --git a/docker/kubernetes/templates/tubemq-broker-ini-configmap.yaml b/docker/kubernetes/templates/tubemq-broker-ini-configmap.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

diff --git a/docker/kubernetes/templates/tubemq-broker-pdb.yaml b/docker/kubernetes/templates/tubemq-broker-pdb.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 {{- if .Values.tubemqBroker.pdb.usePolicy }}
 {{- if .Capabilities.APIVersions.Has "policy/v1" }}
 apiVersion: policy/v1

diff --git a/docker/kubernetes/templates/tubemq-broker-service.yaml b/docker/kubernetes/templates/tubemq-broker-service.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: v1
 kind: Service
 metadata:

diff --git a/docker/kubernetes/templates/tubemq-broker-statefulset.yaml b/docker/kubernetes/templates/tubemq-broker-statefulset.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:

diff --git a/docker/kubernetes/templates/tubemq-manager-service.yaml b/docker/kubernetes/templates/tubemq-manager-service.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqManager }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: v1
 kind: Service
 metadata:

diff --git a/docker/kubernetes/templates/tubemq-manager-statefulset.yaml b/docker/kubernetes/templates/tubemq-manager-statefulset.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqManager }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:

diff --git a/docker/kubernetes/templates/tubemq-master-configmap.yaml b/docker/kubernetes/templates/tubemq-master-configmap.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

diff --git a/docker/kubernetes/templates/tubemq-master-ini-configmap.yaml b/docker/kubernetes/templates/tubemq-master-ini-configmap.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

diff --git a/docker/kubernetes/templates/tubemq-master-pdb.yaml b/docker/kubernetes/templates/tubemq-master-pdb.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 {{- if .Values.tubemqMaster.pdb.usePolicy }}
 {{- if .Capabilities.APIVersions.Has "policy/v1" }}
 apiVersion: policy/v1

diff --git a/docker/kubernetes/templates/tubemq-master-service.yaml b/docker/kubernetes/templates/tubemq-master-service.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: v1
 kind: Service
 metadata:

diff --git a/docker/kubernetes/templates/tubemq-master-statefulset.yaml b/docker/kubernetes/templates/tubemq-master-statefulset.yaml
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:

diff --git a/docker/kubernetes/templates/zookeeper-configmap.yaml b/docker/kubernetes/templates/zookeeper-configmap.yaml
@@ -15,7 +15,8 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.zookeeper }}
+# only TubeMQ need zookeeper
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

diff --git a/docker/kubernetes/templates/zookeeper-pdb.yaml b/docker/kubernetes/templates/zookeeper-pdb.yaml
@@ -15,7 +15,8 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.zookeeper }}
+# only TubeMQ need zookeeper
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 {{- if .Values.zookeeper.pdb.usePolicy }}
 {{- if .Capabilities.APIVersions.Has "policy/v1" }}
 apiVersion: policy/v1

diff --git a/docker/kubernetes/templates/zookeeper-service.yaml b/docker/kubernetes/templates/zookeeper-service.yaml
@@ -15,7 +15,8 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.zookeeper }}
+# only TubeMQ need zookeeper
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: v1
 kind: Service
 metadata:

diff --git a/docker/kubernetes/templates/zookeeper-statefulset.yaml b/docker/kubernetes/templates/zookeeper-statefulset.yaml
@@ -15,7 +15,8 @@
 # limitations under the License.
 #
 
-{{- if .Values.components.zookeeper }}
+# only TubeMQ need zookeeper
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:

diff --git a/docker/kubernetes/values.yaml b/docker/kubernetes/values.yaml
@@ -21,12 +21,9 @@ components:
   agent: true
   dashboard: true
   dataproxy: true
-  tubemqManager: true
-  tubemqServer: true
   manager: true
   audit: true
   mysql: true
-  zookeeper: true
 
 images:
   agent:
@@ -59,7 +56,7 @@ images:
   initContainer:
     repository: busybox
     tag: latest
-  pullPolicy: "IfNotPresent"
+  pullPolicy: "Always"
 
 ingress:
   enabled: false
@@ -170,6 +167,8 @@ dashboard:
 
 dataproxy:
   component: "dataproxy"
+  # option, tubemq/pulsar/kafka, pulsar as default.
+  mqType: "pulsar"
   replicas: 1
   # The updateStrategy field allows you to configure and disable automated rolling updates for containers, labels, resource request/limits, and annotations for the Pods in a StatefulSet.
   # There are two possible values: OnDelete and RollingUpdate.
@@ -658,20 +657,20 @@ tubemqBroker:
       -XX:MaxRAMPercentage=80.0
       -XX:-UseAdaptiveSizePolicy
 
-# If exists external MySQL or Pulsar, you can set the 'enable' field value to true and configure related information.
 external:
+  # If exists external MySQL, you can set the 'enable' field value to true and configure related information.
   mysql:
     enabled: false
     hostname: "localhost"
     port: 3306
     username: "root"
-    password: "password"
-  # If there is no external Pulsar, InLong will use TubeMQ.
+    password: "inlong"
+  # If choose Pulsar as an MQ service for DataProxy, you need to configure the cluster address.
   pulsar:
-    enabled: false
     serviceUrl: "localhost:6650"
     adminUrl: "localhost:8080"
+    auditTopicName: "persistent://public/default/inlong-audit"
+  # the REST server address for Flink
   flink:
-    enabled: true
     hostname: "127.0.0.1"
     port: 8081