diff --git a/docker/kubernetes/README.md b/docker/kubernetes/README.md index e877a370e3a..3d946a973ce 100644 --- a/docker/kubernetes/README.md +++ b/docker/kubernetes/README.md @@ -4,10 +4,16 @@ - Kubernetes 1.10+ - Helm 3.0+ +- [Flink 1.13.5](https://nightlies.apache.org/flink/flink-docs-release-1.13/docs/try-flink/local_installation/) +- [Apache Pulsar 2.8.x](https://pulsar.apache.org/docs/en/2.8.1/standalone/) - A dynamic provisioner for the PersistentVolumes(`production environment`) ## Usage +### Configure +- configure the `external.pulsar` for Apache Pulsar info in `values.yaml` +- configure the `external.flink` for Apache Flink info in `values.yaml` + ### Install If the namespace named `inlong` does not exist, create it first by running: @@ -98,7 +104,6 @@ The configuration file is [values.yaml](values.yaml), and the following tables l | `external.mysql.port` | `3306` | External MySQL port | | `external.mysql.username` | `root` | External MySQL username | | `external.mysql.password` | `password` | External MySQL password | -| `external.pulsar.enabled` | `false` | If not exists external Pulsar, InLong will use the internal TubeMQ by default | | `external.pulsar.serviceUrl` | `localhost:6650` | External Pulsar service URL | | `external.pulsar.adminUrl` | `localhost:8080` | External Pulsar admin URL | diff --git a/docker/kubernetes/templates/audit-statefulset.yaml b/docker/kubernetes/templates/audit-statefulset.yaml index ef2be510521..5a1a0b40ac4 100644 --- a/docker/kubernetes/templates/audit-statefulset.yaml +++ b/docker/kubernetes/templates/audit-statefulset.yaml @@ -116,14 +116,24 @@ spec: secretKeyRef: name: {{ template "inlong.fullname" . }}-{{ .Values.mysql.component }} key: mysql-password + {{ if eq .Values.dataproxy.mqType "pulsar" }} + - name: MQ_TYPE + value: "pulsar" + - name: PULSAR_BROKER_LIST + value: {{ .Values.external.pulsar.serviceUrl }} + - name: PULSAR_AUDIT_TOPIC + value: "{{ .Values.external.pulsar.auditTopicName }}" + {{- end }} + {{ if eq .Values.dataproxy.mqType "tubemq" }} + - name: MQ_TYPE + value: "tubemq" + - name: TUBE_MASTER_LIST + value: "{{ template "inlong.tubemqMaster.hostname" . }}:{{ .Values.tubemqMaster.ports.rpcPort }}" + {{- end }} - name: MANAGER_OPENAPI_IP value: {{ include "inlong.manager.hostname" . | quote }} - name: MANAGER_OPENAPI_PORT value: {{ .Values.manager.port | quote }} - - name: PULSAR_BROKER_LIST - value: {{ .Values.external.pulsar.serviceUrl | quote }} - - name: TUBE_MASTER_LIST - value: "{{ template "inlong.tubemqMaster.hostname" . }}:{{ .Values.tubemqMaster.ports.rpcPort }}" {{- range $key, $value := .Values.audit.env }} - name: {{ $key }} value: {{ $value | quote }} diff --git a/docker/kubernetes/templates/tubemq-broker-configmap.yaml b/docker/kubernetes/templates/tubemq-broker-configmap.yaml index 84e39426d79..97af397fcd9 100644 --- a/docker/kubernetes/templates/tubemq-broker-configmap.yaml +++ b/docker/kubernetes/templates/tubemq-broker-configmap.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqServer }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/docker/kubernetes/templates/tubemq-broker-ini-configmap.yaml b/docker/kubernetes/templates/tubemq-broker-ini-configmap.yaml index aff65fe4bbe..df3e86c1f0e 100644 --- a/docker/kubernetes/templates/tubemq-broker-ini-configmap.yaml +++ b/docker/kubernetes/templates/tubemq-broker-ini-configmap.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqServer }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/docker/kubernetes/templates/tubemq-broker-pdb.yaml b/docker/kubernetes/templates/tubemq-broker-pdb.yaml index 7b41d0eb930..225e63b3baf 100644 --- a/docker/kubernetes/templates/tubemq-broker-pdb.yaml +++ b/docker/kubernetes/templates/tubemq-broker-pdb.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqServer }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} {{- if .Values.tubemqBroker.pdb.usePolicy }} {{- if .Capabilities.APIVersions.Has "policy/v1" }} apiVersion: policy/v1 diff --git a/docker/kubernetes/templates/tubemq-broker-service.yaml b/docker/kubernetes/templates/tubemq-broker-service.yaml index 785e9a1551e..e466be6e5d9 100644 --- a/docker/kubernetes/templates/tubemq-broker-service.yaml +++ b/docker/kubernetes/templates/tubemq-broker-service.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqServer }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: v1 kind: Service metadata: diff --git a/docker/kubernetes/templates/tubemq-broker-statefulset.yaml b/docker/kubernetes/templates/tubemq-broker-statefulset.yaml index 0666815fd29..7201b18cddb 100644 --- a/docker/kubernetes/templates/tubemq-broker-statefulset.yaml +++ b/docker/kubernetes/templates/tubemq-broker-statefulset.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqServer }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/docker/kubernetes/templates/tubemq-manager-service.yaml b/docker/kubernetes/templates/tubemq-manager-service.yaml index b284bf7ce57..e02f6840ffe 100644 --- a/docker/kubernetes/templates/tubemq-manager-service.yaml +++ b/docker/kubernetes/templates/tubemq-manager-service.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqManager }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: v1 kind: Service metadata: diff --git a/docker/kubernetes/templates/tubemq-manager-statefulset.yaml b/docker/kubernetes/templates/tubemq-manager-statefulset.yaml index 2042622fcf4..bb26f81f901 100644 --- a/docker/kubernetes/templates/tubemq-manager-statefulset.yaml +++ b/docker/kubernetes/templates/tubemq-manager-statefulset.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqManager }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/docker/kubernetes/templates/tubemq-master-configmap.yaml b/docker/kubernetes/templates/tubemq-master-configmap.yaml index 644e76de319..ac804bc5d8c 100644 --- a/docker/kubernetes/templates/tubemq-master-configmap.yaml +++ b/docker/kubernetes/templates/tubemq-master-configmap.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqServer }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/docker/kubernetes/templates/tubemq-master-ini-configmap.yaml b/docker/kubernetes/templates/tubemq-master-ini-configmap.yaml index 019d8da7b74..86ad1f4d45e 100644 --- a/docker/kubernetes/templates/tubemq-master-ini-configmap.yaml +++ b/docker/kubernetes/templates/tubemq-master-ini-configmap.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqServer }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/docker/kubernetes/templates/tubemq-master-pdb.yaml b/docker/kubernetes/templates/tubemq-master-pdb.yaml index 94f145a72dd..4e19a16cd0d 100644 --- a/docker/kubernetes/templates/tubemq-master-pdb.yaml +++ b/docker/kubernetes/templates/tubemq-master-pdb.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqServer }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} {{- if .Values.tubemqMaster.pdb.usePolicy }} {{- if .Capabilities.APIVersions.Has "policy/v1" }} apiVersion: policy/v1 diff --git a/docker/kubernetes/templates/tubemq-master-service.yaml b/docker/kubernetes/templates/tubemq-master-service.yaml index 26607c9645a..1bb4ae25189 100644 --- a/docker/kubernetes/templates/tubemq-master-service.yaml +++ b/docker/kubernetes/templates/tubemq-master-service.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqServer }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: v1 kind: Service metadata: diff --git a/docker/kubernetes/templates/tubemq-master-statefulset.yaml b/docker/kubernetes/templates/tubemq-master-statefulset.yaml index 63115563ccb..fd93b2b76ab 100644 --- a/docker/kubernetes/templates/tubemq-master-statefulset.yaml +++ b/docker/kubernetes/templates/tubemq-master-statefulset.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -{{- if .Values.components.tubemqServer }} +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/docker/kubernetes/templates/zookeeper-configmap.yaml b/docker/kubernetes/templates/zookeeper-configmap.yaml index 0452a502904..f8ffb77ea39 100644 --- a/docker/kubernetes/templates/zookeeper-configmap.yaml +++ b/docker/kubernetes/templates/zookeeper-configmap.yaml @@ -15,7 +15,8 @@ # limitations under the License. # -{{- if .Values.components.zookeeper }} +# only TubeMQ need zookeeper +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/docker/kubernetes/templates/zookeeper-pdb.yaml b/docker/kubernetes/templates/zookeeper-pdb.yaml index 083d6563781..94dc9d7d12a 100644 --- a/docker/kubernetes/templates/zookeeper-pdb.yaml +++ b/docker/kubernetes/templates/zookeeper-pdb.yaml @@ -15,7 +15,8 @@ # limitations under the License. # -{{- if .Values.components.zookeeper }} +# only TubeMQ need zookeeper +{{ if eq .Values.dataproxy.mqType "tubemq" }} {{- if .Values.zookeeper.pdb.usePolicy }} {{- if .Capabilities.APIVersions.Has "policy/v1" }} apiVersion: policy/v1 diff --git a/docker/kubernetes/templates/zookeeper-service.yaml b/docker/kubernetes/templates/zookeeper-service.yaml index 23352f37362..6f6e006bfcb 100644 --- a/docker/kubernetes/templates/zookeeper-service.yaml +++ b/docker/kubernetes/templates/zookeeper-service.yaml @@ -15,7 +15,8 @@ # limitations under the License. # -{{- if .Values.components.zookeeper }} +# only TubeMQ need zookeeper +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: v1 kind: Service metadata: diff --git a/docker/kubernetes/templates/zookeeper-statefulset.yaml b/docker/kubernetes/templates/zookeeper-statefulset.yaml index 4d08a5070c4..cad0c0ca476 100644 --- a/docker/kubernetes/templates/zookeeper-statefulset.yaml +++ b/docker/kubernetes/templates/zookeeper-statefulset.yaml @@ -15,7 +15,8 @@ # limitations under the License. # -{{- if .Values.components.zookeeper }} +# only TubeMQ need zookeeper +{{ if eq .Values.dataproxy.mqType "tubemq" }} apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/docker/kubernetes/values.yaml b/docker/kubernetes/values.yaml index 2e13432fd63..ca41b97aa86 100644 --- a/docker/kubernetes/values.yaml +++ b/docker/kubernetes/values.yaml @@ -21,12 +21,9 @@ components: agent: true dashboard: true dataproxy: true - tubemqManager: true - tubemqServer: true manager: true audit: true mysql: true - zookeeper: true images: agent: @@ -59,7 +56,7 @@ images: initContainer: repository: busybox tag: latest - pullPolicy: "IfNotPresent" + pullPolicy: "Always" ingress: enabled: false @@ -170,6 +167,8 @@ dashboard: dataproxy: component: "dataproxy" + # option, tubemq/pulsar/kafka, pulsar as default. + mqType: "pulsar" replicas: 1 # The updateStrategy field allows you to configure and disable automated rolling updates for containers, labels, resource request/limits, and annotations for the Pods in a StatefulSet. # There are two possible values: OnDelete and RollingUpdate. @@ -658,20 +657,20 @@ tubemqBroker: -XX:MaxRAMPercentage=80.0 -XX:-UseAdaptiveSizePolicy -# If exists external MySQL or Pulsar, you can set the 'enable' field value to true and configure related information. external: + # If exists external MySQL, you can set the 'enable' field value to true and configure related information. mysql: enabled: false hostname: "localhost" port: 3306 username: "root" - password: "password" - # If there is no external Pulsar, InLong will use TubeMQ. + password: "inlong" + # If choose Pulsar as an MQ service for DataProxy, you need to configure the cluster address. pulsar: - enabled: false serviceUrl: "localhost:6650" adminUrl: "localhost:8080" + auditTopicName: "persistent://public/default/inlong-audit" + # the REST server address for Flink flink: - enabled: true hostname: "127.0.0.1" port: 8081