feast(eso): rename topic

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/buildindices/PostBuildIndicesStep.java

@@ -10,7 +10,7 @@
 import com.linkedin.metadata.search.elasticsearch.indexbuilder.ReindexConfig;
 import com.linkedin.metadata.shared.ElasticSearchIndexed;
 import com.linkedin.metadata.version.GitVersion;
-import com.linkedin.mxe.BuildIndicesHistoryEvent;
+import com.linkedin.mxe.DataHubUpgradeHistoryEvent;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
@@ -69,9 +69,9 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
           }
         }
 
-        BuildIndicesHistoryEvent buildIndicesHistoryEvent = new BuildIndicesHistoryEvent()
+        DataHubUpgradeHistoryEvent dataHubUpgradeHistoryEvent = new DataHubUpgradeHistoryEvent()
             .setVersion(_gitVersion.getVersion());
-        _kafkaEventProducer.produceBuildIndicesHistoryEvent(buildIndicesHistoryEvent);
+        _kafkaEventProducer.produceDataHubUpgradeHistoryEvent(dataHubUpgradeHistoryEvent);
       } catch (Exception e) {
         log.error("PostBuildIndicesStep failed.", e);
         return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED);

docker/datahub-gms/env/docker-without-neo4j.env

@@ -1,4 +1,4 @@
-BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
+DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
 EBEAN_DATASOURCE_USERNAME=datahub
 EBEAN_DATASOURCE_PASSWORD=datahub
 EBEAN_DATASOURCE_HOST=mysql:3306

docker/datahub-gms/env/docker.cassandra.env

@@ -1,4 +1,4 @@
-BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
+DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
 KAFKA_BOOTSTRAP_SERVER=broker:29092
 KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
 ELASTICSEARCH_HOST=elasticsearch

docker/datahub-gms/env/docker.env

@@ -1,4 +1,4 @@
-BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
+DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
 EBEAN_DATASOURCE_USERNAME=datahub
 EBEAN_DATASOURCE_PASSWORD=datahub
 EBEAN_DATASOURCE_HOST=mysql:3306

docker/datahub-gms/env/docker.mariadb.env

@@ -1,4 +1,4 @@
-BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
+DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
 EBEAN_DATASOURCE_USERNAME=datahub
 EBEAN_DATASOURCE_PASSWORD=datahub
 EBEAN_DATASOURCE_HOST=mariadb:3306

docker/datahub-gms/env/docker.postgres.env

@@ -1,4 +1,4 @@
-BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
+DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
 EBEAN_DATASOURCE_USERNAME=datahub
 EBEAN_DATASOURCE_PASSWORD=datahub
 EBEAN_DATASOURCE_HOST=postgres:5432

docker/datahub-mae-consumer/env/docker-without-neo4j.env

@@ -1,4 +1,4 @@
-BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-mcl
+DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-mcl
 DATAHUB_GMS_HOST=datahub-gms
 DATAHUB_GMS_PORT=8080
 

docker/datahub-mae-consumer/env/docker.env

@@ -1,4 +1,4 @@
-BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-mcl
+DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-mcl
 DATAHUB_GMS_HOST=datahub-gms
 DATAHUB_GMS_PORT=8080
 

docker/kafka-setup/Dockerfile

@@ -56,7 +56,7 @@ ENV METADATA_CHANGE_LOG_TIMESERIES_TOPIC="MetadataChangeLog_Timeseries_v1"
 ENV METADATA_CHANGE_PROPOSAL_TOPIC="MetadataChangeProposal_v1"
 ENV FAILED_METADATA_CHANGE_PROPOSAL_TOPIC="FailedMetadataChangeProposal_v1"
 ENV PLATFORM_EVENT_TOPIC_NAME="PlatformEvent_v1"
-ENV BUILD_INDICES_HISTORY_TOPIC="BuildIndicesHistory_v1"
+ENV DATAHUB_UPGRADE_HISTORY_TOPIC="DataHubUpgradeHistory_v1"
 
 COPY docker/kafka-setup/kafka-setup.sh ./kafka-setup.sh
 COPY docker/kafka-setup/kafka-config.sh ./kafka-config.sh

docker/kafka-setup/env/docker.env

@@ -8,7 +8,7 @@ KAFKA_BOOTSTRAP_SERVER=broker:29092
 # FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4
 # PLATFORM_EVENT_TOPIC_NAME=PlatformEvent_v1
 # DATAHUB_USAGE_EVENT_NAME=DataHubUsageEvent_v1
-# BUILD_INDICES_HISTORY_TOPIC=BuildIndicesHistory_v1
+# DATAHUB_UPGRADE_HISTORY_TOPIC=DataHubUpgradeHistory_v1
 # PARTITIONS=1
 # REPLICATION_FACTOR=1
 

docker/kafka-setup/kafka-setup.sh

@@ -108,6 +108,8 @@ send "$METADATA_CHANGE_PROPOSAL_TOPIC" "--topic $METADATA_CHANGE_PROPOSAL_TOPIC"
 send "$FAILED_METADATA_CHANGE_PROPOSAL_TOPIC" "--topic $FAILED_METADATA_CHANGE_PROPOSAL_TOPIC"
 send "$PLATFORM_EVENT_TOPIC_NAME" "--topic $PLATFORM_EVENT_TOPIC_NAME"
 
+# Infinite retention upgrade topic
+send "$DATAHUB_UPGRADE_HISTORY_TOPIC" "config retention.ms=-1 --topic $DATAHUB_UPGRADE_HISTORY_TOPIC"
 # Create topic for datahub usage event
 if [[ $DATAHUB_ANALYTICS_ENABLED == true ]]; then
   send "$DATAHUB_USAGE_EVENT_NAME" "--topic $DATAHUB_USAGE_EVENT_NAME"

docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml

@@ -74,7 +74,7 @@ services:
     depends_on:
     - mysql
     environment:
-    - BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
+    - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
     - EBEAN_DATASOURCE_USERNAME=datahub
     - EBEAN_DATASOURCE_PASSWORD=datahub
     - EBEAN_DATASOURCE_HOST=mysql:3306

docker/quickstart/docker-compose-without-neo4j.quickstart.yml

@@ -76,7 +76,7 @@ services:
     environment:
     - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
     - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
-    - BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
+    - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
     - EBEAN_DATASOURCE_USERNAME=datahub
     - EBEAN_DATASOURCE_PASSWORD=datahub
     - EBEAN_DATASOURCE_HOST=mysql:3306

docker/quickstart/docker-compose.consumers-without-neo4j.quickstart.yml

@@ -6,7 +6,7 @@ services:
   datahub-mae-consumer:
     container_name: datahub-mae-consumer
     environment:
-    - BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-mcl
+    - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-mcl
     - DATAHUB_GMS_HOST=datahub-gms
     - DATAHUB_GMS_PORT=8080
     - MAE_CONSUMER_ENABLED=true

docker/quickstart/docker-compose.consumers.quickstart.yml

@@ -8,7 +8,7 @@ services:
     depends_on:
     - neo4j
     environment:
-    - BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-mcl
+    - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-mcl
     - DATAHUB_GMS_HOST=datahub-gms
     - DATAHUB_GMS_PORT=8080
     - MAE_CONSUMER_ENABLED=true

docker/quickstart/docker-compose.quickstart.yml

@@ -78,7 +78,7 @@ services:
     environment:
     - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
     - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
-    - BUILD_INDICES_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
+    - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-bihe-consumer-job-client-gms
     - EBEAN_DATASOURCE_USERNAME=datahub
     - EBEAN_DATASOURCE_PASSWORD=datahub
     - EBEAN_DATASOURCE_HOST=mysql:3306

docker/quickstart/temp-without-neo4j.quickstart.yml

@@ -0,0 +1,186 @@
+networks:
+  default:
+    name: datahub_network
+services:
+  broker:
+    container_name: broker
+    depends_on:
+    - zookeeper
+    environment:
+    - KAFKA_BROKER_ID=1
+    - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+    - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+    - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
+    - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+    - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
+    - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
+    hostname: broker
+    image: confluentinc/cp-kafka:5.4.0
+    ports:
+    - 29092:29092
+    - 9092:9092
+  datahub-actions:
+    depends_on:
+    - datahub-gms
+    environment:
+    - GMS_HOST=datahub-gms
+    - GMS_PORT=8080
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - SCHEMA_REGISTRY_URL=http://schema-registry:8081
+    - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
+    - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
+    - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
+    - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
+    - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
+    hostname: actions
+    image: acryldata/datahub-actions:${ACTIONS_VERSION:-head}
+    restart: on-failure:5
+  datahub-frontend-react:
+    container_name: datahub-frontend-react
+    depends_on:
+    - datahub-gms
+    environment:
+    - DATAHUB_GMS_HOST=datahub-gms
+    - DATAHUB_GMS_PORT=8080
+    - DATAHUB_SECRET=YouKnowNothing
+    - DATAHUB_APP_VERSION=1.0
+    - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
+    - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
+      -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
+      -Dlogback.debug=false -Dpidfile.path=/dev/null
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
+    - ELASTIC_CLIENT_HOST=elasticsearch
+    - ELASTIC_CLIENT_PORT=9200
+    hostname: datahub-frontend-react
+    image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head}
+    ports:
+    - 9002:9002
+    volumes:
+    - ${HOME}/.datahub/plugins:/etc/datahub/plugins
+  datahub-gms:
+    container_name: datahub-gms
+    depends_on:
+    - mysql
+    environment:
+    - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
+    - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
+    - DATASET_ENABLE_SCSI=false
+    - EBEAN_DATASOURCE_USERNAME=datahub
+    - EBEAN_DATASOURCE_PASSWORD=datahub
+    - EBEAN_DATASOURCE_HOST=mysql:3306
+    - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
+    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
+    - ELASTICSEARCH_HOST=elasticsearch
+    - ELASTICSEARCH_PORT=9200
+    - GRAPH_SERVICE_IMPL=elasticsearch
+    - JAVA_OPTS=-Xms1g -Xmx1g
+    - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
+    - MAE_CONSUMER_ENABLED=true
+    - MCE_CONSUMER_ENABLED=true
+    - PE_CONSUMER_ENABLED=true
+    - UI_INGESTION_ENABLED=true
+    - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.35
+    hostname: datahub-gms
+    image: linkedin/datahub-gms:${DATAHUB_VERSION:-head}
+    ports:
+    - 8080:8080
+    volumes:
+    - ${HOME}/.datahub/plugins:/etc/datahub/plugins
+  elasticsearch:
+    container_name: elasticsearch
+    environment:
+    - discovery.type=single-node
+    - xpack.security.enabled=false
+    - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true
+    healthcheck:
+      retries: 4
+      start_period: 2m
+      test:
+      - CMD-SHELL
+      - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s'
+        || exit 1
+    hostname: elasticsearch
+    image: elasticsearch:7.9.3
+    mem_limit: 1g
+    ports:
+    - 9200:9200
+    volumes:
+    - esdata:/usr/share/elasticsearch/data
+  elasticsearch-setup:
+    container_name: elasticsearch-setup
+    depends_on:
+    - elasticsearch
+    environment:
+    - ELASTICSEARCH_HOST=elasticsearch
+    - ELASTICSEARCH_PORT=9200
+    - ELASTICSEARCH_PROTOCOL=http
+    hostname: elasticsearch-setup
+    image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head}
+  kafka-setup:
+    container_name: kafka-setup
+    depends_on:
+    - broker
+    - schema-registry
+    environment:
+    - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+    - KAFKA_BOOTSTRAP_SERVER=broker:29092
+    hostname: kafka-setup
+    image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head}
+  mysql:
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
+    container_name: mysql
+    environment:
+    - MYSQL_DATABASE=datahub
+    - MYSQL_USER=datahub
+    - MYSQL_PASSWORD=datahub
+    - MYSQL_ROOT_PASSWORD=datahub
+    hostname: mysql
+    image: mysql:5.7
+    ports:
+    - 3306:3306
+    volumes:
+    - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
+    - mysqldata:/var/lib/mysql
+  mysql-setup:
+    container_name: mysql-setup
+    depends_on:
+    - mysql
+    environment:
+    - MYSQL_HOST=mysql
+    - MYSQL_PORT=3306
+    - MYSQL_USERNAME=datahub
+    - MYSQL_PASSWORD=datahub
+    - DATAHUB_DB_NAME=datahub
+    hostname: mysql-setup
+    image: acryldata/datahub-mysql-setup:head
+  schema-registry:
+    container_name: schema-registry
+    depends_on:
+    - zookeeper
+    - broker
+    environment:
+    - SCHEMA_REGISTRY_HOST_NAME=schemaregistry
+    - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
+    hostname: schema-registry
+    image: confluentinc/cp-schema-registry:5.4.0
+    ports:
+    - 8081:8081
+  zookeeper:
+    container_name: zookeeper
+    environment:
+    - ZOOKEEPER_CLIENT_PORT=2181
+    - ZOOKEEPER_TICK_TIME=2000
+    hostname: zookeeper
+    image: confluentinc/cp-zookeeper:5.4.0
+    ports:
+    - 2181:2181
+    volumes:
+    - zkdata:/var/opt/zookeeper
+version: '2.3'
+volumes:
+  esdata: null
+  mysqldata: null
+  zkdata: null

docker/quickstart/temp.monitoring.quickstart.yml

@@ -0,0 +1,37 @@
+services:
+  datahub-gms:
+    environment:
+    - ENABLE_PROMETHEUS=true
+    - ENABLE_OTEL=true
+    - OTEL_TRACES_EXPORTER=jaeger
+    - OTEL_EXPORTER_JAEGER_ENDPOINT=http://jaeger-all-in-one:14250
+    - OTEL_METRICS_EXPORTER=none
+    - OTEL_SERVICE_NAME=datahub-gms
+    ports:
+    - '4318'
+  grafana:
+    depends_on:
+    - prometheus
+    image: grafana/grafana:latest
+    ports:
+    - 3001:3000
+    volumes:
+    - grafana-storage:/var/lib/grafana
+    - ../monitoring/grafana/datasources:/etc/grafana/provisioning/datasources
+    - ../monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
+  jaeger-all-in-one:
+    image: jaegertracing/all-in-one:latest
+    ports:
+    - 16686:16686
+    - '14268'
+    - '14250'
+  prometheus:
+    container_name: prometheus
+    image: prom/prometheus:latest
+    ports:
+    - 9089:9090
+    volumes:
+    - ../monitoring/prometheus.yaml:/etc/prometheus/prometheus.yml
+version: '2.3'
+volumes:
+  grafana-storage: null