From d9c51e9dd7780f41ede915779468a60f280ae160 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 23 Apr 2020 11:21:25 +0800 Subject: [PATCH 1/5] Update end-to-end test config --- infra/scripts/test-end-to-end-batch.sh | 82 +++++++++++++------------- infra/scripts/test-end-to-end.sh | 68 +++++++++------------ 2 files changed, 68 insertions(+), 82 deletions(-) diff --git a/infra/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh index 35553a92814..db23e86d704 100755 --- a/infra/scripts/test-end-to-end-batch.sh +++ b/infra/scripts/test-end-to-end-batch.sh @@ -115,13 +115,17 @@ grpc: enable-reflection: true feast: - version: 0.3 jobs: - runner: DirectRunner - options: {} - updates: - pollingIntervalMillis: 30000 - timeoutSeconds: 240 + polling_interval_milliseconds: 30000 + job_update_timeout_seconds: 240 + + active_runner: direct + + runners: + - name: direct + type: DirectRunner + options: {} + metrics: enabled: false @@ -137,21 +141,15 @@ spring: jpa: properties.hibernate: format_sql: true - event.merge.entity_copy_observer: allow + event: + merge: + entity_copy_observer: allow hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl hibernate.ddl-auto: update datasource: url: jdbc:postgresql://localhost:5432/postgres username: postgres password: password - -management: - metrics: - export: - simple: - enabled: false - statsd: - enabled: false EOF nohup java -jar core/target/feast-core-*${JAR_VERSION_SUFFIX}.jar \ @@ -175,43 +173,43 @@ bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \ ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME} # Start Feast Online Serving in background -cat < /tmp/serving.store.bigquery.yml -name: warehouse -type: BIGQUERY -bigquery_config: - projectId: ${GOOGLE_CLOUD_PROJECT} - datasetId: ${DATASET_NAME} -subscriptions: - - name: "*" - version: "*" - project: "*" -EOF - cat < /tmp/serving.warehouse.application.yml feast: - version: 0.3 + # GRPC service address for Feast Core + # Feast Serving requires connection to Feast Core to retrieve and reload Feast metadata (e.g. FeatureSpecs, Store information) core-host: localhost core-grpc-port: 6565 + + # Indicates the active store. Only a single store in the last can be active at one time. In the future this key + # will be deprecated in order to allow multiple stores to be served from a single serving instance + active_store: historical + + # List of store configurations + stores: + - name: historical + type: BIGQUERY + config: + project_id: my_project + dataset_id: my_dataset + staging_location: gs://mybucket/myprefix + initial_retry_delay_seconds: 1 + total_timeout_seconds: 21600 + subscriptions: + - name: "*" + project: "*" + version: "*" + + job_store: + redis_host: localhost + redis_port: 6379 + tracing: enabled: false - store: - config-path: /tmp/serving.store.bigquery.yml - jobs: - staging-location: ${JOBS_STAGING_LOCATION} - store-type: REDIS - bigquery-initial-retry-delay-secs: 1 - bigquery-total-timeout-secs: 900 - store-options: - host: localhost - port: 6379 + grpc: port: 6566 enable-reflection: true -spring: - main: - web-environment: false - EOF nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \ diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index ee7304f208d..e9a9573a80c 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -98,13 +98,17 @@ grpc: enable-reflection: true feast: - version: 0.3 jobs: - runner: DirectRunner - options: {} - updates: - pollingIntervalMillis: 30000 - timeoutSeconds: 240 + polling_interval_milliseconds: 30000 + job_update_timeout_seconds: 240 + + active_runner: direct + + runners: + - name: direct + type: DirectRunner + options: {} + metrics: enabled: false @@ -120,7 +124,9 @@ spring: jpa: properties.hibernate: format_sql: true - event.merge.entity_copy_observer: allow + event: + merge: + entity_copy_observer: allow hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl hibernate.ddl-auto: update datasource: @@ -128,13 +134,6 @@ spring: username: postgres password: password -management: - metrics: - export: - simple: - enabled: false - statsd: - enabled: false EOF nohup java -jar core/target/feast-core-*${JAR_VERSION_SUFFIX}.jar \ @@ -149,43 +148,32 @@ echo " Starting Feast Online Serving ============================================================ " -# Start Feast Online Serving in background -cat < /tmp/serving.store.redis.yml -name: serving -type: REDIS -redis_config: - host: localhost - port: 6379 -subscriptions: - - name: "*" - version: "*" - project: "*" -EOF cat < /tmp/serving.online.application.yml feast: - version: 0.3 core-host: localhost core-grpc-port: 6565 + + active_store: serving + + # List of store configurations + stores: + - name: serving + type: REDIS # Type of the store. REDIS, BIGQUERY are available options + config: + host: localhost + port: 6379 + subscriptions: + - name: "*" + project: "*" + version: "*" + tracing: enabled: false - store: - config-path: /tmp/serving.store.redis.yml - redis-pool-max-size: 128 - redis-pool-max-idle: 16 - jobs: - staging-location: ${JOBS_STAGING_LOCATION} - store-type: - store-options: {} grpc: port: 6566 enable-reflection: true - -spring: - main: - web-environment: false - EOF nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \ From 1efc01e504ff1759176893ec1145383088c47168 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 23 Apr 2020 12:08:53 +0800 Subject: [PATCH 2/5] Change http port for batch serving --- infra/scripts/test-end-to-end-batch.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/infra/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh index db23e86d704..ad8c58481ca 100755 --- a/infra/scripts/test-end-to-end-batch.sh +++ b/infra/scripts/test-end-to-end-batch.sh @@ -210,6 +210,9 @@ grpc: port: 6566 enable-reflection: true +server: + port: 8081 + EOF nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \ From 713f5ba232727395f48eec8fc7514ec7ba983571 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 23 Apr 2020 12:25:59 +0800 Subject: [PATCH 3/5] Change serving port to 8081 to avoid clashing with core --- infra/scripts/test-end-to-end.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index e9a9573a80c..757b1fdc3ef 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -174,6 +174,10 @@ feast: grpc: port: 6566 enable-reflection: true + +server: + port: 8081 + EOF nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \ From 474c1c2e766285dbb2015625091dcf1ba323ea17 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 23 Apr 2020 13:59:08 +0800 Subject: [PATCH 4/5] Decrease job polling interval --- infra/scripts/test-end-to-end-batch.sh | 2 +- infra/scripts/test-end-to-end.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh index ad8c58481ca..251f9ef833b 100755 --- a/infra/scripts/test-end-to-end-batch.sh +++ b/infra/scripts/test-end-to-end-batch.sh @@ -116,7 +116,7 @@ grpc: feast: jobs: - polling_interval_milliseconds: 30000 + polling_interval_milliseconds: 10000 job_update_timeout_seconds: 240 active_runner: direct diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index 757b1fdc3ef..c33dadc5413 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -177,7 +177,7 @@ grpc: server: port: 8081 - + EOF nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \ From 50bb46573f9a812bf893d758e7e90eaea17e3b53 Mon Sep 17 00:00:00 2001 From: zhilingc Date: Thu, 23 Apr 2020 15:32:36 +0800 Subject: [PATCH 5/5] Set google cloud project in batch store definition --- infra/scripts/test-end-to-end-batch.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/scripts/test-end-to-end-batch.sh b/infra/scripts/test-end-to-end-batch.sh index 251f9ef833b..0e7bfe8bf8d 100755 --- a/infra/scripts/test-end-to-end-batch.sh +++ b/infra/scripts/test-end-to-end-batch.sh @@ -189,9 +189,9 @@ feast: - name: historical type: BIGQUERY config: - project_id: my_project - dataset_id: my_dataset - staging_location: gs://mybucket/myprefix + project_id: ${GOOGLE_CLOUD_PROJECT} + dataset_id: ${DATASET_NAME} + staging_location: ${JOBS_STAGING_LOCATION} initial_retry_delay_seconds: 1 total_timeout_seconds: 21600 subscriptions: