diff --git a/.github/issue_label_bot.yaml b/.github/issue_label_bot.yaml new file mode 100644 index 00000000000..de9464b29de --- /dev/null +++ b/.github/issue_label_bot.yaml @@ -0,0 +1,5 @@ +label-alias: + bug: 'kind/bug' + feature_request: 'kind/feature' + feature: 'kind/feature' + question: 'kind/question' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000000..b9c8cd6dff8 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,31 @@ + + +**What this PR does / why we need it**: + +**Which issue(s) this PR fixes**: + +Fixes # + +**Does this PR introduce a user-facing change?**: + +```release-note + +``` diff --git a/.prow/config.yaml b/.prow/config.yaml index c63d3dce797..3ae9fcbe609 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -66,30 +66,68 @@ presubmits: always_run: true spec: containers: - - image: maven:3.6-jdk-8 + - image: maven:3.6-jdk-11 command: [".prow/scripts/test-core-ingestion.sh"] resources: requests: - cpu: "1500m" + cpu: "2000m" memory: "1536Mi" - limit: - memory: "4096Mi" + skip_branches: + - ^v0\.(3|4)-branch$ + + - name: test-core-and-ingestion-java-8 + decorate: true + always_run: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: [".prow/scripts/test-core-ingestion.sh"] + resources: + requests: + cpu: "2000m" + memory: "1536Mi" + branches: + - ^v0\.(3|4)-branch$ - name: test-serving decorate: true always_run: true spec: containers: - - image: maven:3.6-jdk-8 + - image: maven:3.6-jdk-11 command: [".prow/scripts/test-serving.sh"] + skip_branches: + - ^v0\.(3|4)-branch$ + + - name: test-serving-java-8 + decorate: true + always_run: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: [".prow/scripts/test-serving.sh"] + branches: + - ^v0\.(3|4)-branch$ - name: test-java-sdk decorate: true always_run: true spec: containers: - - image: maven:3.6-jdk-8 + - image: maven:3.6-jdk-11 command: [".prow/scripts/test-java-sdk.sh"] + skip_branches: + - ^v0\.(3|4)-branch$ + + - name: test-java-sdk-java-8 + decorate: true + always_run: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: [".prow/scripts/test-java-sdk.sh"] + branches: + - ^v0\.(3|4)-branch$ - name: test-python-sdk decorate: true @@ -112,14 +150,28 @@ presubmits: always_run: true spec: containers: - - image: maven:3.6-jdk-8 + - image: maven:3.6-jdk-11 command: [".prow/scripts/test-end-to-end.sh"] resources: requests: - cpu: "3000m" - memory: "4096Mi" - limit: + cpu: "6" memory: "6144Mi" + skip_branches: + - ^v0\.(3|4)-branch$ + + - name: test-end-to-end-java-8 + decorate: true + always_run: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: [".prow/scripts/test-end-to-end.sh"] + resources: + requests: + cpu: "6" + memory: "6144Mi" + branches: + - ^v0\.(3|4)-branch$ - name: test-end-to-end-batch decorate: true @@ -130,17 +182,38 @@ presubmits: secret: secretName: feast-service-account containers: - - image: maven:3.6-jdk-8 + - image: maven:3.6-jdk-11 command: [".prow/scripts/test-end-to-end-batch.sh"] resources: requests: - cpu: "1000m" - memory: "1024Mi" - limit: - memory: "4096Mi" + cpu: "6" + memory: "6144Mi" volumeMounts: - name: service-account mountPath: "/etc/service-account" + skip_branches: + - ^v0\.(3|4)-branch$ + + - name: test-end-to-end-batch-java-8 + decorate: true + always_run: true + spec: + volumes: + - name: service-account + secret: + secretName: feast-service-account + containers: + - image: maven:3.6-jdk-8 + command: [".prow/scripts/test-end-to-end-batch.sh"] + resources: + requests: + cpu: "6" + memory: "6144Mi" + volumeMounts: + - name: service-account + mountPath: "/etc/service-account" + branches: + - ^v0\.(3|4)-branch$ postsubmits: gojek/feast: @@ -173,7 +246,7 @@ postsubmits: decorate: true spec: containers: - - image: maven:3.6-jdk-8 + - image: maven:3.6-jdk-11 command: - bash - -c @@ -193,10 +266,42 @@ postsubmits: - name: maven-settings secret: secretName: maven-settings + skip_branches: + # Skip version 0.3 and 0.4 + - ^v0\.(3|4)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ + branches: - # Filter on tags with semantic versioning, prefixed with "v" + # Filter on tags with semantic versioning, prefixed with "v". - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ + - name: publish-java-8-sdk + decorate: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: + - bash + - -c + - .prow/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1} + volumeMounts: + - name: gpg-keys + mountPath: /etc/gpg + readOnly: true + - name: maven-settings + mountPath: /root/.m2/settings.xml + subPath: settings.xml + readOnly: true + volumes: + - name: gpg-keys + secret: + secretName: gpg-keys + - name: maven-settings + secret: + secretName: maven-settings + branches: + # Filter on tags with semantic versioning, prefixed with "v". v0.3 and v0.4 only. + - ^v0\.(3|4)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ + - name: publish-docker-images decorate: true spec: diff --git a/.prow/scripts/publish-java-sdk.sh b/.prow/scripts/publish-java-sdk.sh index 17513d0eb0d..91123c8d4ee 100755 --- a/.prow/scripts/publish-java-sdk.sh +++ b/.prow/scripts/publish-java-sdk.sh @@ -69,4 +69,4 @@ gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key echo "============================================================" echo "Deploying Java SDK with revision: $REVISION" echo "============================================================" -mvn --projects sdk/java -Drevision=$REVISION --batch-mode clean deploy +mvn --projects datatypes/java,sdk/java -Drevision=$REVISION --batch-mode clean deploy diff --git a/.prow/scripts/test-end-to-end-batch.sh b/.prow/scripts/test-end-to-end-batch.sh index 5ff947f20f5..47da7219daa 100755 --- a/.prow/scripts/test-end-to-end-batch.sh +++ b/.prow/scripts/test-end-to-end-batch.sh @@ -3,11 +3,12 @@ set -e set -o pipefail -if ! cat /etc/*release | grep -q stretch; then - echo ${BASH_SOURCE} only supports Debian stretch. - echo Please change your operating system to use this script. - exit 1 -fi +test -z ${GOOGLE_APPLICATION_CREDENTIALS} && GOOGLE_APPLICATION_CREDENTIALS="/etc/service-account/service-account.json" +test -z ${SKIP_BUILD_JARS} && SKIP_BUILD_JARS="false" +test -z ${GOOGLE_CLOUD_PROJECT} && GOOGLE_CLOUD_PROJECT="kf-feast" +test -z ${TEMP_BUCKET} && TEMP_BUCKET="feast-templocation-kf-feast" +test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location" +test -z ${JAR_VERSION_SUFFIX} && JAR_VERSION_SUFFIX="-SNAPSHOT" echo " This script will run end-to-end tests for Feast Core and Batch Serving. @@ -16,10 +17,13 @@ This script will run end-to-end tests for Feast Core and Batch Serving. 2. Install Redis as the job store for Feast Batch Serving. 4. Install Postgres for persisting Feast metadata. 5. Install Kafka and Zookeeper as the Source in Feast. -6. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from +6. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from tests/e2e via pytest. " +apt-get -qq update +apt-get -y install wget netcat kafkacat + echo " ============================================================ @@ -31,8 +35,8 @@ if [[ ! $(command -v gsutil) ]]; then . "${CURRENT_DIR}"/install-google-cloud-sdk.sh fi -export GOOGLE_APPLICATION_CREDENTIALS=/etc/service-account/service-account.json -gcloud auth activate-service-account --key-file /etc/service-account/service-account.json +export GOOGLE_APPLICATION_CREDENTIALS +gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS} @@ -41,10 +45,9 @@ echo " Installing Redis at localhost:6379 ============================================================ " -apt-get -qq update # Allow starting serving in this Maven Docker image. Default set to not allowed. echo "exit 0" > /usr/sbin/policy-rc.d -apt-get -y install redis-server wget > /var/log/redis.install.log +apt-get -y install redis-server > /var/log/redis.install.log redis-server --daemonize yes redis-cli ping @@ -73,24 +76,32 @@ Installing Kafka at localhost:9092 wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz mv kafka_2.12-2.3.0/ /tmp/kafka nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 & -sleep 10 +sleep 5 tail -n10 /var/log/zookeeper.log nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &> /var/log/kafka.log 2>&1 & -sleep 30 +sleep 20 tail -n10 /var/log/kafka.log - -echo " -============================================================ -Building jars for Feast -============================================================ -" - -.prow/scripts/download-maven-cache.sh \ - --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ - --output-dir /root/ - -# Build jars for Feast -mvn --quiet --batch-mode --define skipTests=true clean package +kafkacat -b localhost:9092 -L + +if [[ ${SKIP_BUILD_JARS} != "true" ]]; then + echo " + ============================================================ + Building jars for Feast + ============================================================ + " + + .prow/scripts/download-maven-cache.sh \ + --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \ + --output-dir /root/ + + # Build jars for Feast + mvn --quiet --batch-mode --define skipTests=true clean package + + ls -lh core/target/*jar + ls -lh serving/target/*jar +else + echo "[DEBUG] Skipping building jars" +fi echo " ============================================================ @@ -142,11 +153,13 @@ management: enabled: false EOF -nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \ +nohup java -jar core/target/feast-core-*${JAR_VERSION_SUFFIX}.jar \ --spring.config.location=file:///tmp/core.application.yml \ &> /var/log/feast-core.log & sleep 35 tail -n10 /var/log/feast-core.log +nc -w2 localhost 6565 < /dev/null + echo " ============================================================ Starting Feast Warehouse Serving @@ -155,18 +168,18 @@ Starting Feast Warehouse Serving DATASET_NAME=feast_$(date +%s) -bq --location=US --project_id=kf-feast mk \ +bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \ --dataset \ --default_table_expiration 86400 \ - kf-feast:$DATASET_NAME + ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME} # Start Feast Online Serving in background cat < /tmp/serving.store.bigquery.yml name: warehouse type: BIGQUERY bigquery_config: - projectId: kf-feast - datasetId: $DATASET_NAME + projectId: ${GOOGLE_CLOUD_PROJECT} + datasetId: ${DATASET_NAME} subscriptions: - name: "*" version: "*" @@ -183,24 +196,29 @@ feast: store: config-path: /tmp/serving.store.bigquery.yml jobs: - staging-location: gs://feast-templocation-kf-feast/staging-location + staging-location: ${JOBS_STAGING_LOCATION} store-type: REDIS + bigquery-initial-retry-delay-secs: 1 + bigquery-total-timeout-secs: 900 store-options: - host: $REMOTE_HOST + host: localhost port: 6379 grpc: port: 6566 enable-reflection: true + spring: main: web-environment: false + EOF -nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \ +nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \ --spring.config.location=file:///tmp/serving.warehouse.application.yml \ &> /var/log/feast-serving-warehouse.log & sleep 15 tail -n100 /var/log/feast-serving-warehouse.log +nc -w2 localhost 6566 < /dev/null echo " ============================================================ @@ -230,9 +248,18 @@ ORIGINAL_DIR=$(pwd) cd tests/e2e set +e -pytest bq-batch-retrieval.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml +pytest bq-batch-retrieval.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml TEST_EXIT_CODE=$? +if [[ ${TEST_EXIT_CODE} != 0 ]]; then + echo "[DEBUG] Printing logs" + ls -ltrh /var/log/feast* + cat /var/log/feast-serving-warehouse.log /var/log/feast-core.log + + echo "[DEBUG] Printing Python packages list" + pip list +fi + cd ${ORIGINAL_DIR} exit ${TEST_EXIT_CODE} @@ -242,4 +269,4 @@ Cleaning up ============================================================ " -bq rm -r -f kf-feast:$DATASET_NAME +bq rm -r -f ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME} diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh index cc65968ca22..7709758345d 100755 --- a/.prow/scripts/test-end-to-end.sh +++ b/.prow/scripts/test-end-to-end.sh @@ -3,11 +3,12 @@ set -e set -o pipefail -if ! cat /etc/*release | grep -q stretch; then - echo ${BASH_SOURCE} only supports Debian stretch. - echo Please change your operating system to use this script. - exit 1 -fi +test -z ${GOOGLE_APPLICATION_CREDENTIALS} && GOOGLE_APPLICATION_CREDENTIALS="/etc/service-account/service-account.json" +test -z ${SKIP_BUILD_JARS} && SKIP_BUILD_JARS="false" +test -z ${GOOGLE_CLOUD_PROJECT} && GOOGLE_CLOUD_PROJECT="kf-feast" +test -z ${TEMP_BUCKET} && TEMP_BUCKET="feast-templocation-kf-feast" +test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location" +test -z ${JAR_VERSION_SUFFIX} && JAR_VERSION_SUFFIX="-SNAPSHOT" echo " This script will run end-to-end tests for Feast Core and Online Serving. @@ -15,7 +16,7 @@ This script will run end-to-end tests for Feast Core and Online Serving. 1. Install Redis as the store for Feast Online Serving. 2. Install Postgres for persisting Feast metadata. 3. Install Kafka and Zookeeper as the Source in Feast. -4. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from +4. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from tests/e2e via pytest. " @@ -27,7 +28,6 @@ echo " Installing Redis at localhost:6379 ============================================================ " - # Allow starting serving in this Maven Docker image. Default set to not allowed. echo "exit 0" > /usr/sbin/policy-rc.d apt-get -y install redis-server > /var/log/redis.install.log @@ -66,6 +66,7 @@ sleep 20 tail -n10 /var/log/kafka.log kafkacat -b localhost:9092 -L +if [[ ${SKIP_BUILD_JARS} != "true" ]]; then echo " ============================================================ Building jars for Feast @@ -81,6 +82,9 @@ mvn --quiet --batch-mode --define skipTests=true clean package ls -lh core/target/*jar ls -lh serving/target/*jar +else + echo "[DEBUG] Skipping building jars" +fi echo " ============================================================ @@ -118,7 +122,6 @@ spring: event.merge.entity_copy_observer: allow hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl hibernate.ddl-auto: update - datasource: url: jdbc:postgresql://localhost:5432/postgres username: postgres @@ -133,7 +136,7 @@ management: enabled: false EOF -nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \ +nohup java -jar core/target/feast-core-*${JAR_VERSION_SUFFIX}.jar \ --spring.config.location=file:///tmp/core.application.yml \ &> /var/log/feast-core.log & sleep 35 @@ -163,17 +166,14 @@ feast: version: 0.3 core-host: localhost core-grpc-port: 6565 - tracing: enabled: false - store: config-path: /tmp/serving.store.redis.yml redis-pool-max-size: 128 redis-pool-max-idle: 16 - jobs: - staging-location: gs://feast-templocation-kf-feast/staging-location + staging-location: ${JOBS_STAGING_LOCATION} store-type: store-options: {} @@ -187,11 +187,11 @@ spring: EOF -nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \ +nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \ --spring.config.location=file:///tmp/serving.online.application.yml \ &> /var/log/feast-serving-online.log & sleep 15 -tail -n10 /var/log/feast-serving-online.log +tail -n100 /var/log/feast-serving-online.log nc -w2 localhost 6566 < /dev/null echo " @@ -225,5 +225,14 @@ set +e pytest basic-ingest-redis-serving.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml TEST_EXIT_CODE=$? +if [[ ${TEST_EXIT_CODE} != 0 ]]; then + echo "[DEBUG] Printing logs" + ls -ltrh /var/log/feast* + cat /var/log/feast-serving-online.log /var/log/feast-core.log + + echo "[DEBUG] Printing Python packages list" + pip list +fi + cd ${ORIGINAL_DIR} exit ${TEST_EXIT_CODE} diff --git a/CHANGELOG.md b/CHANGELOG.md index f6ad89e0c0d..7758ae3fb97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,188 @@ # Changelog +## [v0.4.6](https://github.com/gojek/feast/tree/v0.4.6) (2020-02-26) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.4.5...v0.4.6) + +**Merged pull requests:** +- Rename metric name for request latency in feast serving [\#488](https://github.com/gojek/feast/pull/488) ([davidheryanto](https://github.com/davidheryanto)) +- Allow use of secure gRPC in Feast Python client [\#459](https://github.com/gojek/feast/pull/459) ([Yanson](https://github.com/Yanson)) +- Extend WriteMetricsTransform in Ingestion to write feature value stats to StatsD [\#486](https://github.com/gojek/feast/pull/486) ([davidheryanto](https://github.com/davidheryanto)) +- Remove transaction from Ingestion [\#480](https://github.com/gojek/feast/pull/480) ([imjuanleonard](https://github.com/imjuanleonard)) +- Fix fastavro version used in Feast to avoid Timestamp delta error [\#490](https://github.com/gojek/feast/pull/490) ([davidheryanto](https://github.com/davidheryanto)) +- Fail Spotless formatting check before tests execute [\#487](https://github.com/gojek/feast/pull/487) ([ches](https://github.com/ches)) +- Reduce refresh rate of specification refresh in Serving to 10 seconds [\#481](https://github.com/gojek/feast/pull/481) ([woop](https://github.com/woop)) + +## [v0.4.5](https://github.com/gojek/feast/tree/v0.4.5) (2020-02-14) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.4.4...v0.4.5) + +**Merged pull requests:** +- Use bzip2 compressed feature set json as pipeline option [\#466](https://github.com/gojek/feast/pull/466) ([khorshuheng](https://github.com/khorshuheng)) +- Make redis key creation more determinisitic [\#471](https://github.com/gojek/feast/pull/471) ([zhilingc](https://github.com/zhilingc)) +- Helm Chart Upgrades [\#458](https://github.com/gojek/feast/pull/458) ([Yanson](https://github.com/Yanson)) +- Exclude version from grouping [\#441](https://github.com/gojek/feast/pull/441) ([khorshuheng](https://github.com/khorshuheng)) +- Use concrete class for AvroCoder compatibility [\#465](https://github.com/gojek/feast/pull/465) ([zhilingc](https://github.com/zhilingc)) +- Fix typo in split string length check [\#464](https://github.com/gojek/feast/pull/464) ([zhilingc](https://github.com/zhilingc)) +- Update README.md and remove versions from Helm Charts [\#457](https://github.com/gojek/feast/pull/457) ([woop](https://github.com/woop)) +- Deduplicate example notebooks [\#456](https://github.com/gojek/feast/pull/456) ([woop](https://github.com/woop)) +- Allow users not to set max age for batch retrieval [\#446](https://github.com/gojek/feast/pull/446) ([zhilingc](https://github.com/zhilingc)) + +## [v0.4.4](https://github.com/gojek/feast/tree/v0.4.4) (2020-01-28) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.4.3...v0.4.4) + +**Merged pull requests:** + +- Change RedisBackedJobService to use a connection pool [\#439](https://github.com/gojek/feast/pull/439) ([zhilingc](https://github.com/zhilingc)) +- Update GKE installation and chart values to work with 0.4.3 [\#434](https://github.com/gojek/feast/pull/434) ([lgvital](https://github.com/lgvital)) +- Remove "resource" concept and the need to specify a kind in feature sets [\#432](https://github.com/gojek/feast/pull/432) ([woop](https://github.com/woop)) +- Add retry options to BigQuery [\#431](https://github.com/gojek/feast/pull/431) ([Yanson](https://github.com/Yanson)) +- Fix logging [\#430](https://github.com/gojek/feast/pull/430) ([Yanson](https://github.com/Yanson)) +- Add documentation for bigquery batch retrieval [\#428](https://github.com/gojek/feast/pull/428) ([zhilingc](https://github.com/zhilingc)) +- Publish datatypes/java along with sdk/java [\#426](https://github.com/gojek/feast/pull/426) ([ches](https://github.com/ches)) +- Update basic Feast example to Feast 0.4 [\#424](https://github.com/gojek/feast/pull/424) ([woop](https://github.com/woop)) +- Introduce datatypes/java module for proto generation [\#391](https://github.com/gojek/feast/pull/391) ([ches](https://github.com/ches)) + +## [v0.4.3](https://github.com/gojek/feast/tree/v0.4.3) (2020-01-08) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.4.2...v0.4.3) + +**Fixed bugs:** + +- Bugfix for redis ingestion retries throwing NullPointerException on remote runners [\#417](https://github.com/gojek/feast/pull/417) ([khorshuheng](https://github.com/khorshuheng)) + +## [v0.4.2](https://github.com/gojek/feast/tree/v0.4.2) (2020-01-07) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.4.1...v0.4.2) + +**Fixed bugs:** + +- Missing argument in error string in ValidateFeatureRowDoFn [\#401](https://github.com/gojek/feast/issues/401) + +**Merged pull requests:** + +- Define maven revision property when packaging jars in Dockerfile so the images are built successfully [\#410](https://github.com/gojek/feast/pull/410) ([davidheryanto](https://github.com/davidheryanto)) +- Deduplicate rows in subquery [\#409](https://github.com/gojek/feast/pull/409) ([zhilingc](https://github.com/zhilingc)) +- Filter out extra fields, deduplicate fields in ingestion [\#404](https://github.com/gojek/feast/pull/404) ([zhilingc](https://github.com/zhilingc)) +- Automatic documentation generation for gRPC API [\#403](https://github.com/gojek/feast/pull/403) ([woop](https://github.com/woop)) +- Update feast core default values to include hibernate merge strategy [\#400](https://github.com/gojek/feast/pull/400) ([zhilingc](https://github.com/zhilingc)) +- Move cli into feast package [\#398](https://github.com/gojek/feast/pull/398) ([zhilingc](https://github.com/zhilingc)) +- Use Nexus staging plugin for deployment [\#394](https://github.com/gojek/feast/pull/394) ([khorshuheng](https://github.com/khorshuheng)) +- Handle retry for redis io flow [\#274](https://github.com/gojek/feast/pull/274) ([khorshuheng](https://github.com/khorshuheng)) + +## [v0.4.1](https://github.com/gojek/feast/tree/v0.4.1) (2019-12-30) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.4.0...v0.4.1) + +**Merged pull requests:** + +- Add project-related commands to CLI [\#397](https://github.com/gojek/feast/pull/397) ([zhilingc](https://github.com/zhilingc)) + +## [v0.4.0](https://github.com/gojek/feast/tree/v0.4.0) (2019-12-28) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.3.5...v0.4.0) + +**Implemented enhancements:** + +- Edit description in feature specification to also reflect in BigQuery schema description. [\#239](https://github.com/gojek/feast/issues/239) +- Allow for disabling of metrics pushing [\#57](https://github.com/gojek/feast/issues/57) + +**Merged pull requests:** + +- Java SDK release script [\#406](https://github.com/gojek/feast/pull/406) ([davidheryanto](https://github.com/davidheryanto)) +- Use fixed 'dev' revision for test-e2e-batch [\#395](https://github.com/gojek/feast/pull/395) ([davidheryanto](https://github.com/davidheryanto)) +- Project Namespacing [\#393](https://github.com/gojek/feast/pull/393) ([woop](https://github.com/woop)) +- \\(concepts\): change data types to upper case because lower case … [\#389](https://github.com/gojek/feast/pull/389) ([david30907d](https://github.com/david30907d)) +- Remove alpha v1 from java package name [\#387](https://github.com/gojek/feast/pull/387) ([khorshuheng](https://github.com/khorshuheng)) +- Minor bug fixes for Python SDK [\#383](https://github.com/gojek/feast/pull/383) ([voonhous](https://github.com/voonhous)) +- Allow user to override job options [\#377](https://github.com/gojek/feast/pull/377) ([khorshuheng](https://github.com/khorshuheng)) +- Add documentation to default values.yaml in Feast chart [\#376](https://github.com/gojek/feast/pull/376) ([davidheryanto](https://github.com/davidheryanto)) +- Add support for file paths for providing entity rows during batch retrieval [\#375](https://github.com/gojek/feast/pull/375) ([voonhous](https://github.com/voonhous)) +- Update sync helm chart script to ensure requirements.lock in in sync with requirements.yaml [\#373](https://github.com/gojek/feast/pull/373) ([davidheryanto](https://github.com/davidheryanto)) +- Catch errors thrown by BQ during entity table loading [\#371](https://github.com/gojek/feast/pull/371) ([zhilingc](https://github.com/zhilingc)) +- Async job management [\#361](https://github.com/gojek/feast/pull/361) ([zhilingc](https://github.com/zhilingc)) +- Infer schema of PyArrow table directly [\#355](https://github.com/gojek/feast/pull/355) ([voonhous](https://github.com/voonhous)) +- Add readiness checks for Feast services in end to end test [\#337](https://github.com/gojek/feast/pull/337) ([davidheryanto](https://github.com/davidheryanto)) +- Create CHANGELOG.md [\#321](https://github.com/gojek/feast/pull/321) ([woop](https://github.com/woop)) + +## [v0.3.6](https://github.com/gojek/feast/tree/v0.3.6) (2020-01-03) + +**Merged pull requests:** + +[Full Changelog](https://github.com/gojek/feast/compare/v0.3.5...v0.3.6) + +- Add support for file paths for providing entity rows during batch retrieval [\#375](https://github.com/gojek/feast/pull/376) ([voonhous](https://github.com/voonhous)) + +## [v0.3.5](https://github.com/gojek/feast/tree/v0.3.5) (2019-12-26) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.3.4...v0.3.5) + +**Merged pull requests:** + +- Always set destination table in BigQuery query config in Feast Batch Serving so it can handle large results [\#392](https://github.com/gojek/feast/pull/392) ([davidheryanto](https://github.com/davidheryanto)) + +## [v0.3.4](https://github.com/gojek/feast/tree/v0.3.4) (2019-12-23) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.3.3...v0.3.4) + +**Merged pull requests:** + +- Make redis key creation more determinisitic [\#380](https://github.com/gojek/feast/pull/380) ([zhilingc](https://github.com/zhilingc)) + +## [v0.3.3](https://github.com/gojek/feast/tree/v0.3.3) (2019-12-18) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.3.2...v0.3.3) + +**Implemented enhancements:** + +- Added Docker Compose for Feast [\#272](https://github.com/gojek/feast/issues/272) +- Added ability to check import job status and cancel job through Python SDK [\#194](https://github.com/gojek/feast/issues/194) +- Added basic customer transactions example [\#354](https://github.com/gojek/feast/pull/354) ([woop](https://github.com/woop)) + +**Merged pull requests:** + +- Added Prow jobs to automate the release of Docker images and Python SDK [\#369](https://github.com/gojek/feast/pull/369) ([davidheryanto](https://github.com/davidheryanto)) +- Fixed installation link in README.md [\#368](https://github.com/gojek/feast/pull/368) ([Jeffwan](https://github.com/Jeffwan)) +- Fixed Java SDK tests not actually running \(missing dependencies\) [\#366](https://github.com/gojek/feast/pull/366) ([woop](https://github.com/woop)) +- Added more batch retrieval tests [\#357](https://github.com/gojek/feast/pull/357) ([zhilingc](https://github.com/zhilingc)) +- Python SDK and Feast Core Bug Fixes [\#353](https://github.com/gojek/feast/pull/353) ([woop](https://github.com/woop)) +- Updated buildFeatureSets method in Golang SDK [\#351](https://github.com/gojek/feast/pull/351) ([davidheryanto](https://github.com/davidheryanto)) +- Python SDK cleanup [\#348](https://github.com/gojek/feast/pull/348) ([woop](https://github.com/woop)) +- Broke up queries for point in time correctness joins [\#347](https://github.com/gojek/feast/pull/347) ([zhilingc](https://github.com/zhilingc)) +- Exports gRPC call metrics and Feast resource metrics in Core [\#345](https://github.com/gojek/feast/pull/345) ([davidheryanto](https://github.com/davidheryanto)) +- Fixed broken Google Group link on Community page [\#343](https://github.com/gojek/feast/pull/343) ([ches](https://github.com/ches)) +- Ensured ImportJobTest is not flaky by checking WriteToStore metric and requesting adequate resources for testing [\#332](https://github.com/gojek/feast/pull/332) ([davidheryanto](https://github.com/davidheryanto)) +- Added docker-compose file with Jupyter notebook [\#328](https://github.com/gojek/feast/pull/328) ([khorshuheng](https://github.com/khorshuheng)) +- Added minimal implementation of ingesting Parquet and CSV files [\#327](https://github.com/gojek/feast/pull/327) ([voonhous](https://github.com/voonhous)) + +## [v0.3.2](https://github.com/gojek/feast/tree/v0.3.2) (2019-11-29) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.3.1...v0.3.2) + +**Merged pull requests:** + +- Fixed incorrect BigQuery schema creation from FeatureSetSpec [\#340](https://github.com/gojek/feast/pull/340) ([davidheryanto](https://github.com/davidheryanto)) +- Filtered out feature sets that dont share the same source [\#339](https://github.com/gojek/feast/pull/339) ([zhilingc](https://github.com/zhilingc)) +- Changed latency calculation method to not use Timer [\#338](https://github.com/gojek/feast/pull/338) ([zhilingc](https://github.com/zhilingc)) +- Moved Prometheus annotations to pod template for serving [\#336](https://github.com/gojek/feast/pull/336) ([zhilingc](https://github.com/zhilingc)) +- Removed metrics windowing, cleaned up step names for metrics writing [\#334](https://github.com/gojek/feast/pull/334) ([zhilingc](https://github.com/zhilingc)) +- Set BigQuery table time partition inside get table function [\#333](https://github.com/gojek/feast/pull/333) ([zhilingc](https://github.com/zhilingc)) +- Added unit test in Redis to return values with no max age set [\#329](https://github.com/gojek/feast/pull/329) ([smadarasmi](https://github.com/smadarasmi)) +- Consolidated jobs into single steps instead of branching out [\#326](https://github.com/gojek/feast/pull/326) ([zhilingc](https://github.com/zhilingc)) +- Pinned Python SDK to minor versions for dependencies [\#322](https://github.com/gojek/feast/pull/322) ([woop](https://github.com/woop)) +- Added Auto format to Google style with Spotless [\#317](https://github.com/gojek/feast/pull/317) ([ches](https://github.com/ches)) + +## [v0.3.1](https://github.com/gojek/feast/tree/v0.3.1) (2019-11-25) + +[Full Changelog](https://github.com/gojek/feast/compare/v0.3.0...v0.3.1) + +**Merged pull requests:** + +- Added Prometheus metrics to serving [\#316](https://github.com/gojek/feast/pull/316) ([zhilingc](https://github.com/zhilingc)) +- Changed default job metrics sink to Statsd [\#315](https://github.com/gojek/feast/pull/315) ([zhilingc](https://github.com/zhilingc)) +- Fixed module import error in Feast CLI [\#314](https://github.com/gojek/feast/pull/314) ([davidheryanto](https://github.com/davidheryanto)) + ## [v0.3.0](https://github.com/gojek/feast/tree/v0.3.0) (2019-11-19) [Full Changelog](https://github.com/gojek/feast/compare/v0.1.8...v0.3.0) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index eb38db30080..00000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,334 +0,0 @@ -# Contributing Guide - -## Getting Started - -The following guide will help you quickly run Feast in your local machine. - -The main components of Feast are: -- **Feast Core** handles FeatureSpec registration, starts and monitors Ingestion - jobs and ensures that Feast internal metadata is consistent. -- **Feast Ingestion** subscribes to streams of FeatureRow and writes the feature - values to registered Stores. -- **Feast Serving** handles requests for features values retrieval from the end users. - -![Feast Components Overview](docs/assets/feast-components-overview.png) - -**Pre-requisites** -- Java SDK version 8 -- Python version 3.6 (or above) and pip -- Access to Postgres database (version 11 and above) -- Access to [Redis](https://redis.io/topics/quickstart) instance (tested on version 5.x) -- Access to [Kafka](https://kafka.apache.org/) brokers (tested on version 2.x) -- [Maven ](https://maven.apache.org/install.html) version 3.6.x -- [grpc_cli](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md) - is useful for debugging and quick testing -- An overview of Feast specifications and [protos](./protos/feast) - -> **Assumptions:** -> -> 1. Postgres is running in "localhost:5432" and has a database called "postgres" which -> can be accessed with credentials user "postgres" and password "password". -> To use different database name and credentials, please update -> "$FEAST_HOME/core/src/main/resources/application.yml" -> or set these environment variables: DB_HOST, DB_USERNAME, DB_PASSWORD. -> 2. Redis is running locally and accessible from "localhost:6379" -> 3. Feast has admin access to BigQuery. - - -``` -# Clone Feast branch 0.3-dev -# $FEAST_HOME will refer to be the root directory of this Feast Git repository - -git clone -b 0.3-dev https://github.com/gojek/feast -cd feast -``` - -#### Starting Feast Core - -``` -# Please check the default configuration for Feast Core in -# "$FEAST_HOME/core/src/main/resources/application.yml" and update it accordingly. -# -# Start Feast Core GRPC server on localhost:6565 -mvn --projects core spring-boot:run - -# If Feast Core starts successfully, verify the correct Stores are registered -# correctly, for example by using grpc_cli. -grpc_cli call localhost:6565 GetStores '' - -# Should return something similar to the following. -# Note that you should change BigQuery projectId and datasetId accordingly -# in "$FEAST_HOME/core/src/main/resources/application.yml" - -store { - name: "SERVING" - type: REDIS - subscriptions { - project: "*" - name: "*" - version: "*" - } - redis_config { - host: "localhost" - port: 6379 - } -} -store { - name: "WAREHOUSE" - type: BIGQUERY - subscriptions { - project: "*" - name: "*" - version: "*" - } - bigquery_config { - project_id: "my-google-project-id" - dataset_id: "my-bigquery-dataset-id" - } -} -``` - -#### Starting Feast Serving - -Feast Serving requires administrators to provide an **existing** store name in Feast. -An instance of Feast Serving can only retrieve features from a **single** store. -> In order to retrieve features from multiple stores you must start **multiple** -instances of Feast serving. If you start multiple Feast serving on a single host, -make sure that they are listening on different ports. - -``` -# Start Feast Serving GRPC server on localhost:6566 with store name "SERVING" -mvn --projects serving spring-boot:run -Dspring-boot.run.arguments='--feast.store-name=SERVING' - -# To verify Feast Serving starts successfully -grpc_cli call localhost:6566 GetFeastServingType '' - -# Should return something similar to the following. -type: FEAST_SERVING_TYPE_ONLINE -``` - - -#### Registering a FeatureSet - -Create a new FeatureSet on Feast by sending a request to Feast Core. When a -feature set is successfully registered, Feast Core will start an **ingestion** job -that listens for new features in the FeatureSet. Note that Feast currently only -supports source of type "KAFKA", so you must have access to a running Kafka broker -to register a FeatureSet successfully. - -``` -# Example of registering a new driver feature set -# Note the source value, it assumes that you have access to a Kafka broker -# running on localhost:9092 - -grpc_cli call localhost:6565 ApplyFeatureSet ' -feature_set { - name: "driver" - version: 1 - - entities { - name: "driver_id" - value_type: INT64 - } - - features { - name: "city" - value_type: STRING - } - - source { - type: KAFKA - kafka_source_config { - bootstrap_servers: "localhost:9092" - } - } -} -' - -# To check that the FeatureSet has been registered correctly. -# You should also see logs from Feast Core of the ingestion job being started -grpc_cli call localhost:6565 GetFeatureSets '' -``` - - -#### Ingestion and Population of Feature Values - -``` -# Produce FeatureRow messages to Kafka so it will be ingested by Feast -# and written to the registered stores. -# Make sure the value here is the topic assigned to the feature set -# ... producer.send("feast-driver-features" ...) -# -# Install Python SDK to help writing FeatureRow messages to Kafka -cd $FEAST_HOME/sdk/python -pip3 install -e . -pip3 install pendulum - -# Produce FeatureRow messages to Kafka so it will be ingested by Feast -# and written to the corresponding store. -# Make sure the value here is the topic assigned to the feature set -# ... producer.send("feast-test_feature_set-features" ...) -python3 - < Tool Windows > Maven` -1. Drill down to e.g. `Feast Core > Plugins > spring-boot:run`, right-click and `Create 'feast-core [spring-boot'…` -1. In the dialog that pops up, check the `Resolve Workspace artifacts` box -1. Click `OK`. You should now be able to select this run configuration for the Play button in the main toolbar, keyboard shortcuts, etc. - -[idea-boot-main]: https://stackoverflow.com/questions/30237768/run-spring-boots-main-using-ide - -#### Tips for Running Postgres, Redis and Kafka with Docker - -This guide assumes you are running Docker service on a bridge network (which -is usually the case if you're running Linux). Otherwise, you may need to -use different network options than shown below. - -> `--net host` usually only works as expected when you're running Docker -> service in bridge networking mode. - -``` -# Start Postgres -docker run --name postgres --rm -it -d --net host -e POSTGRES_DB=postgres -e POSTGRES_USER=postgres \ --e POSTGRES_PASSWORD=password postgres:12-alpine - -# Start Redis -docker run --name redis --rm -it --net host -d redis:5-alpine - -# Start Zookeeper (needed by Kafka) -docker run --rm \ - --net=host \ - --name=zookeeper \ - --env=ZOOKEEPER_CLIENT_PORT=2181 \ - --detach confluentinc/cp-zookeeper:5.2.1 - -# Start Kafka -docker run --rm \ - --net=host \ - --name=kafka \ - --env=KAFKA_ZOOKEEPER_CONNECT=localhost:2181 \ - --env=KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \ - --env=KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ - --detach confluentinc/cp-kafka:5.2.1 -``` - -## Code reviews - -Code submission to Feast (including submission from project maintainers) requires review and approval. -Please submit a **pull request** to initiate the code review process. We use [prow](https://github.com/kubernetes/test-infra/tree/master/prow) to manage the testing and reviewing of pull requests. Please refer to [config.yaml](../.prow/config.yaml) for details on the test jobs. - -## Code conventions - -### Java - -We conform to the [Google Java Style Guide]. Maven can helpfully take care of -that for you before you commit: - - $ mvn spotless:apply - -Formatting will be checked automatically during the `verify` phase. This can be -skipped temporarily: - - $ mvn spotless:check # Check is automatic upon `mvn verify` - $ mvn verify -Dspotless.check.skip - -If you're using IntelliJ, you can import [these code style settings][G -IntelliJ] if you'd like to use the IDE's reformat function as you work. - -### Go - -Make sure you apply `go fmt`. - -[Google Java Style Guide]: https://google.github.io/styleguide/javaguide.html -[G IntelliJ]: https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml diff --git a/Makefile b/Makefile index de61fe2892f..e4cbd787b23 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,12 @@ -# +# # Copyright 2019 The Feast Authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # https://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,42 +14,149 @@ # limitations under the License. # -PROJECT_ROOT := $(shell git rev-parse --show-toplevel) +ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) +PROTO_TYPE_SUBDIRS = core serving types storage +PROTO_SERVICE_SUBDIRS = core serving -test: - mvn test +# General + +format: format-python format-go format-java + +lint: lint-python lint-go lint-java + +test: test-python test-java test-go + +protos: compile-protos-go compile-protos-python compile-protos-docs + +build: protos build-java build-docker build-html + +install-ci-dependencies: install-python-ci-dependencies install-go-ci-dependencies install-java-ci-dependencies + +# Java -test-integration: - $(MAKE) -C testing/integration test-integration TYPE=$(TYPE) ID=$(ID) +install-java-ci-dependencies: + mvn verify clean --fail-never -build-proto: - $(MAKE) -C protos gen-go - $(MAKE) -C protos gen-python - $(MAKE) -C protos gen-docs +format-java: + mvn spotless:apply -build-cli: - $(MAKE) build-proto - $(MAKE) -C cli build-all +lint-java: + mvn spotless:check + +test-java: + mvn test + +test-java-with-coverage: + mvn test jacoco:report-aggregate build-java: mvn clean verify -build-docker: - docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . - docker build -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . +# Python SDK + +install-python-ci-dependencies: + pip install -r sdk/python/requirements-ci.txt + +compile-protos-python: install-python-ci-dependencies + @$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/$(dir)/*.proto;) + @$(foreach dir,$(PROTO_SERVICE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/ feast/$(dir)/*.proto;) + +install-python: compile-protos-python + pip install -e sdk/python --upgrade + +test-python: + pytest --verbose --color=yes sdk/python/tests + +format-python: + cd ${ROOT_DIR}/sdk/python; isort -rc feast tests + cd ${ROOT_DIR}/sdk/python; black --target-version py37 feast tests + +lint-python: + # TODO: This mypy test needs to be re-enabled and all failures fixed + #cd ${ROOT_DIR}/sdk/python; mypy feast/ tests/ + cd ${ROOT_DIR}/sdk/python; flake8 feast/ tests/ + cd ${ROOT_DIR}/sdk/python; black --check feast tests + +# Go SDK + +install-go-ci-dependencies: + go get -u github.com/golang/protobuf/protoc-gen-go + go get -u golang.org/x/lint/golint + +compile-protos-go: install-go-ci-dependencies + @$(foreach dir,types serving, cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos/ feast/$(dir)/*.proto;) + +test-go: + cd ${ROOT_DIR}/sdk/go; go test ./... + +format-go: + cd ${ROOT_DIR}/sdk/go; gofmt -s -w *.go + +lint-go: + cd ${ROOT_DIR}/sdk/go; go vet + +# Docker build-push-docker: @$(MAKE) build-docker registry=$(REGISTRY) version=$(VERSION) + @$(MAKE) push-core-docker registry=$(REGISTRY) version=$(VERSION) + @$(MAKE) push-serving-docker registry=$(REGISTRY) version=$(VERSION) + @$(MAKE) push-ci-docker registry=$(REGISTRY) + +build-docker: build-core-docker build-serving-docker build-ci-docker + +push-core-docker: docker push $(REGISTRY)/feast-core:$(VERSION) + +push-serving-docker: docker push $(REGISTRY)/feast-serving:$(VERSION) +push-ci-docker: + docker push $(REGISTRY)/feast-ci:latest + +build-core-docker: + docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . + +build-serving-docker: + docker build -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . + +build-ci-docker: + docker build -t $(REGISTRY)/feast-ci:latest -f infra/docker/ci/Dockerfile . + +# Documentation + +install-dependencies-proto-docs: + cd ${ROOT_DIR}/protos; + mkdir -p $$HOME/bin + mkdir -p $$HOME/include + go get github.com/golang/protobuf/proto && \ + go get github.com/russross/blackfriday/v2 && \ + cd $$(mktemp -d) && \ + git clone https://github.com/istio/tools/ && \ + cd tools/cmd/protoc-gen-docs && \ + go build && \ + cp protoc-gen-docs $$HOME/bin && \ + cd $$HOME && curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.11.2/protoc-3.11.2-linux-x86_64.zip && \ + unzip protoc-3.11.2-linux-x86_64.zip -d protoc3 && \ + mv protoc3/bin/* $$HOME/bin/ && \ + chmod +x $$HOME/bin/protoc && \ + mv protoc3/include/* $$HOME/include + +compile-protos-docs: + cd ${ROOT_DIR}/protos; protoc --docs_out=../dist/grpc feast/*/*.proto || \ + cd ${ROOT_DIR}; $(MAKE) install-dependencies-proto-docs && cd ${ROOT_DIR}/protos; PATH=$$HOME/bin:$$PATH protoc -I $$HOME/include/ -I . --docs_out=../dist/grpc feast/*/*.proto + clean-html: - rm -rf $(PROJECT_ROOT)/dist - -build-html: - rm -rf $(PROJECT_ROOT)/dist/ - mkdir -p $(PROJECT_ROOT)/dist/python - mkdir -p $(PROJECT_ROOT)/dist/grpc - cd $(PROJECT_ROOT)/protos && $(MAKE) gen-docs - cd $(PROJECT_ROOT)/sdk/python/docs && $(MAKE) html - cp -r $(PROJECT_ROOT)/sdk/python/docs/html/* $(PROJECT_ROOT)/dist/python \ No newline at end of file + rm -rf $(ROOT_DIR)/dist + +build-html: clean-html + mkdir -p $(ROOT_DIR)/dist/python + mkdir -p $(ROOT_DIR)/dist/grpc + + # Build Protobuf documentation + $(MAKE) compile-protos-docs + + # Build Python SDK documentation + $(MAKE) compile-protos-python + cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) html + cp -r $(ROOT_DIR)/sdk/python/docs/html/* $(ROOT_DIR)/dist/python diff --git a/README.md b/README.md index d9b16748266..63b1d45d389 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,31 @@ my_model = ml.fit(data) prediction = my_model.predict(fs.get_online_features(customer_features, customer_entities)) ``` +## Getting Started with Docker Compose +The following commands will start Feast in online-only mode. +``` +git clone https://github.com/gojek/feast.git +cd feast/infra/docker-compose +cp .env.sample .env +docker-compose up -d +``` + +A [Jupyter Notebook](http://localhost:8888/tree/feast/examples) is now available to start using Feast. + +Please see the links below to set up Feast for batch/historical serving with BigQuery. + ## Important resources - * [Why Feast?](docs/why-feast.md) - * [Concepts](docs/concepts.md) - * [Installation](docs/getting-started/installing-feast.md) - * [Getting Help](docs/community.md) + +Please refer to the official documentation at + + * [Why Feast?](https://docs.feast.dev/why-feast) + * [Concepts](https://docs.feast.dev/concepts) + * [Installation](https://docs.feast.dev/installing-feast/overview) + * [Examples](https://github.com/gojek/feast/blob/master/examples/) + * [Roadmap](https://docs.feast.dev/roadmap) + * [Change Log](https://github.com/gojek/feast/blob/master/CHANGELOG.md) + * [Slack (#Feast)](https://join.slack.com/t/kubeflow/shared_invite/enQtNDg5MTM4NTQyNjczLTdkNTVhMjg1ZTExOWI0N2QyYTQ2MTIzNTJjMWRiOTFjOGRlZWEzODc1NzMwNTMwM2EzNjY1MTFhODczNjk4MTk) ## Notice -Feast is a community project and is still under active development. Your feedback and contributions are important to us. Please have a look at our [contributing guide](CONTRIBUTING.md) for details. +Feast is a community project and is still under active development. Your feedback and contributions are important to us. Please have a look at our [contributing guide](docs/contributing.md) for details. diff --git a/core/pom.xml b/core/pom.xml index 954c7c00185..ec1e7780b72 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -39,10 +39,6 @@ false - - org.xolstice.maven.plugins - protobuf-maven-plugin - @@ -76,6 +72,10 @@ org.springframework.boot spring-boot-starter-log4j2 + + org.apache.logging.log4j + log4j-web + io.github.lognet @@ -114,7 +114,7 @@ protobuf-java-util - + com.google.guava guava @@ -201,5 +201,11 @@ spring-boot-test-autoconfigure test + + + javax.xml.bind + jaxb-api + + diff --git a/core/src/main/java/feast/core/config/FeatureStreamConfig.java b/core/src/main/java/feast/core/config/FeatureStreamConfig.java index 45de359ac76..4c9f23af657 100644 --- a/core/src/main/java/feast/core/config/FeatureStreamConfig.java +++ b/core/src/main/java/feast/core/config/FeatureStreamConfig.java @@ -16,7 +16,6 @@ */ package feast.core.config; -import com.google.common.base.Strings; import feast.core.SourceProto.KafkaSourceConfig; import feast.core.SourceProto.SourceType; import feast.core.config.FeastProperties.StreamProperties; @@ -69,7 +68,7 @@ public Source getDefaultSource(FeastProperties feastProperties) { } catch (InterruptedException | ExecutionException e) { if (e.getCause().getClass().equals(TopicExistsException.class)) { log.warn( - Strings.lenientFormat( + String.format( "Unable to create topic %s in the feature stream, topic already exists, using existing topic.", topicName)); } else { diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index b8d0670d0d2..661bbe24039 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -16,6 +16,7 @@ */ package feast.core.grpc; +import com.google.protobuf.InvalidProtocolBufferException; import feast.core.CoreServiceGrpc.CoreServiceImplBase; import feast.core.CoreServiceProto.ApplyFeatureSetRequest; import feast.core.CoreServiceProto.ApplyFeatureSetResponse; @@ -77,7 +78,7 @@ public void getFeatureSet( GetFeatureSetResponse response = specService.getFeatureSet(request); responseObserver.onNext(response); responseObserver.onCompleted(); - } catch (RetrievalException | StatusRuntimeException e) { + } catch (RetrievalException | StatusRuntimeException | InvalidProtocolBufferException e) { log.error("Exception has occurred in GetFeatureSet method: ", e); responseObserver.onError( Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); @@ -91,7 +92,7 @@ public void listFeatureSets( ListFeatureSetsResponse response = specService.listFeatureSets(request.getFilter()); responseObserver.onNext(response); responseObserver.onCompleted(); - } catch (RetrievalException | IllegalArgumentException e) { + } catch (RetrievalException | IllegalArgumentException | InvalidProtocolBufferException e) { log.error("Exception has occurred in ListFeatureSet method: ", e); responseObserver.onError( Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); diff --git a/core/src/main/java/feast/core/job/JobUpdateTask.java b/core/src/main/java/feast/core/job/JobUpdateTask.java index 57b2dfee4f3..87578cce25a 100644 --- a/core/src/main/java/feast/core/job/JobUpdateTask.java +++ b/core/src/main/java/feast/core/job/JobUpdateTask.java @@ -173,6 +173,7 @@ private Job startJob( return job; } catch (Exception e) { + log.error(e.getMessage()); AuditLogger.log( Resource.JOB, jobId, diff --git a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java index 2de46ae1f2d..a95ed3fb1fa 100644 --- a/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java +++ b/core/src/main/java/feast/core/job/dataflow/DataflowJobManager.java @@ -19,10 +19,8 @@ import static feast.core.util.PipelineUtil.detectClassPathResourcesToStage; import com.google.api.services.dataflow.Dataflow; -import com.google.common.base.Strings; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import com.google.protobuf.util.JsonFormat.Printer; import feast.core.FeatureSetProto; import feast.core.SourceProto; import feast.core.StoreProto; @@ -30,15 +28,13 @@ import feast.core.exception.JobExecutionException; import feast.core.job.JobManager; import feast.core.job.Runner; -import feast.core.model.FeatureSet; -import feast.core.model.Job; -import feast.core.model.JobStatus; -import feast.core.model.Project; -import feast.core.model.Source; -import feast.core.model.Store; +import feast.core.job.option.FeatureSetJsonByteConverter; +import feast.core.model.*; import feast.core.util.TypeConversion; import feast.ingestion.ImportJob; +import feast.ingestion.options.BZip2Compressor; import feast.ingestion.options.ImportOptions; +import feast.ingestion.options.OptionCompressor; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -78,17 +74,25 @@ public Runner getRunnerType() { @Override public Job startJob(Job job) { - List featureSetProtos = - job.getFeatureSets().stream().map(FeatureSet::toProto).collect(Collectors.toList()); try { + List featureSetProtos = new ArrayList<>(); + for (FeatureSet featureSet : job.getFeatureSets()) { + featureSetProtos.add(featureSet.toProto()); + } return submitDataflowJob( job.getId(), featureSetProtos, job.getSource().toProto(), job.getStore().toProto(), false); + } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(String.format("Unable to start job %s", job.getId()), e); + log.error(e.getMessage()); + throw new IllegalArgumentException( + String.format( + "DataflowJobManager failed to START job with id '%s' because the job" + + "has an invalid spec. Please check the FeatureSet, Source and Store specs. Actual error message: %s", + job.getId(), e.getMessage())); } } @@ -101,14 +105,19 @@ public Job startJob(Job job) { @Override public Job updateJob(Job job) { try { - List featureSetProtos = - job.getFeatureSets().stream().map(FeatureSet::toProto).collect(Collectors.toList()); - + List featureSetProtos = new ArrayList<>(); + for (FeatureSet featureSet : job.getFeatureSets()) { + featureSetProtos.add(featureSet.toProto()); + } return submitDataflowJob( job.getId(), featureSetProtos, job.getSource().toProto(), job.getStore().toProto(), true); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(String.format("Unable to update job %s", job.getId()), e); + log.error(e.getMessage()); + throw new IllegalArgumentException( + String.format( + "DataflowJobManager failed to UPDATE job with id '%s' because the job" + + "has an invalid spec. Please check the FeatureSet, Source and Store specs. Actual error message: %s", + job.getId(), e.getMessage())); } } @@ -138,7 +147,7 @@ public void abortJob(String dataflowJobId) { } catch (Exception e) { log.error("Unable to drain job with id: {}, cause: {}", dataflowJobId, e.getMessage()); throw new RuntimeException( - Strings.lenientFormat("Unable to drain job with id: %s", dataflowJobId), e); + String.format("Unable to drain job with id: %s", dataflowJobId), e); } } @@ -210,13 +219,12 @@ private ImportOptions getPipelineOptions( throws IOException { String[] args = TypeConversion.convertMapToArgs(defaultOptions); ImportOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args).as(ImportOptions.class); - Printer printer = JsonFormat.printer(); - List featureSetsJson = new ArrayList<>(); - for (FeatureSetProto.FeatureSet featureSet : featureSets) { - featureSetsJson.add(printer.print(featureSet.getSpec())); - } - pipelineOptions.setFeatureSetJson(featureSetsJson); - pipelineOptions.setStoreJson(Collections.singletonList(printer.print(sink))); + + OptionCompressor> featureSetJsonCompressor = + new BZip2Compressor<>(new FeatureSetJsonByteConverter()); + + pipelineOptions.setFeatureSetJson(featureSetJsonCompressor.compress(featureSets)); + pipelineOptions.setStoreJson(Collections.singletonList(JsonFormat.printer().print(sink))); pipelineOptions.setProject(projectId); pipelineOptions.setUpdate(update); pipelineOptions.setRunner(DataflowRunner.class); diff --git a/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java b/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java index f7ded9fec76..94b3a8fd571 100644 --- a/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java +++ b/core/src/main/java/feast/core/job/direct/DirectJobRegistry.java @@ -16,7 +16,6 @@ */ package feast.core.job.direct; -import com.google.common.base.Strings; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -41,7 +40,7 @@ public DirectJobRegistry() { public void add(DirectJob job) { if (jobs.containsKey(job.getJobId())) { throw new IllegalArgumentException( - Strings.lenientFormat("Job with id %s already exists and is running", job.getJobId())); + String.format("Job with id %s already exists and is running", job.getJobId())); } jobs.put(job.getJobId(), job); } diff --git a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java index fdf3aad9bc3..5c4a9972799 100644 --- a/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java +++ b/core/src/main/java/feast/core/job/direct/DirectRunnerJobManager.java @@ -16,29 +16,27 @@ */ package feast.core.job.direct; -import com.google.common.base.Strings; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import com.google.protobuf.util.JsonFormat.Printer; import feast.core.FeatureSetProto; -import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.StoreProto; import feast.core.config.FeastProperties.MetricsProperties; import feast.core.exception.JobExecutionException; import feast.core.job.JobManager; import feast.core.job.Runner; +import feast.core.job.option.FeatureSetJsonByteConverter; import feast.core.model.FeatureSet; import feast.core.model.Job; import feast.core.model.JobStatus; import feast.core.util.TypeConversion; import feast.ingestion.ImportJob; +import feast.ingestion.options.BZip2Compressor; import feast.ingestion.options.ImportOptions; +import feast.ingestion.options.OptionCompressor; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.beam.runners.direct.DirectRunner; import org.apache.beam.sdk.PipelineResult; @@ -75,8 +73,10 @@ public Runner getRunnerType() { @Override public Job startJob(Job job) { try { - List featureSetProtos = - job.getFeatureSets().stream().map(FeatureSet::toProto).collect(Collectors.toList()); + List featureSetProtos = new ArrayList<>(); + for (FeatureSet featureSet : job.getFeatureSets()) { + featureSetProtos.add(featureSet.toProto()); + } ImportOptions pipelineOptions = getPipelineOptions(featureSetProtos, job.getStore().toProto()); PipelineResult pipelineResult = runPipeline(pipelineOptions); @@ -92,20 +92,20 @@ public Job startJob(Job job) { } private ImportOptions getPipelineOptions( - List featureSets, StoreProto.Store sink) - throws InvalidProtocolBufferException { + List featureSets, StoreProto.Store sink) throws IOException { String[] args = TypeConversion.convertMapToArgs(defaultOptions); ImportOptions pipelineOptions = PipelineOptionsFactory.fromArgs(args).as(ImportOptions.class); - Printer printer = JsonFormat.printer(); - List featureSetsJson = new ArrayList<>(); - for (FeatureSetProto.FeatureSet featureSet : featureSets) { - featureSetsJson.add(printer.print(featureSet.getSpec())); - } - pipelineOptions.setFeatureSetJson(featureSetsJson); - pipelineOptions.setStoreJson(Collections.singletonList(printer.print(sink))); + + OptionCompressor> featureSetJsonCompressor = + new BZip2Compressor<>(new FeatureSetJsonByteConverter()); + + pipelineOptions.setFeatureSetJson(featureSetJsonCompressor.compress(featureSets)); + pipelineOptions.setStoreJson(Collections.singletonList(JsonFormat.printer().print(sink))); pipelineOptions.setRunner(DirectRunner.class); pipelineOptions.setProject(""); // set to default value to satisfy validation + log.info("FINDING METRICS!\n{}", metrics); if (metrics.isEnabled()) { + log.info("METRICS ENABLED!"); pipelineOptions.setMetricsExporterType(metrics.getType()); if (metrics.getType().equals("statsd")) { pipelineOptions.setStatsdHost(metrics.getHost()); @@ -131,10 +131,6 @@ public Job updateJob(Job job) { String jobId = job.getExtId(); abortJob(jobId); try { - List featureSetSpecs = new ArrayList<>(); - for (FeatureSet featureSet : job.getFeatureSets()) { - featureSetSpecs.add(featureSet.toProto().getSpec()); - } return startJob(job); } catch (JobExecutionException e) { throw new JobExecutionException(String.format("Error running ingestion job: %s", e), e); @@ -152,8 +148,7 @@ public void abortJob(String extId) { try { job.abort(); } catch (IOException e) { - throw new RuntimeException( - Strings.lenientFormat("Unable to abort DirectRunner job %s", extId), e); + throw new RuntimeException(String.format("Unable to abort DirectRunner job %s", extId), e); } jobs.remove(extId); } diff --git a/core/src/main/java/feast/core/job/option/FeatureSetJsonByteConverter.java b/core/src/main/java/feast/core/job/option/FeatureSetJsonByteConverter.java new file mode 100644 index 00000000000..dbd04d668fd --- /dev/null +++ b/core/src/main/java/feast/core/job/option/FeatureSetJsonByteConverter.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.job.option; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import feast.core.FeatureSetProto; +import feast.ingestion.options.OptionByteConverter; +import java.util.ArrayList; +import java.util.List; + +public class FeatureSetJsonByteConverter + implements OptionByteConverter> { + + /** + * Convert list of feature sets to json strings joined by new line, represented as byte arrays + * + * @param featureSets List of feature set protobufs + * @return Byte array representation of the json strings + * @throws InvalidProtocolBufferException + */ + @Override + public byte[] toByte(List featureSets) + throws InvalidProtocolBufferException { + JsonFormat.Printer printer = + JsonFormat.printer().omittingInsignificantWhitespace().printingEnumsAsInts(); + List featureSetsJson = new ArrayList<>(); + for (FeatureSetProto.FeatureSet featureSet : featureSets) { + featureSetsJson.add(printer.print(featureSet.getSpec())); + } + return String.join("\n", featureSetsJson).getBytes(); + } +} diff --git a/core/src/main/java/feast/core/log/AuditLogger.java b/core/src/main/java/feast/core/log/AuditLogger.java index 5349b5548b0..275aa74edfa 100644 --- a/core/src/main/java/feast/core/log/AuditLogger.java +++ b/core/src/main/java/feast/core/log/AuditLogger.java @@ -16,7 +16,6 @@ */ package feast.core.log; -import com.google.common.base.Strings; import java.util.Date; import java.util.Map; import java.util.TreeMap; @@ -44,7 +43,7 @@ public static void log( map.put("resource", resource.toString()); map.put("id", id); map.put("action", action.toString()); - map.put("detail", Strings.lenientFormat(detail, args)); + map.put("detail", String.format(detail, args)); ObjectMessage msg = new ObjectMessage(map); log.log(AUDIT_LEVEL, msg); diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java index e4687050208..232a5f67d14 100644 --- a/core/src/main/java/feast/core/model/FeatureSet.java +++ b/core/src/main/java/feast/core/model/FeatureSet.java @@ -17,6 +17,7 @@ package feast.core.model; import com.google.protobuf.Duration; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; import feast.core.FeatureSetProto; import feast.core.FeatureSetProto.EntitySpec; @@ -24,7 +25,7 @@ import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.FeatureSetProto.FeatureSetStatus; import feast.core.FeatureSetProto.FeatureSpec; -import feast.types.ValueProto.ValueType; +import feast.types.ValueProto.ValueType.Enum; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -47,6 +48,21 @@ import org.apache.commons.lang3.builder.HashCodeBuilder; import org.hibernate.annotations.Fetch; import org.hibernate.annotations.FetchMode; +import org.tensorflow.metadata.v0.BoolDomain; +import org.tensorflow.metadata.v0.FeaturePresence; +import org.tensorflow.metadata.v0.FeaturePresenceWithinGroup; +import org.tensorflow.metadata.v0.FixedShape; +import org.tensorflow.metadata.v0.FloatDomain; +import org.tensorflow.metadata.v0.ImageDomain; +import org.tensorflow.metadata.v0.IntDomain; +import org.tensorflow.metadata.v0.MIDDomain; +import org.tensorflow.metadata.v0.NaturalLanguageDomain; +import org.tensorflow.metadata.v0.StringDomain; +import org.tensorflow.metadata.v0.StructDomain; +import org.tensorflow.metadata.v0.TimeDomain; +import org.tensorflow.metadata.v0.TimeOfDayDomain; +import org.tensorflow.metadata.v0.URLDomain; +import org.tensorflow.metadata.v0.ValueCount; @Getter @Setter @@ -157,14 +173,14 @@ public static FeatureSet fromProto(FeatureSetProto.FeatureSet featureSetProto) { FeatureSetSpec featureSetSpec = featureSetProto.getSpec(); Source source = Source.fromProto(featureSetSpec.getSource()); - List features = new ArrayList<>(); - for (FeatureSpec feature : featureSetSpec.getFeaturesList()) { - features.add(new Field(feature.getName(), feature.getValueType())); + List featureSpecs = new ArrayList<>(); + for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { + featureSpecs.add(new Field(featureSpec)); } - List entities = new ArrayList<>(); - for (EntitySpec entity : featureSetSpec.getEntitiesList()) { - entities.add(new Field(entity.getName(), entity.getValueType())); + List entitySpecs = new ArrayList<>(); + for (EntitySpec entitySpec : featureSetSpec.getEntitiesList()) { + entitySpecs.add(new Field(entitySpec)); } return new FeatureSet( @@ -172,8 +188,8 @@ public static FeatureSet fromProto(FeatureSetProto.FeatureSet featureSetProto) { featureSetProto.getSpec().getProject(), featureSetProto.getSpec().getVersion(), featureSetSpec.getMaxAge().getSeconds(), - entities, - features, + entitySpecs, + featureSpecs, source, featureSetProto.getMeta().getStatus()); } @@ -202,24 +218,21 @@ public void addFeature(Field field) { features.add(field); } - public FeatureSetProto.FeatureSet toProto() { + public FeatureSetProto.FeatureSet toProto() throws InvalidProtocolBufferException { List entitySpecs = new ArrayList<>(); - for (Field entity : entities) { - entitySpecs.add( - EntitySpec.newBuilder() - .setName(entity.getName()) - .setValueType(ValueType.Enum.valueOf(entity.getType())) - .build()); + for (Field entityField : entities) { + EntitySpec.Builder entitySpecBuilder = EntitySpec.newBuilder(); + setEntitySpecFields(entitySpecBuilder, entityField); + entitySpecs.add(entitySpecBuilder.build()); } List featureSpecs = new ArrayList<>(); - for (Field feature : features) { - featureSpecs.add( - FeatureSpec.newBuilder() - .setName(feature.getName()) - .setValueType(ValueType.Enum.valueOf(feature.getType())) - .build()); + for (Field featureField : features) { + FeatureSpec.Builder featureSpecBuilder = FeatureSpec.newBuilder(); + setFeatureSpecFields(featureSpecBuilder, featureField); + featureSpecs.add(featureSpecBuilder.build()); } + FeatureSetMeta.Builder meta = FeatureSetMeta.newBuilder() .setCreatedTimestamp( @@ -239,6 +252,108 @@ public FeatureSetProto.FeatureSet toProto() { return FeatureSetProto.FeatureSet.newBuilder().setMeta(meta).setSpec(spec).build(); } + // setEntitySpecFields and setFeatureSpecFields methods contain duplicated code because + // Feast internally treat EntitySpec and FeatureSpec as Field class. However, the proto message + // builder for EntitySpec and FeatureSpec are of different class. + @SuppressWarnings("DuplicatedCode") + private void setEntitySpecFields(EntitySpec.Builder entitySpecBuilder, Field entityField) + throws InvalidProtocolBufferException { + entitySpecBuilder + .setName(entityField.getName()) + .setValueType(Enum.valueOf(entityField.getType())); + + if (entityField.getPresence() != null) { + entitySpecBuilder.setPresence(FeaturePresence.parseFrom(entityField.getPresence())); + } else if (entityField.getGroupPresence() != null) { + entitySpecBuilder.setGroupPresence( + FeaturePresenceWithinGroup.parseFrom(entityField.getGroupPresence())); + } + + if (entityField.getShape() != null) { + entitySpecBuilder.setShape(FixedShape.parseFrom(entityField.getShape())); + } else if (entityField.getValueCount() != null) { + entitySpecBuilder.setValueCount(ValueCount.parseFrom(entityField.getValueCount())); + } + + if (entityField.getDomain() != null) { + entitySpecBuilder.setDomain(entityField.getDomain()); + } else if (entityField.getIntDomain() != null) { + entitySpecBuilder.setIntDomain(IntDomain.parseFrom(entityField.getIntDomain())); + } else if (entityField.getFloatDomain() != null) { + entitySpecBuilder.setFloatDomain(FloatDomain.parseFrom(entityField.getFloatDomain())); + } else if (entityField.getStringDomain() != null) { + entitySpecBuilder.setStringDomain(StringDomain.parseFrom(entityField.getStringDomain())); + } else if (entityField.getBoolDomain() != null) { + entitySpecBuilder.setBoolDomain(BoolDomain.parseFrom(entityField.getBoolDomain())); + } else if (entityField.getStructDomain() != null) { + entitySpecBuilder.setStructDomain(StructDomain.parseFrom(entityField.getStructDomain())); + } else if (entityField.getNaturalLanguageDomain() != null) { + entitySpecBuilder.setNaturalLanguageDomain( + NaturalLanguageDomain.parseFrom(entityField.getNaturalLanguageDomain())); + } else if (entityField.getImageDomain() != null) { + entitySpecBuilder.setImageDomain(ImageDomain.parseFrom(entityField.getImageDomain())); + } else if (entityField.getMidDomain() != null) { + entitySpecBuilder.setIntDomain(IntDomain.parseFrom(entityField.getIntDomain())); + } else if (entityField.getUrlDomain() != null) { + entitySpecBuilder.setUrlDomain(URLDomain.parseFrom(entityField.getUrlDomain())); + } else if (entityField.getTimeDomain() != null) { + entitySpecBuilder.setTimeDomain(TimeDomain.parseFrom(entityField.getTimeDomain())); + } else if (entityField.getTimeOfDayDomain() != null) { + entitySpecBuilder.setTimeOfDayDomain( + TimeOfDayDomain.parseFrom(entityField.getTimeOfDayDomain())); + } + } + + // Refer to setEntitySpecFields method for the reason for code duplication. + @SuppressWarnings("DuplicatedCode") + private void setFeatureSpecFields(FeatureSpec.Builder featureSpecBuilder, Field featureField) + throws InvalidProtocolBufferException { + featureSpecBuilder + .setName(featureField.getName()) + .setValueType(Enum.valueOf(featureField.getType())); + + if (featureField.getPresence() != null) { + featureSpecBuilder.setPresence(FeaturePresence.parseFrom(featureField.getPresence())); + } else if (featureField.getGroupPresence() != null) { + featureSpecBuilder.setGroupPresence( + FeaturePresenceWithinGroup.parseFrom(featureField.getGroupPresence())); + } + + if (featureField.getShape() != null) { + featureSpecBuilder.setShape(FixedShape.parseFrom(featureField.getShape())); + } else if (featureField.getValueCount() != null) { + featureSpecBuilder.setValueCount(ValueCount.parseFrom(featureField.getValueCount())); + } + + if (featureField.getDomain() != null) { + featureSpecBuilder.setDomain(featureField.getDomain()); + } else if (featureField.getIntDomain() != null) { + featureSpecBuilder.setIntDomain(IntDomain.parseFrom(featureField.getIntDomain())); + } else if (featureField.getFloatDomain() != null) { + featureSpecBuilder.setFloatDomain(FloatDomain.parseFrom(featureField.getFloatDomain())); + } else if (featureField.getStringDomain() != null) { + featureSpecBuilder.setStringDomain(StringDomain.parseFrom(featureField.getStringDomain())); + } else if (featureField.getBoolDomain() != null) { + featureSpecBuilder.setBoolDomain(BoolDomain.parseFrom(featureField.getBoolDomain())); + } else if (featureField.getStructDomain() != null) { + featureSpecBuilder.setStructDomain(StructDomain.parseFrom(featureField.getStructDomain())); + } else if (featureField.getNaturalLanguageDomain() != null) { + featureSpecBuilder.setNaturalLanguageDomain( + NaturalLanguageDomain.parseFrom(featureField.getNaturalLanguageDomain())); + } else if (featureField.getImageDomain() != null) { + featureSpecBuilder.setImageDomain(ImageDomain.parseFrom(featureField.getImageDomain())); + } else if (featureField.getMidDomain() != null) { + featureSpecBuilder.setMidDomain(MIDDomain.parseFrom(featureField.getMidDomain())); + } else if (featureField.getUrlDomain() != null) { + featureSpecBuilder.setUrlDomain(URLDomain.parseFrom(featureField.getUrlDomain())); + } else if (featureField.getTimeDomain() != null) { + featureSpecBuilder.setTimeDomain(TimeDomain.parseFrom(featureField.getTimeDomain())); + } else if (featureField.getTimeOfDayDomain() != null) { + featureSpecBuilder.setTimeOfDayDomain( + TimeOfDayDomain.parseFrom(featureField.getTimeOfDayDomain())); + } + } + /** * Checks if the given featureSet's schema and source has is different from this one. * diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java index 7573fcbf5e3..cb23e4eceb7 100644 --- a/core/src/main/java/feast/core/model/Field.java +++ b/core/src/main/java/feast/core/model/Field.java @@ -16,7 +16,10 @@ */ package feast.core.model; +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.FeatureSpec; import feast.types.ValueProto.ValueType; +import java.util.Arrays; import java.util.Objects; import javax.persistence.Column; import javax.persistence.Embeddable; @@ -44,6 +47,31 @@ public class Field { @Column(name = "project") private String project; + // Presence constraints (refer to proto feast.core.FeatureSet.FeatureSpec) + // Only one of them can be set. + private byte[] presence; + private byte[] groupPresence; + + // Shape type (refer to proto feast.core.FeatureSet.FeatureSpec) + // Only one of them can be set. + private byte[] shape; + private byte[] valueCount; + + // Domain info for the values (refer to proto feast.core.FeatureSet.FeatureSpec) + // Only one of them can be set. + private String domain; + private byte[] intDomain; + private byte[] floatDomain; + private byte[] stringDomain; + private byte[] boolDomain; + private byte[] structDomain; + private byte[] naturalLanguageDomain; + private byte[] imageDomain; + private byte[] midDomain; + private byte[] urlDomain; + private byte[] timeDomain; + private byte[] timeOfDayDomain; + public Field() {} public Field(String name, ValueType.Enum type) { @@ -51,6 +79,142 @@ public Field(String name, ValueType.Enum type) { this.type = type.toString(); } + public Field(FeatureSpec featureSpec) { + this.name = featureSpec.getName(); + this.type = featureSpec.getValueType().toString(); + + switch (featureSpec.getPresenceConstraintsCase()) { + case PRESENCE: + this.presence = featureSpec.getPresence().toByteArray(); + break; + case GROUP_PRESENCE: + this.groupPresence = featureSpec.getGroupPresence().toByteArray(); + break; + case PRESENCECONSTRAINTS_NOT_SET: + break; + } + + switch (featureSpec.getShapeTypeCase()) { + case SHAPE: + this.shape = featureSpec.getShape().toByteArray(); + break; + case VALUE_COUNT: + this.valueCount = featureSpec.getValueCount().toByteArray(); + break; + case SHAPETYPE_NOT_SET: + break; + } + + switch (featureSpec.getDomainInfoCase()) { + case DOMAIN: + this.domain = featureSpec.getDomain(); + break; + case INT_DOMAIN: + this.intDomain = featureSpec.getIntDomain().toByteArray(); + break; + case FLOAT_DOMAIN: + this.floatDomain = featureSpec.getFloatDomain().toByteArray(); + break; + case STRING_DOMAIN: + this.stringDomain = featureSpec.getStringDomain().toByteArray(); + break; + case BOOL_DOMAIN: + this.boolDomain = featureSpec.getBoolDomain().toByteArray(); + break; + case STRUCT_DOMAIN: + this.structDomain = featureSpec.getStructDomain().toByteArray(); + break; + case NATURAL_LANGUAGE_DOMAIN: + this.naturalLanguageDomain = featureSpec.getNaturalLanguageDomain().toByteArray(); + break; + case IMAGE_DOMAIN: + this.imageDomain = featureSpec.getImageDomain().toByteArray(); + break; + case MID_DOMAIN: + this.midDomain = featureSpec.getMidDomain().toByteArray(); + break; + case URL_DOMAIN: + this.urlDomain = featureSpec.getUrlDomain().toByteArray(); + break; + case TIME_DOMAIN: + this.timeDomain = featureSpec.getTimeDomain().toByteArray(); + break; + case TIME_OF_DAY_DOMAIN: + this.timeOfDayDomain = featureSpec.getTimeOfDayDomain().toByteArray(); + break; + case DOMAININFO_NOT_SET: + break; + } + } + + public Field(EntitySpec entitySpec) { + this.name = entitySpec.getName(); + this.type = entitySpec.getValueType().toString(); + + switch (entitySpec.getPresenceConstraintsCase()) { + case PRESENCE: + this.presence = entitySpec.getPresence().toByteArray(); + break; + case GROUP_PRESENCE: + this.groupPresence = entitySpec.getGroupPresence().toByteArray(); + break; + case PRESENCECONSTRAINTS_NOT_SET: + break; + } + + switch (entitySpec.getShapeTypeCase()) { + case SHAPE: + this.shape = entitySpec.getShape().toByteArray(); + break; + case VALUE_COUNT: + this.valueCount = entitySpec.getValueCount().toByteArray(); + break; + case SHAPETYPE_NOT_SET: + break; + } + + switch (entitySpec.getDomainInfoCase()) { + case DOMAIN: + this.domain = entitySpec.getDomain(); + break; + case INT_DOMAIN: + this.intDomain = entitySpec.getIntDomain().toByteArray(); + break; + case FLOAT_DOMAIN: + this.floatDomain = entitySpec.getFloatDomain().toByteArray(); + break; + case STRING_DOMAIN: + this.stringDomain = entitySpec.getStringDomain().toByteArray(); + break; + case BOOL_DOMAIN: + this.boolDomain = entitySpec.getBoolDomain().toByteArray(); + break; + case STRUCT_DOMAIN: + this.structDomain = entitySpec.getStructDomain().toByteArray(); + break; + case NATURAL_LANGUAGE_DOMAIN: + this.naturalLanguageDomain = entitySpec.getNaturalLanguageDomain().toByteArray(); + break; + case IMAGE_DOMAIN: + this.imageDomain = entitySpec.getImageDomain().toByteArray(); + break; + case MID_DOMAIN: + this.midDomain = entitySpec.getMidDomain().toByteArray(); + break; + case URL_DOMAIN: + this.urlDomain = entitySpec.getUrlDomain().toByteArray(); + break; + case TIME_DOMAIN: + this.timeDomain = entitySpec.getTimeDomain().toByteArray(); + break; + case TIME_OF_DAY_DOMAIN: + this.timeOfDayDomain = entitySpec.getTimeOfDayDomain().toByteArray(); + break; + case DOMAININFO_NOT_SET: + break; + } + } + @Override public boolean equals(Object o) { if (this == o) { @@ -60,7 +224,25 @@ public boolean equals(Object o) { return false; } Field field = (Field) o; - return name.equals(field.getName()) && type.equals(field.getType()); + return Objects.equals(name, field.name) + && Objects.equals(type, field.type) + && Objects.equals(project, field.project) + && Arrays.equals(presence, field.presence) + && Arrays.equals(groupPresence, field.groupPresence) + && Arrays.equals(shape, field.shape) + && Arrays.equals(valueCount, field.valueCount) + && Objects.equals(domain, field.domain) + && Arrays.equals(intDomain, field.intDomain) + && Arrays.equals(floatDomain, field.floatDomain) + && Arrays.equals(stringDomain, field.stringDomain) + && Arrays.equals(boolDomain, field.boolDomain) + && Arrays.equals(structDomain, field.structDomain) + && Arrays.equals(naturalLanguageDomain, field.naturalLanguageDomain) + && Arrays.equals(imageDomain, field.imageDomain) + && Arrays.equals(midDomain, field.midDomain) + && Arrays.equals(urlDomain, field.urlDomain) + && Arrays.equals(timeDomain, field.timeDomain) + && Arrays.equals(timeOfDayDomain, field.timeOfDayDomain); } @Override diff --git a/core/src/main/java/feast/core/service/AccessManagementService.java b/core/src/main/java/feast/core/service/AccessManagementService.java index 6f627df33d6..df92750e94f 100644 --- a/core/src/main/java/feast/core/service/AccessManagementService.java +++ b/core/src/main/java/feast/core/service/AccessManagementService.java @@ -71,7 +71,6 @@ public void archiveProject(String name) { * * @return List of active projects */ - @Transactional public List listProjects() { return projectRepository.findAllByArchivedIsFalse(); } diff --git a/core/src/main/java/feast/core/service/JobCoordinatorService.java b/core/src/main/java/feast/core/service/JobCoordinatorService.java index 23ad041b81d..3113d32a37d 100644 --- a/core/src/main/java/feast/core/service/JobCoordinatorService.java +++ b/core/src/main/java/feast/core/service/JobCoordinatorService.java @@ -16,6 +16,7 @@ */ package feast.core.service; +import com.google.protobuf.InvalidProtocolBufferException; import feast.core.CoreServiceProto.ListFeatureSetsRequest; import feast.core.CoreServiceProto.ListStoresRequest.Filter; import feast.core.CoreServiceProto.ListStoresResponse; @@ -87,7 +88,7 @@ public JobCoordinatorService( */ @Transactional @Scheduled(fixedDelay = POLLING_INTERVAL_MILLISECONDS) - public void Poll() { + public void Poll() throws InvalidProtocolBufferException { log.info("Polling for new jobs..."); List jobUpdateTasks = new ArrayList<>(); ListStoresResponse listStoresResponse = specService.listStores(Filter.newBuilder().build()); @@ -144,11 +145,16 @@ public void Poll() { } } catch (ExecutionException | InterruptedException e) { log.warn("Unable to start or update job: {}", e.getMessage()); + } catch (Exception e) { + log.info("Unexpeced Exception :{}", e.getMessage()); } completedTasks++; } - log.info("Updating feature set status"); + log.info( + "Updating feature set status. {} tasks completed out of {}", + jobUpdateTasks.size(), + completedTasks); updateFeatureSetStatuses(jobUpdateTasks); } diff --git a/core/src/main/java/feast/core/service/JobStatusService.java b/core/src/main/java/feast/core/service/JobStatusService.java new file mode 100644 index 00000000000..26d81647faa --- /dev/null +++ b/core/src/main/java/feast/core/service/JobStatusService.java @@ -0,0 +1,80 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class JobStatusService { + // + // private JobInfoRepository jobInfoRepository; + // private MetricsRepository metricsRepository; + // + // @Autowired + // public JobStatusService( + // JobInfoRepository jobInfoRepository, + // MetricsRepository metricsRepository) { + // this.jobInfoRepository = jobInfoRepository; + // this.metricsRepository = metricsRepository; + // } + // + // /** + // * Lists all jobs registered to the db, sorted by provided orderBy + // * + // * @param orderBy list order + // * @return list of JobDetails + // */ + // @Transactional + // public List listJobs(Sort orderBy) { + // List jobs = jobInfoRepository.findAll(orderBy); + // return jobs.stream().map(JobInfo::getJobDetail).collect(Collectors.toList()); + // } + // + // /** + // * Lists all jobs registered to the db, sorted chronologically by creation time + // * + // * @return list of JobDetails + // */ + // @Transactional + // public List listJobs() { + // return listJobs(Sort.by(Sort.Direction.ASC, "created")); + // } + // + // /** + // * Gets information regarding a single job. + // * + // * @param id feast-internal job id + // * @return JobDetail for that job + // */ + // @Transactional + // public JobDetail getJob(String id) { + // Optional job = jobInfoRepository.findById(id); + // if (!job.isPresent()) { + // throw new RetrievalException(String.format("Unable to retrieve job with id %s", + // id)); + // } + // JobDetail.Builder jobDetailBuilder = job.get().getJobDetail().toBuilder(); + // List metrics = metricsRepository.findByJobInfo_Id(id); + // for (Metrics metric : metrics) { + // jobDetailBuilder.putMetrics(metric.getName(), metric.getValue()); + // } + // return jobDetailBuilder.build(); + // } + +} diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index 1d6ce16de54..5b98d065977 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -86,7 +86,8 @@ public SpecService( * @param request: GetFeatureSetRequest Request containing filter parameters. * @return Returns a GetFeatureSetResponse containing a feature set.. */ - public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) { + public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) + throws InvalidProtocolBufferException { // Validate input arguments checkValidCharacters(request.getName(), "featureSetName"); @@ -143,14 +144,15 @@ public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) { * possible if a project name is not set explicitly * *

The version field can be one of - '*' - This will match all versions - 'latest' - This will - * match the latest feature set version - '<number>' - This will match a specific feature set - * version. This property can only be set if both the feature set name and project name are + * match the latest feature set version - '<number>' - This will match a specific feature + * set version. This property can only be set if both the feature set name and project name are * explicitly set. * * @param filter filter containing the desired featureSet name and version filter * @return ListFeatureSetsResponse with list of featureSets found matching the filter */ - public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter filter) { + public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter filter) + throws InvalidProtocolBufferException { String name = filter.getFeatureSetName(); String project = filter.getProject(); String version = filter.getFeatureSetVersion(); @@ -280,7 +282,8 @@ public ListStoresResponse listStores(ListStoresRequest.Filter filter) { * * @param newFeatureSet Feature set that will be created or updated. */ - public ApplyFeatureSetResponse applyFeatureSet(FeatureSetProto.FeatureSet newFeatureSet) { + public ApplyFeatureSetResponse applyFeatureSet(FeatureSetProto.FeatureSet newFeatureSet) + throws InvalidProtocolBufferException { // Validate incoming feature set FeatureSetValidator.validateSpec(newFeatureSet); diff --git a/core/src/main/java/feast/core/util/PackageUtil.java b/core/src/main/java/feast/core/util/PackageUtil.java index 20b2310644b..99c5d73ba78 100644 --- a/core/src/main/java/feast/core/util/PackageUtil.java +++ b/core/src/main/java/feast/core/util/PackageUtil.java @@ -44,9 +44,9 @@ public class PackageUtil { * points to the resource location. Note that the extraction process can take several minutes to * complete. * - *

One use case of this function is to detect the class path of resources to stage when - * using Dataflow runner. The resource URL however is in "jar:file:" format, which cannot be - * handled by default in Apache Beam. + *

One use case of this function is to detect the class path of resources to stage when using + * Dataflow runner. The resource URL however is in "jar:file:" format, which cannot be handled by + * default in Apache Beam. * *

    * 
diff --git a/core/src/main/java/feast/core/util/PipelineUtil.java b/core/src/main/java/feast/core/util/PipelineUtil.java
index ef1b50d8c9a..8a84caf672c 100644
--- a/core/src/main/java/feast/core/util/PipelineUtil.java
+++ b/core/src/main/java/feast/core/util/PipelineUtil.java
@@ -24,7 +24,9 @@
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 
 public class PipelineUtil {
 
@@ -45,12 +47,7 @@ public class PipelineUtil {
   public static List detectClassPathResourcesToStage(ClassLoader classLoader)
       throws IOException {
     if (!(classLoader instanceof URLClassLoader)) {
-      String message =
-          String.format(
-              "Unable to use ClassLoader to detect classpath elements. "
-                  + "Current ClassLoader is %s, only URLClassLoaders are supported.",
-              classLoader);
-      throw new IllegalArgumentException(message);
+      return getClasspathFiles();
     }
 
     List files = new ArrayList<>();
@@ -69,4 +66,10 @@ public static List detectClassPathResourcesToStage(ClassLoader classLoad
     }
     return files;
   }
+
+  private static List getClasspathFiles() {
+    return Arrays.stream(System.getProperty("java.class.path").split(File.pathSeparator))
+        .map(entry -> new File(entry).getPath())
+        .collect(Collectors.toList());
+  }
 }
diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java
index e01a5511359..5fe69819476 100644
--- a/core/src/main/java/feast/core/util/TypeConversion.java
+++ b/core/src/main/java/feast/core/util/TypeConversion.java
@@ -16,7 +16,6 @@
  */
 package feast.core.util;
 
-import com.google.common.base.Strings;
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 import java.lang.reflect.Type;
@@ -85,7 +84,7 @@ public static String convertMapToJsonString(Map map) {
   public static String[] convertMapToArgs(Map map) {
     List args = new ArrayList<>();
     for (Entry arg : map.entrySet()) {
-      args.add(Strings.lenientFormat("--%s=%s", arg.getKey(), arg.getValue()));
+      args.add(String.format("--%s=%s", arg.getKey(), arg.getValue()));
     }
     return args.toArray(new String[] {});
   }
diff --git a/core/src/main/proto/feast b/core/src/main/proto/feast
deleted file mode 120000
index d520da9126b..00000000000
--- a/core/src/main/proto/feast
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/feast
\ No newline at end of file
diff --git a/core/src/main/proto/third_party b/core/src/main/proto/third_party
deleted file mode 120000
index 363d20598e6..00000000000
--- a/core/src/main/proto/third_party
+++ /dev/null
@@ -1 +0,0 @@
-../../../../protos/third_party
\ No newline at end of file
diff --git a/core/src/main/resources/log4j2.xml b/core/src/main/resources/log4j2.xml
index 65b3c5aa4bb..efbf7d1f624 100644
--- a/core/src/main/resources/log4j2.xml
+++ b/core/src/main/resources/log4j2.xml
@@ -22,9 +22,10 @@
             %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex
         
         ${env:LOG_TYPE:-Console}
+        ${env:LOG_LEVEL:-info}
     
     
-        
+        
     
     
         
@@ -35,8 +36,11 @@
         
     
     
-        
-            
-        
+      
+        
+      
+      
+        
+      
     
 
diff --git a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java b/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java
index c263515ed08..9f26c6919e4 100644
--- a/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java
+++ b/core/src/test/java/feast/core/job/dataflow/DataflowJobManagerTest.java
@@ -19,11 +19,7 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
 import static org.mockito.MockitoAnnotations.initMocks;
 
 import com.google.api.services.dataflow.Dataflow;
@@ -44,14 +40,15 @@
 import feast.core.config.FeastProperties.MetricsProperties;
 import feast.core.exception.JobExecutionException;
 import feast.core.job.Runner;
-import feast.core.model.FeatureSet;
-import feast.core.model.Job;
-import feast.core.model.JobStatus;
-import feast.core.model.Source;
-import feast.core.model.Store;
+import feast.core.job.option.FeatureSetJsonByteConverter;
+import feast.core.model.*;
+import feast.ingestion.options.BZip2Compressor;
 import feast.ingestion.options.ImportOptions;
+import feast.ingestion.options.OptionCompressor;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import org.apache.beam.runners.dataflow.DataflowPipelineJob;
 import org.apache.beam.runners.dataflow.DataflowRunner;
@@ -131,8 +128,11 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException {
     expectedPipelineOptions.setAppName("DataflowJobManager");
     expectedPipelineOptions.setJobName(jobName);
     expectedPipelineOptions.setStoreJson(Lists.newArrayList(printer.print(store)));
+
+    OptionCompressor> featureSetJsonCompressor =
+        new BZip2Compressor<>(new FeatureSetJsonByteConverter());
     expectedPipelineOptions.setFeatureSetJson(
-        Lists.newArrayList(printer.print(featureSet.getSpec())));
+        featureSetJsonCompressor.compress(Collections.singletonList(featureSet)));
 
     ArgumentCaptor captor = ArgumentCaptor.forClass(ImportOptions.class);
 
@@ -170,7 +170,19 @@ public void shouldStartJobWithCorrectPipelineOptions() throws IOException {
     // Assume the files that are staged are correct
     expectedPipelineOptions.setFilesToStage(actualPipelineOptions.getFilesToStage());
 
-    assertThat(actualPipelineOptions.toString(), equalTo(expectedPipelineOptions.toString()));
+    assertThat(
+        actualPipelineOptions.getFeatureSetJson(),
+        equalTo(expectedPipelineOptions.getFeatureSetJson()));
+    assertThat(
+        actualPipelineOptions.getDeadLetterTableSpec(),
+        equalTo(expectedPipelineOptions.getDeadLetterTableSpec()));
+    assertThat(
+        actualPipelineOptions.getStatsdHost(), equalTo(expectedPipelineOptions.getStatsdHost()));
+    assertThat(
+        actualPipelineOptions.getMetricsExporterType(),
+        equalTo(expectedPipelineOptions.getMetricsExporterType()));
+    assertThat(
+        actualPipelineOptions.getStoreJson(), equalTo(expectedPipelineOptions.getStoreJson()));
     assertThat(actual.getExtId(), equalTo(expectedExtJobId));
   }
 
diff --git a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java
index 2dd87cfc6e3..64412f4391e 100644
--- a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java
+++ b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java
@@ -40,14 +40,19 @@
 import feast.core.StoreProto.Store.Subscription;
 import feast.core.config.FeastProperties.MetricsProperties;
 import feast.core.job.Runner;
+import feast.core.job.option.FeatureSetJsonByteConverter;
 import feast.core.model.FeatureSet;
 import feast.core.model.Job;
 import feast.core.model.JobStatus;
 import feast.core.model.Source;
 import feast.core.model.Store;
+import feast.ingestion.options.BZip2Compressor;
 import feast.ingestion.options.ImportOptions;
+import feast.ingestion.options.OptionCompressor;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import org.apache.beam.runners.direct.DirectRunner;
 import org.apache.beam.sdk.PipelineResult;
@@ -121,8 +126,11 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException {
     expectedPipelineOptions.setProject("");
     expectedPipelineOptions.setStoreJson(Lists.newArrayList(printer.print(store)));
     expectedPipelineOptions.setProject("");
+
+    OptionCompressor> featureSetJsonCompressor =
+        new BZip2Compressor<>(new FeatureSetJsonByteConverter());
     expectedPipelineOptions.setFeatureSetJson(
-        Lists.newArrayList(printer.print(featureSet.getSpec())));
+        featureSetJsonCompressor.compress(Collections.singletonList(featureSet)));
 
     String expectedJobId = "feast-job-0";
     ArgumentCaptor pipelineOptionsCaptor =
@@ -150,7 +158,20 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException {
     expectedPipelineOptions.setOptionsId(
         actualPipelineOptions.getOptionsId()); // avoid comparing this value
 
-    assertThat(actualPipelineOptions.toString(), equalTo(expectedPipelineOptions.toString()));
+    assertThat(
+        actualPipelineOptions.getFeatureSetJson(),
+        equalTo(expectedPipelineOptions.getFeatureSetJson()));
+    assertThat(
+        actualPipelineOptions.getDeadLetterTableSpec(),
+        equalTo(expectedPipelineOptions.getDeadLetterTableSpec()));
+    assertThat(
+        actualPipelineOptions.getStatsdHost(), equalTo(expectedPipelineOptions.getStatsdHost()));
+    assertThat(
+        actualPipelineOptions.getMetricsExporterType(),
+        equalTo(expectedPipelineOptions.getMetricsExporterType()));
+    assertThat(
+        actualPipelineOptions.getStoreJson(), equalTo(expectedPipelineOptions.getStoreJson()));
+
     assertThat(jobStarted.getPipelineResult(), equalTo(mockPipelineResult));
     assertThat(jobStarted.getJobId(), equalTo(expectedJobId));
     assertThat(actual.getExtId(), equalTo(expectedJobId));
diff --git a/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java b/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java
new file mode 100644
index 00000000000..2dfeef1d969
--- /dev/null
+++ b/core/src/test/java/feast/core/job/option/FeatureSetJsonByteConverterTest.java
@@ -0,0 +1,83 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright 2018-2020 The Feast Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package feast.core.job.option;
+
+import static org.junit.Assert.*;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import feast.core.FeatureSetProto;
+import feast.core.SourceProto;
+import feast.types.ValueProto;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.junit.Test;
+
+public class FeatureSetJsonByteConverterTest {
+
+  private FeatureSetProto.FeatureSet newFeatureSet(Integer version, Integer numberOfFeatures) {
+    List features =
+        IntStream.range(1, numberOfFeatures + 1)
+            .mapToObj(
+                i ->
+                    FeatureSetProto.FeatureSpec.newBuilder()
+                        .setValueType(ValueProto.ValueType.Enum.FLOAT)
+                        .setName("feature".concat(Integer.toString(i)))
+                        .build())
+            .collect(Collectors.toList());
+
+    return FeatureSetProto.FeatureSet.newBuilder()
+        .setSpec(
+            FeatureSetProto.FeatureSetSpec.newBuilder()
+                .setSource(
+                    SourceProto.Source.newBuilder()
+                        .setType(SourceProto.SourceType.KAFKA)
+                        .setKafkaSourceConfig(
+                            SourceProto.KafkaSourceConfig.newBuilder()
+                                .setBootstrapServers("somebrokers:9092")
+                                .setTopic("sometopic")))
+                .addAllFeatures(features)
+                .setVersion(version)
+                .addEntities(
+                    FeatureSetProto.EntitySpec.newBuilder()
+                        .setName("entity")
+                        .setValueType(ValueProto.ValueType.Enum.STRING)))
+        .build();
+  }
+
+  @Test
+  public void shouldConvertFeatureSetsAsJsonStringBytes() throws InvalidProtocolBufferException {
+    int nrOfFeatureSet = 1;
+    int nrOfFeatures = 1;
+    List featureSets =
+        IntStream.range(1, nrOfFeatureSet + 1)
+            .mapToObj(i -> newFeatureSet(i, nrOfFeatures))
+            .collect(Collectors.toList());
+
+    String expectedOutputString =
+        "{\"version\":1,"
+            + "\"entities\":[{\"name\":\"entity\",\"valueType\":2}],"
+            + "\"features\":[{\"name\":\"feature1\",\"valueType\":6}],"
+            + "\"source\":{"
+            + "\"type\":1,"
+            + "\"kafkaSourceConfig\":{"
+            + "\"bootstrapServers\":\"somebrokers:9092\","
+            + "\"topic\":\"sometopic\"}}}";
+    FeatureSetJsonByteConverter byteConverter = new FeatureSetJsonByteConverter();
+    assertEquals(expectedOutputString, new String(byteConverter.toByte(featureSets)));
+  }
+}
diff --git a/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java b/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java
index 775cb028b02..67a87e93167 100644
--- a/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java
+++ b/core/src/test/java/feast/core/service/JobCoordinatorServiceTest.java
@@ -75,7 +75,7 @@ public void setUp() {
   }
 
   @Test
-  public void shouldDoNothingIfNoStoresFound() {
+  public void shouldDoNothingIfNoStoresFound() throws InvalidProtocolBufferException {
     when(specService.listStores(any())).thenReturn(ListStoresResponse.newBuilder().build());
     JobCoordinatorService jcs =
         new JobCoordinatorService(
diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java
index edd99aa4940..1eb56caac26 100644
--- a/core/src/test/java/feast/core/service/SpecServiceTest.java
+++ b/core/src/test/java/feast/core/service/SpecServiceTest.java
@@ -37,6 +37,7 @@
 import feast.core.CoreServiceProto.UpdateStoreRequest;
 import feast.core.CoreServiceProto.UpdateStoreResponse;
 import feast.core.FeatureSetProto;
+import feast.core.FeatureSetProto.EntitySpec;
 import feast.core.FeatureSetProto.FeatureSetSpec;
 import feast.core.FeatureSetProto.FeatureSetStatus;
 import feast.core.FeatureSetProto.FeatureSpec;
@@ -61,7 +62,11 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.stream.Collectors;
 import org.junit.Before;
@@ -71,6 +76,21 @@
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mock;
+import org.tensorflow.metadata.v0.BoolDomain;
+import org.tensorflow.metadata.v0.FeaturePresence;
+import org.tensorflow.metadata.v0.FeaturePresenceWithinGroup;
+import org.tensorflow.metadata.v0.FixedShape;
+import org.tensorflow.metadata.v0.FloatDomain;
+import org.tensorflow.metadata.v0.ImageDomain;
+import org.tensorflow.metadata.v0.IntDomain;
+import org.tensorflow.metadata.v0.MIDDomain;
+import org.tensorflow.metadata.v0.NaturalLanguageDomain;
+import org.tensorflow.metadata.v0.StringDomain;
+import org.tensorflow.metadata.v0.StructDomain;
+import org.tensorflow.metadata.v0.TimeDomain;
+import org.tensorflow.metadata.v0.TimeOfDayDomain;
+import org.tensorflow.metadata.v0.URLDomain;
+import org.tensorflow.metadata.v0.ValueCount;
 
 public class SpecServiceTest {
 
@@ -163,7 +183,8 @@ public void setUp() {
   }
 
   @Test
-  public void shouldGetAllFeatureSetsIfOnlyWildcardsProvided() {
+  public void shouldGetAllFeatureSetsIfOnlyWildcardsProvided()
+      throws InvalidProtocolBufferException {
     ListFeatureSetsResponse actual =
         specService.listFeatureSets(
             Filter.newBuilder()
@@ -182,7 +203,8 @@ public void shouldGetAllFeatureSetsIfOnlyWildcardsProvided() {
   }
 
   @Test
-  public void listFeatureSetShouldFailIfFeatureSetProvidedWithoutProject() {
+  public void listFeatureSetShouldFailIfFeatureSetProvidedWithoutProject()
+      throws InvalidProtocolBufferException {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage(
         "Invalid listFeatureSetRequest, missing arguments. Must provide project, feature set name, and version.");
@@ -191,7 +213,8 @@ public void listFeatureSetShouldFailIfFeatureSetProvidedWithoutProject() {
   }
 
   @Test
-  public void shouldGetAllFeatureSetsMatchingNameIfWildcardVersionProvided() {
+  public void shouldGetAllFeatureSetsMatchingNameIfWildcardVersionProvided()
+      throws InvalidProtocolBufferException {
     ListFeatureSetsResponse actual =
         specService.listFeatureSets(
             Filter.newBuilder()
@@ -212,7 +235,8 @@ public void shouldGetAllFeatureSetsMatchingNameIfWildcardVersionProvided() {
   }
 
   @Test
-  public void shouldGetAllFeatureSetsMatchingNameWithWildcardSearch() {
+  public void shouldGetAllFeatureSetsMatchingNameWithWildcardSearch()
+      throws InvalidProtocolBufferException {
     ListFeatureSetsResponse actual =
         specService.listFeatureSets(
             Filter.newBuilder()
@@ -235,7 +259,8 @@ public void shouldGetAllFeatureSetsMatchingNameWithWildcardSearch() {
   }
 
   @Test
-  public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator() {
+  public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator()
+      throws InvalidProtocolBufferException {
     ListFeatureSetsResponse actual =
         specService.listFeatureSets(
             Filter.newBuilder()
@@ -259,7 +284,8 @@ public void shouldGetAllFeatureSetsMatchingVersionIfNoComparator() {
   }
 
   @Test
-  public void shouldThrowExceptionIfGetAllFeatureSetsGivenVersionWithComparator() {
+  public void shouldThrowExceptionIfGetAllFeatureSetsGivenVersionWithComparator()
+      throws InvalidProtocolBufferException {
     expectedException.expect(IllegalArgumentException.class);
     specService.listFeatureSets(
         Filter.newBuilder()
@@ -270,7 +296,8 @@ public void shouldThrowExceptionIfGetAllFeatureSetsGivenVersionWithComparator()
   }
 
   @Test
-  public void shouldGetLatestFeatureSetGivenMissingVersionFilter() {
+  public void shouldGetLatestFeatureSetGivenMissingVersionFilter()
+      throws InvalidProtocolBufferException {
     GetFeatureSetResponse actual =
         specService.getFeatureSet(
             GetFeatureSetRequest.newBuilder().setName("f1").setProject("project1").build());
@@ -279,7 +306,8 @@ public void shouldGetLatestFeatureSetGivenMissingVersionFilter() {
   }
 
   @Test
-  public void shouldGetSpecificFeatureSetGivenSpecificVersionFilter() {
+  public void shouldGetSpecificFeatureSetGivenSpecificVersionFilter()
+      throws InvalidProtocolBufferException {
     when(featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion("f1", "project1", 2))
         .thenReturn(featureSets.get(1));
     GetFeatureSetResponse actual =
@@ -294,14 +322,15 @@ public void shouldGetSpecificFeatureSetGivenSpecificVersionFilter() {
   }
 
   @Test
-  public void shouldThrowExceptionGivenMissingFeatureSetName() {
+  public void shouldThrowExceptionGivenMissingFeatureSetName()
+      throws InvalidProtocolBufferException {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage("No feature set name provided");
     specService.getFeatureSet(GetFeatureSetRequest.newBuilder().setVersion(2).build());
   }
 
   @Test
-  public void shouldThrowExceptionGivenMissingFeatureSet() {
+  public void shouldThrowExceptionGivenMissingFeatureSet() throws InvalidProtocolBufferException {
     expectedException.expect(RetrievalException.class);
     expectedException.expectMessage(
         "Feature set with name \"f1000\" and version \"2\" could not be found.");
@@ -314,7 +343,8 @@ public void shouldThrowExceptionGivenMissingFeatureSet() {
   }
 
   @Test
-  public void shouldThrowRetrievalExceptionGivenInvalidFeatureSetVersionComparator() {
+  public void shouldThrowRetrievalExceptionGivenInvalidFeatureSetVersionComparator()
+      throws InvalidProtocolBufferException {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage(
         "Invalid listFeatureSetRequest. Version must be set to \"*\" if the project name and feature set name aren't set explicitly: \n"
@@ -361,7 +391,8 @@ public void shouldThrowRetrievalExceptionIfNoStoresFoundWithName() {
   }
 
   @Test
-  public void applyFeatureSetShouldReturnFeatureSetWithLatestVersionIfFeatureSetHasNotChanged() {
+  public void applyFeatureSetShouldReturnFeatureSetWithLatestVersionIfFeatureSetHasNotChanged()
+      throws InvalidProtocolBufferException {
     FeatureSetSpec incomingFeatureSetSpec =
         featureSets.get(2).toProto().getSpec().toBuilder().clearVersion().build();
 
@@ -375,7 +406,8 @@ public void applyFeatureSetShouldReturnFeatureSetWithLatestVersionIfFeatureSetHa
   }
 
   @Test
-  public void applyFeatureSetShouldApplyFeatureSetWithInitVersionIfNotExists() {
+  public void applyFeatureSetShouldApplyFeatureSetWithInitVersionIfNotExists()
+      throws InvalidProtocolBufferException {
     when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
             "f2", "project1"))
         .thenReturn(Lists.newArrayList());
@@ -408,7 +440,8 @@ public void applyFeatureSetShouldApplyFeatureSetWithInitVersionIfNotExists() {
   }
 
   @Test
-  public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists() {
+  public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists()
+      throws InvalidProtocolBufferException {
     FeatureSetProto.FeatureSet incomingFeatureSet = featureSets.get(2).toProto();
     incomingFeatureSet =
         incomingFeatureSet
@@ -450,7 +483,8 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists() {
   }
 
   @Test
-  public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered() {
+  public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered()
+      throws InvalidProtocolBufferException {
 
     Field f3f1 = new Field("f3f1", Enum.INT64);
     Field f3f2 = new Field("f3f2", Enum.INT64);
@@ -481,6 +515,197 @@ public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered() {
         equalTo(incomingFeatureSet.getSpec().getName()));
   }
 
+  @Test
+  public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints()
+      throws InvalidProtocolBufferException {
+    List entitySpecs = new ArrayList<>();
+    entitySpecs.add(
+        EntitySpec.newBuilder()
+            .setName("entity1")
+            .setValueType(Enum.INT64)
+            .setPresence(FeaturePresence.getDefaultInstance())
+            .setShape(FixedShape.getDefaultInstance())
+            .setDomain("mydomain")
+            .build());
+    entitySpecs.add(
+        EntitySpec.newBuilder()
+            .setName("entity2")
+            .setValueType(Enum.INT64)
+            .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance())
+            .setValueCount(ValueCount.getDefaultInstance())
+            .setIntDomain(IntDomain.getDefaultInstance())
+            .build());
+    entitySpecs.add(
+        EntitySpec.newBuilder()
+            .setName("entity3")
+            .setValueType(Enum.FLOAT)
+            .setPresence(FeaturePresence.getDefaultInstance())
+            .setValueCount(ValueCount.getDefaultInstance())
+            .setFloatDomain(FloatDomain.getDefaultInstance())
+            .build());
+    entitySpecs.add(
+        EntitySpec.newBuilder()
+            .setName("entity4")
+            .setValueType(Enum.STRING)
+            .setPresence(FeaturePresence.getDefaultInstance())
+            .setValueCount(ValueCount.getDefaultInstance())
+            .setStringDomain(StringDomain.getDefaultInstance())
+            .build());
+    entitySpecs.add(
+        EntitySpec.newBuilder()
+            .setName("entity5")
+            .setValueType(Enum.BOOL)
+            .setPresence(FeaturePresence.getDefaultInstance())
+            .setValueCount(ValueCount.getDefaultInstance())
+            .setBoolDomain(BoolDomain.getDefaultInstance())
+            .build());
+
+    List featureSpecs = new ArrayList<>();
+    featureSpecs.add(
+        FeatureSpec.newBuilder()
+            .setName("feature1")
+            .setValueType(Enum.INT64)
+            .setPresence(FeaturePresence.getDefaultInstance())
+            .setShape(FixedShape.getDefaultInstance())
+            .setDomain("mydomain")
+            .build());
+    featureSpecs.add(
+        FeatureSpec.newBuilder()
+            .setName("feature2")
+            .setValueType(Enum.INT64)
+            .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance())
+            .setValueCount(ValueCount.getDefaultInstance())
+            .setIntDomain(IntDomain.getDefaultInstance())
+            .build());
+    featureSpecs.add(
+        FeatureSpec.newBuilder()
+            .setName("feature3")
+            .setValueType(Enum.FLOAT)
+            .setPresence(FeaturePresence.getDefaultInstance())
+            .setValueCount(ValueCount.getDefaultInstance())
+            .setFloatDomain(FloatDomain.getDefaultInstance())
+            .build());
+    featureSpecs.add(
+        FeatureSpec.newBuilder()
+            .setName("feature4")
+            .setValueType(Enum.STRING)
+            .setPresence(FeaturePresence.getDefaultInstance())
+            .setValueCount(ValueCount.getDefaultInstance())
+            .setStringDomain(StringDomain.getDefaultInstance())
+            .build());
+    featureSpecs.add(
+        FeatureSpec.newBuilder()
+            .setName("feature5")
+            .setValueType(Enum.BOOL)
+            .setPresence(FeaturePresence.getDefaultInstance())
+            .setValueCount(ValueCount.getDefaultInstance())
+            .setBoolDomain(BoolDomain.getDefaultInstance())
+            .build());
+
+    FeatureSetSpec featureSetSpec =
+        FeatureSetSpec.newBuilder()
+            .setProject("project1")
+            .setName("featureSetWithConstraints")
+            .addAllEntities(entitySpecs)
+            .addAllFeatures(featureSpecs)
+            .build();
+    FeatureSetProto.FeatureSet featureSet =
+        FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build();
+
+    ApplyFeatureSetResponse applyFeatureSetResponse = specService.applyFeatureSet(featureSet);
+    FeatureSetSpec appliedFeatureSetSpec = applyFeatureSetResponse.getFeatureSet().getSpec();
+
+    // appliedEntitySpecs needs to be sorted because the list returned by specService may not
+    // follow the order in the request
+    List appliedEntitySpecs = new ArrayList<>(appliedFeatureSetSpec.getEntitiesList());
+    appliedEntitySpecs.sort(Comparator.comparing(EntitySpec::getName));
+
+    // appliedFeatureSpecs needs to be sorted because the list returned by specService may not
+    // follow the order in the request
+    List appliedFeatureSpecs =
+        new ArrayList<>(appliedFeatureSetSpec.getFeaturesList());
+    appliedFeatureSpecs.sort(Comparator.comparing(FeatureSpec::getName));
+
+    assertEquals(appliedEntitySpecs.size(), entitySpecs.size());
+    assertEquals(appliedFeatureSpecs.size(), featureSpecs.size());
+
+    for (int i = 0; i < appliedEntitySpecs.size(); i++) {
+      assertEquals(entitySpecs.get(i), appliedEntitySpecs.get(i));
+    }
+
+    for (int i = 0; i < appliedFeatureSpecs.size(); i++) {
+      assertEquals(featureSpecs.get(i), appliedFeatureSpecs.get(i));
+    }
+  }
+
+  @Test
+  public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated()
+      throws InvalidProtocolBufferException {
+    FeatureSetProto.FeatureSet existingFeatureSet = featureSets.get(2).toProto();
+    assertThat(
+        "Existing feature set has version 3", existingFeatureSet.getSpec().getVersion() == 3);
+    assertThat(
+        "Existing feature set has at least 1 feature",
+        existingFeatureSet.getSpec().getFeaturesList().size() > 0);
+
+    // Map of constraint field name -> value, e.g. "shape" -> FixedShape object.
+    // If any of these fields are updated, SpecService should update the FeatureSet.
+    Map contraintUpdates = new HashMap<>();
+    contraintUpdates.put("presence", FeaturePresence.newBuilder().setMinFraction(0.5).build());
+    contraintUpdates.put(
+        "group_presence", FeaturePresenceWithinGroup.newBuilder().setRequired(true).build());
+    contraintUpdates.put("shape", FixedShape.getDefaultInstance());
+    contraintUpdates.put("value_count", ValueCount.newBuilder().setMin(2).build());
+    contraintUpdates.put("domain", "new_domain");
+    contraintUpdates.put("int_domain", IntDomain.newBuilder().setMax(100).build());
+    contraintUpdates.put("float_domain", FloatDomain.newBuilder().setMin(-0.5f).build());
+    contraintUpdates.put("string_domain", StringDomain.newBuilder().addValue("string1").build());
+    contraintUpdates.put("bool_domain", BoolDomain.newBuilder().setFalseValue("falsy").build());
+    contraintUpdates.put("struct_domain", StructDomain.getDefaultInstance());
+    contraintUpdates.put("natural_language_domain", NaturalLanguageDomain.getDefaultInstance());
+    contraintUpdates.put("image_domain", ImageDomain.getDefaultInstance());
+    contraintUpdates.put("mid_domain", MIDDomain.getDefaultInstance());
+    contraintUpdates.put("url_domain", URLDomain.getDefaultInstance());
+    contraintUpdates.put(
+        "time_domain", TimeDomain.newBuilder().setStringFormat("string_format").build());
+    contraintUpdates.put("time_of_day_domain", TimeOfDayDomain.getDefaultInstance());
+
+    for (Entry constraint : contraintUpdates.entrySet()) {
+      String name = constraint.getKey();
+      Object value = constraint.getValue();
+      FeatureSpec newFeatureSpec =
+          existingFeatureSet
+              .getSpec()
+              .getFeatures(0)
+              .toBuilder()
+              .setField(FeatureSpec.getDescriptor().findFieldByName(name), value)
+              .build();
+      FeatureSetSpec newFeatureSetSpec =
+          existingFeatureSet.getSpec().toBuilder().setFeatures(0, newFeatureSpec).build();
+      FeatureSetProto.FeatureSet newFeatureSet =
+          existingFeatureSet.toBuilder().setSpec(newFeatureSetSpec).build();
+
+      ApplyFeatureSetResponse response = specService.applyFeatureSet(newFeatureSet);
+
+      assertEquals(
+          "Response should have CREATED status when field '" + name + "' is updated",
+          Status.CREATED,
+          response.getStatus());
+      assertEquals(
+          "FeatureSet should have new version when field '" + name + "' is updated",
+          existingFeatureSet.getSpec().getVersion() + 1,
+          response.getFeatureSet().getSpec().getVersion());
+      assertEquals(
+          "Feature should have field '" + name + "' set correctly",
+          constraint.getValue(),
+          response
+              .getFeatureSet()
+              .getSpec()
+              .getFeatures(0)
+              .getField(FeatureSpec.getDescriptor().findFieldByName(name)));
+    }
+  }
+
   @Test
   public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferException {
     when(storeRepository.findById("SERVING")).thenReturn(Optional.of(stores.get(0)));
@@ -521,7 +746,7 @@ public void shouldDoNothingIfNoChange() throws InvalidProtocolBufferException {
   }
 
   @Test
-  public void shouldFailIfGetFeatureSetWithoutProject() {
+  public void shouldFailIfGetFeatureSetWithoutProject() throws InvalidProtocolBufferException {
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectMessage("No project provided");
     specService.getFeatureSet(GetFeatureSetRequest.newBuilder().setName("f1").build());
@@ -530,6 +755,7 @@ public void shouldFailIfGetFeatureSetWithoutProject() {
   private FeatureSet newDummyFeatureSet(String name, int version, String project) {
     Field feature = new Field("feature", Enum.INT64);
     Field entity = new Field("entity", Enum.STRING);
+
     FeatureSet fs =
         new FeatureSet(
             name,
diff --git a/core/src/test/java/feast/core/validators/MatchersTest.java b/core/src/test/java/feast/core/validators/MatchersTest.java
index 774e58c7a87..3bf09dd474f 100644
--- a/core/src/test/java/feast/core/validators/MatchersTest.java
+++ b/core/src/test/java/feast/core/validators/MatchersTest.java
@@ -19,7 +19,6 @@
 import static feast.core.validators.Matchers.checkLowerSnakeCase;
 import static feast.core.validators.Matchers.checkUpperSnakeCase;
 
-import com.google.common.base.Strings;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -43,7 +42,7 @@ public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCaseWithNumbers() {
   public void checkUpperSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() {
     exception.expect(IllegalArgumentException.class);
     exception.expectMessage(
-        Strings.lenientFormat(
+        String.format(
             "invalid value for field %s: %s",
             "someField",
             "argument must be in upper snake case, and cannot include any special characters."));
@@ -61,7 +60,7 @@ public void checkLowerSnakeCaseShouldPassForLegitLowerSnakeCase() {
   public void checkLowerSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForInvalidString() {
     exception.expect(IllegalArgumentException.class);
     exception.expectMessage(
-        Strings.lenientFormat(
+        String.format(
             "invalid value for field %s: %s",
             "someField",
             "argument must be in lower snake case, and cannot include any special characters."));
diff --git a/datatypes/java/README.md b/datatypes/java/README.md
new file mode 100644
index 00000000000..535fac73d2e
--- /dev/null
+++ b/datatypes/java/README.md
@@ -0,0 +1,55 @@
+Feast Data Types for Java
+=========================
+
+This module produces Java class files for Feast's data type and gRPC service
+definitions, from Protobuf IDL. These are used across Feast components for wire
+interchange, contracts, etc.
+
+End users of Feast will be best served by our Java SDK which adds higher-level
+conveniences, but the data types are published independently for custom needs,
+without any additional dependencies the SDK may add.
+
+Dependency Coordinates
+----------------------
+
+```xml
+
+  dev.feast
+  datatypes-java
+  0.4.0-SNAPSHOT
+
+```
+
+Use the version corresponding to the Feast release you have deployed in your
+environment—see the [Feast release notes] for details.
+
+[Feast release notes]: ../../CHANGELOG.md
+
+Using the `.proto` Definitions
+------------------------------
+
+The `.proto` definitions are packaged as resources within the Maven artifact,
+which may be useful to `include` them in dependent Protobuf definitions in a
+downstream project, or for other JVM languages to consume from their builds to
+generate more idiomatic bindings.
+
+Google's Gradle plugin, for instance, [can use protos in dependencies][Gradle]
+either for `include` or to compile with a different `protoc` plugin than Java.
+
+[sbt-protoc] offers similar functionality for sbt/Scala.
+
+[Gradle]: https://github.com/google/protobuf-gradle-plugin#protos-in-dependencies
+[sbt-protoc]: https://github.com/thesamet/sbt-protoc
+
+Releases
+--------
+
+The module is published to Maven Central upon each release of Feast (since
+v0.3.7).
+
+For developers, the publishing process is automated along with the Java SDK by
+[the `publish-java-sdk` build task in Prow][prow task], where you can see how
+it works. Artifacts are staged to Sonatype where a maintainer needs to take a
+release action for them to go live on Maven Central.
+
+[prow task]: https://github.com/gojek/feast/blob/17e7dca8238aae4dcbf0ff9f0db5d80ef8e035cf/.prow/config.yaml#L166-L192
diff --git a/datatypes/java/pom.xml b/datatypes/java/pom.xml
new file mode 100644
index 00000000000..5810a6db96a
--- /dev/null
+++ b/datatypes/java/pom.xml
@@ -0,0 +1,111 @@
+
+
+
+    4.0.0
+
+    Feast Data Types for Java
+    
+        Data types and service contracts used throughout Feast components and
+        their interchanges. These are generated from Protocol Buffers and gRPC
+        definitions included in the package.
+    
+    datatypes-java
+
+    
+      dev.feast
+      feast-parent
+      ${revision}
+      ../..
+    
+
+    
+      
+        
+          org.apache.maven.plugins
+          maven-dependency-plugin
+          
+            
+            
+              javax.annotation
+            
+          
+        
+
+        
+          org.xolstice.maven.plugins
+          protobuf-maven-plugin
+          
+            true
+            
+                com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier}
+            
+            grpc-java
+            
+                io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier}
+            
+          
+          
+            
+              
+                compile
+                compile-custom
+                test-compile
+              
+            
+          
+        
+      
+    
+
+    
+      
+      
+        com.google.guava
+        guava
+      
+      
+        com.google.protobuf
+        protobuf-java
+      
+
+      
+        io.grpc
+        grpc-core
+      
+      
+        io.grpc
+        grpc-protobuf
+      
+      
+        io.grpc
+        grpc-services
+      
+      
+        io.grpc
+        grpc-stub
+      
+
+      
+          javax.annotation
+          javax.annotation-api
+      
+    
+
+
diff --git a/sdk/java/src/main/proto/feast b/datatypes/java/src/main/proto/feast
similarity index 100%
rename from sdk/java/src/main/proto/feast
rename to datatypes/java/src/main/proto/feast
diff --git a/datatypes/java/src/main/proto/tensorflow_metadata b/datatypes/java/src/main/proto/tensorflow_metadata
new file mode 120000
index 00000000000..a633bb850f3
--- /dev/null
+++ b/datatypes/java/src/main/proto/tensorflow_metadata
@@ -0,0 +1 @@
+../../../../../protos/tensorflow_metadata
\ No newline at end of file
diff --git a/datatypes/java/src/main/proto/third_party b/datatypes/java/src/main/proto/third_party
new file mode 120000
index 00000000000..f015f8477d1
--- /dev/null
+++ b/datatypes/java/src/main/proto/third_party
@@ -0,0 +1 @@
+../../../../../protos/third_party
\ No newline at end of file
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index 535fbe6081c..3cab9a11927 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -5,10 +5,17 @@
 * [Concepts](concepts.md)
 * [Getting Help](getting-help.md)
 * [Contributing](contributing.md)
+* [Roadmap](roadmap.md)
 
-## Getting Started
+## Installing Feast
+
+* [Overview](installing-feast/overview.md)
+* [Docker Compose](installing-feast/docker-compose.md)
+* [Google Kubernetes Engine \(GKE\)](installing-feast/gke.md)
+* [Troubleshooting](installing-feast/troubleshooting.md)
+
+## Using Feast
 
-* [Installing Feast](getting-started/installing-feast.md)
 * [Using Feast](https://github.com/gojek/feast/blob/master/examples/basic/basic.ipynb)
 
 ## Reference
diff --git a/docs/concepts.md b/docs/concepts.md
index 860515c3699..ae158f8f829 100644
--- a/docs/concepts.md
+++ b/docs/concepts.md
@@ -2,7 +2,7 @@
 
 ## Architecture
 
-![Logical diagram of a typical Feast deployment](.gitbook/assets/basic-architecture-diagram.svg)
+![Logical diagram of a typical Feast deployment](.gitbook/assets/basic-architecture-diagram%20%282%29.svg)
 
 The core components of a Feast deployment are
 
@@ -106,13 +106,13 @@ Feast supports the following types for feature values
 * DOUBLE
 * FLOAT
 * BOOL
-* BYTES_LIST
-* STRING_LIST
-* INT32_LIST
-* INT64_LIST
-* DOUBLE_LIST
-* FLOAT_LIST
-* BOOL_LIST
+* BYTES\_LIST
+* STRING\_LIST
+* INT32\_LIST
+* INT64\_LIST
+* DOUBLE\_LIST
+* FLOAT\_LIST
+* BOOL\_LIST
 
 ## Glossary
 
diff --git a/docs/contributing.md b/docs/contributing.md
index 38caffd654b..ef91a7586d2 100644
--- a/docs/contributing.md
+++ b/docs/contributing.md
@@ -1,78 +1,285 @@
 # Contributing
 
-## Getting Started
+## 1. Contribution process
+
+We use [RFCs](https://en.wikipedia.org/wiki/Request_for_Comments) and [GitHub issues](https://github.com/gojek/feast/issues) to communicate development ideas. The simplest way to contribute to Feast is to leave comments in our [RFCs](https://drive.google.com/drive/u/0/folders/1Lj1nIeRB868oZvKTPLYqAvKQ4O0BksjY) in the [Feast Google Drive](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) or our GitHub issues.
+
+Please communicate your ideas through a GitHub issue or through our Slack Channel before starting development.
+
+Please [submit a PR ](https://github.com/gojek/feast/pulls)to the master branch of the Feast repository once you are ready to submit your contribution. Code submission to Feast \(including submission from project maintainers\) require review and approval from maintainers or code owners.
+
+PRs that are submitted by the general public need to be identified as `ok-to-test`. Once enabled, [Prow](https://github.com/kubernetes/test-infra/tree/master/prow) will run a range of tests to verify the submission, after which community members will help to review the pull request.
+
+{% hint style="success" %}
+Please sign the [Google CLA](https://cla.developers.google.com/) in order to have your code merged into the Feast repository.
+{% endhint %}
+
+## 2. Development guide
+
+### 2.1 Overview
 
 The following guide will help you quickly run Feast in your local machine.
 
 The main components of Feast are:
 
-* **Feast Core** handles FeatureSpec registration, starts and monitors Ingestion 
+* **Feast Core:** Handles feature registration, starts and manages ingestion jobs and ensures that Feast internal metadata is consistent.
+* **Feast Ingestion Jobs:** Subscribes to streams of FeatureRows and writes these as feature
 
-  jobs and ensures that Feast internal metadata is consistent.
+  values to registered databases \(online, historical\) that can be read by Feast Serving.
 
-* **Feast Ingestion** subscribes to streams of FeatureRow and writes the feature
+* **Feast Serving:** Service that handles requests for features values, either online or batch.
 
-  values to registered Stores. 
+### 2.**2 Requirements**
 
-* **Feast Serving** handles requests for features values retrieval from the end users.
+#### 2.**2.1 Development environment**
 
-**Pre-requisites**
+The following software is required for Feast development
 
-* Java SDK version 8
+* Java SE Development Kit 11
 * Python version 3.6 \(or above\) and pip
-* Access to Postgres database \(version 11 and above\)
-* Access to [Redis](https://redis.io/topics/quickstart) instance \(tested on version 5.x\)
-* Access to [Kafka](https://kafka.apache.org/) brokers \(tested on version 2.x\)
-* [Maven ](https://maven.apache.org/install.html) version 3.6.x
-* [grpc\_cli](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md) is useful for debugging and quick testing
-* An overview of Feast specifications and protos
-
-> **Assumptions:**
->
-> 1. Postgres is running in "localhost:5432" and has a database called "postgres" which 
->
->    can be accessed with credentials user "postgres" and password "password". 
->
->    To use different database name and credentials, please update 
->
->    "$FEAST\_HOME/core/src/main/resources/application.yml" 
->
->    or set these environment variables: DB\_HOST, DB\_USERNAME, DB\_PASSWORD.
->
-> 2. Redis is running locally and accessible from "localhost:6379"
-> 3. Feast has admin access to BigQuery.
+* [Maven ](https://maven.apache.org/install.html)version 3.6.x
+
+Additionally, [grpc\_cli](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md) is useful for debugging and quick testing of gRPC endpoints.
+
+#### 2.**2.2 Services**
+
+The following components/services are required to develop Feast:
+
+* **Feast Core:** Requires PostgreSQL \(version 11 and above\) to store state, and requires a Kafka \(tested on version 2.x\) setup to allow for ingestion of FeatureRows.
+* **Feast Serving:** Requires Redis \(tested on version 5.x\).
+
+These services should be running before starting development. The following snippet will start the services using Docker.
+
+```bash
+# Start Postgres
+docker run --name postgres --rm -it -d --net host -e POSTGRES_DB=postgres -e POSTGRES_USER=postgres \
+-e POSTGRES_PASSWORD=password postgres:12-alpine
+
+# Start Redis
+docker run --name redis --rm -it --net host -d redis:5-alpine
+
+# Start Zookeeper (needed by Kafka)
+docker run --rm \
+  --net=host \
+  --name=zookeeper \
+  --env=ZOOKEEPER_CLIENT_PORT=2181 \
+  --detach confluentinc/cp-zookeeper:5.2.1
+
+# Start Kafka
+docker run --rm \
+  --net=host \
+  --name=kafka \
+  --env=KAFKA_ZOOKEEPER_CONNECT=localhost:2181 \
+  --env=KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \
+  --env=KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
+  --detach confluentinc/cp-kafka:5.2.1
+```
+
+### 2.3 Testing and development
+
+#### 2.3.1 Running unit tests
 
 ```text
-# $FEAST_HOME will refer to be the root directory of this Feast Git repository
+$ mvn test
+```
 
-git clone https://github.com/gojek/feast
-cd feast
+#### 2.3.2 Running integration tests
+
+_Note: integration suite isn't yet separated from unit._
+
+```text
+$ mvn verify
 ```
 
-#### Starting Feast Core
+#### 2.3.3 Running components locally
+
+The `core` and `serving` modules are Spring Boot applications. These may be run as usual for [the Spring Boot Maven plugin](https://docs.spring.io/spring-boot/docs/current/maven-plugin/index.html):
 
 ```text
-# Please check the default configuration for Feast Core in 
+<<<<<<< HEAD
+$ mvn --projects core spring-boot:run
+
+# Or for short:
+$ mvn -pl core spring-boot:run
+```
+
+Note that you should execute `mvn` from the Feast repository root directory, as there are intermodule dependencies that Maven will not resolve if you `cd` to subdirectories to run.
+
+#### 2.3.4 Running from IntelliJ
+
+Compiling and running tests in IntelliJ should work as usual.
+
+Running the Spring Boot apps may work out of the box in IDEA Ultimate, which has built-in support for Spring Boot projects, but the Community Edition needs a bit of help:
+
+The Spring Boot Maven plugin automatically puts dependencies with `provided` scope on the runtime classpath when using `spring-boot:run`, such as its embedded Tomcat server. The "Play" buttons in the gutter or right-click menu of a `main()` method [do not do this](https://stackoverflow.com/questions/30237768/run-spring-boots-main-using-ide).
+
+A solution to this is:
+
+1. Open `View > Tool Windows > Maven`
+2. Drill down to e.g. `Feast Core > Plugins > spring-boot:run`, right-click and `Create 'feast-core [spring-boot'…`
+3. In the dialog that pops up, check the `Resolve Workspace artifacts` box
+4. Click `OK`. You should now be able to select this run configuration for the Play button in the main toolbar, keyboard shortcuts, etc.
+
+### 2.**4** Validating your setup
+
+The following section is a quick walk-through to test whether your local Feast deployment is functional for development purposes.
+
+**2.4.1 Assumptions**
+
+* PostgreSQL is running in `localhost:5432` and has a database called `postgres` which
+
+  can be accessed with credentials user `postgres` and password `password`. Different database configurations can be supplied here \(`/core/src/main/resources/application.yml`\)
+
+* Redis is running locally and accessible from `localhost:6379`
+* \(optional\) The local environment has been authentication with Google Cloud Platform and has full access to BigQuery. This is only necessary for BigQuery testing/development.
+
+#### 2.4.2 Clone Feast
+
+```bash
+git clone https://github.com/gojek/feast.git && cd feast && \
+export FEAST_HOME_DIR=$(pwd)
+```
+
+#### 2.4.3 Starting Feast Core
+
+To run Feast Core locally using Maven:
+
+```bash
+# Feast Core can be configured from the following .yml file
+# $FEAST_HOME_DIR/core/src/main/resources/application.yml
+=======
+# Please check the default configuration for Feast Core in
 # "$FEAST_HOME/core/src/main/resources/application.yml" and update it accordingly.
-# 
+#
 # Start Feast Core GRPC server on localhost:6565
+>>>>>>> 98198d8... Add Cassandra Store (#360)
 mvn --projects core spring-boot:run
+```
+
+Test whether Feast Core is running
+
+```text
+grpc_cli call localhost:6565 ListStores ''
+```
+
+The output should list **no** stores since no Feast Serving has registered its stores to Feast Core:
+
+```text
+connecting to localhost:6565
+
+Rpc succeeded with OK status
+```
+
+#### 2.4.4 Starting Feast Serving
+
+Feast Serving is configured through the `$FEAST_HOME_DIR/serving/src/main/resources/application.yml`. Each Serving deployment must be configured with a store. The default store is Redis \(used for online serving\).
+
+The configuration for this default store is located in a separate `.yml` file. The default location is `$FEAST_HOME_DIR/serving/sample_redis_config.yml`:
+
+```text
+name: serving
+type: REDIS
+redis_config:
+  host: localhost
+  port: 6379
+subscriptions:
+  - name: "*"
+    project: "*"
+    version: "*"
+```
+
+Once Feast Serving is started, it will register its store with Feast Core \(by name\) and start to subscribe to a feature sets based on its subscription.
+
+Start Feast Serving GRPC server on localhost:6566 with store name `serving`
+
+```text
+mvn --projects serving spring-boot:run
+```
 
+Test connectivity to Feast Serving
+
+```text
+grpc_cli call localhost:6566 GetFeastServingInfo ''
+```
+
+<<<<<<< HEAD
+```text
+connecting to localhost:6566
+version: "0.4.2-SNAPSHOT"
+type: FEAST_SERVING_TYPE_ONLINE
+
+Rpc succeeded with OK status
+```
+=======
 # If Feast Core starts successfully, verify the correct Stores are registered
 # correctly, for example by using grpc_cli.
-grpc_cli call localhost:6565 GetStores ''
+grpc_cli call localhost:6565 ListStores ''
 
+<<<<<<< HEAD:docs/contributing.md
 # Should return something similar to the following.
 # Note that you should change BigQuery projectId and datasetId accordingly
 # in "$FEAST_HOME/core/src/main/resources/application.yml"
+>>>>>>> 98198d8... Add Cassandra Store (#360)
+
+Test Feast Core to see whether it is aware of the Feast Serving deployment
 
+```text
+grpc_cli call localhost:6565 ListStores ''
+```
+
+```text
+connecting to localhost:6565
 store {
-  name: "SERVING"
+  name: "serving"
   type: REDIS
   subscriptions {
+    name: "*"
+    version: "*"
+    project: "*"
+  }
+  redis_config {
+    host: "localhost"
+    port: 6379
+  }
+}
+
+Rpc succeeded with OK status
+```
+
+In order to use BigQuery as a historical store, it is necessary to start Feast Serving with a different store type.
+
+Copy `$FEAST_HOME_DIR/serving/sample_redis_config.yml`  to the following location `$FEAST_HOME_DIR/serving/my_bigquery_config.yml` and update the configuration as below:
+
+```text
+name: bigquery
+type: BIGQUERY
+bigquery_config:
+  project_id: YOUR_GCP_PROJECT_ID
+  dataset_id: YOUR_GCP_DATASET
+subscriptions:
+  - name: "*"
+    version: "*"
     project: "*"
+```
+
+Then inside `serving/src/main/resources/application.yml` modify the following key `feast.store.config-path`  to point to the new store configuration.
+
+After making these changes, restart Feast Serving:
+
+```text
+mvn --projects serving spring-boot:run
+```
+
+You should see two stores registered:
+
+```text
+store {
+  name: "serving"
+  type: REDIS
+  subscriptions {
     name: "*"
     version: "*"
+    project: "*"
   }
   redis_config {
     host: "localhost"
@@ -80,85 +287,186 @@ store {
   }
 }
 store {
-  name: "WAREHOUSE"
+  name: "bigquery"
   type: BIGQUERY
   subscriptions {
-    project: "*"
     name: "*"
     version: "*"
+    project: "*"
   }
   bigquery_config {
-    project_id: "my-google-project-id"
-    dataset_id: "my-bigquery-dataset-id"
+    project_id: "my_project"
+    dataset_id: "my_bq_dataset"
   }
+# Should return something similar to the following if you have not updated any stores
+{
+  "store": []
 }
 ```
 
+<<<<<<< HEAD
+#### 2.4.5 Registering a FeatureSet
+=======
 #### Starting Feast Serving
 
-Feast Serving requires administrators to provide an **existing** store name in Feast. An instance of Feast Serving can only retrieve features from a **single** store.
+Feast Serving requires administrators to provide an **existing** store name in Feast.
+An instance of Feast Serving can only retrieve features from a **single** store.
+> In order to retrieve features from multiple stores you must start **multiple**
+instances of Feast serving. If you start multiple Feast serving on a single host,
+make sure that they are listening on different ports.
+>>>>>>> 98198d8... Add Cassandra Store (#360)
 
-> In order to retrieve features from multiple stores you must start **multiple** instances of Feast serving. If you start multiple Feast serving on a single host, make sure that they are listening on different ports.
+Before registering a new FeatureSet, a project is required.
 
 ```text
-# Start Feast Serving GRPC server on localhost:6566 with store name "SERVING"
-mvn --projects serving spring-boot:run -Dspring-boot.run.arguments='--feast.store-name=SERVING'
+grpc_cli call localhost:6565 CreateProject '
+  name: "your_project_name"
+'
+```
 
-# To verify Feast Serving starts successfully
-grpc_cli call localhost:6566 GetFeastServingType ''
+<<<<<<< HEAD
+When a feature set is successfully registered, Feast Core will start an **ingestion** job that listens for new features in the feature set.
+=======
+#### Updating a store
+
+Create a new Store by sending a request to Feast Core.
 
-# Should return something similar to the following.
-type: FEAST_SERVING_TYPE_ONLINE
+```
+# Example of updating a redis store
+
+grpc_cli call localhost:6565 UpdateStore '
+store {
+  name: "SERVING"
+  type: REDIS
+  subscriptions {
+    name: "*"
+    version: ">0"
+  }
+  redis_config {
+    host: "localhost"
+    port: 6379
+  }
+}
+'
+
+# Other supported stores examples (replacing redis_config):
+# BigQuery
+bigquery_config {
+  project_id: "my-google-project-id"
+  dataset_id: "my-bigquery-dataset-id"
+}
+
+# Cassandra: two options in cassandra depending on replication strategy
+# See details: https://docs.datastax.com/en/cassandra/3.0/cassandra/architecture/archDataDistributeReplication.html
+#
+# Please note that table name must be "feature_store" as is specified in the @Table annotation of the
+# datastax object mapper
+
+# SimpleStrategy
+cassandra_config {
+  bootstrap_hosts: "localhost"
+  port: 9042
+  keyspace: "feast"
+  table_name: "feature_store"
+  replication_options {
+    class: "SimpleStrategy"
+    replication_factor: 1
+  }
+}
+
+# NetworkTopologyStrategy
+cassandra_config {
+  bootstrap_hosts: "localhost"
+  port: 9042
+  keyspace: "feast"
+  table_name: "feature_store"
+  replication_options {
+    class: "NetworkTopologyStrategy"
+    east: 2
+    west: 2
+  }
+}
+
+# To check that the Stores has been updated correctly.
+grpc_cli call localhost:6565 ListStores ''
 ```
 
 #### Registering a FeatureSet
+>>>>>>> 98198d8... Add Cassandra Store (#360)
+
+{% hint style="info" %}
+Note that Feast currently only supports source of type `KAFKA`, so you must have access to a running Kafka broker to register a FeatureSet successfully. It is possible to omit the `source` from a Feature Set, but Feast Core will still use Kafka behind the scenes, it is simply abstracted away from the user.
+{% endhint %}
 
-Create a new FeatureSet on Feast by sending a request to Feast Core. When a feature set is successfully registered, Feast Core will start an **ingestion** job that listens for new features in the FeatureSet. Note that Feast currently only supports source of type "KAFKA", so you must have access to a running Kafka broker to register a FeatureSet successfully.
+Create a new FeatureSet in Feast by sending a request to Feast Core:
 
 ```text
-# Example of registering a new driver feature set 
+# Example of registering a new driver feature set
 # Note the source value, it assumes that you have access to a Kafka broker
 # running on localhost:9092
 
 grpc_cli call localhost:6565 ApplyFeatureSet '
 feature_set {
-  name: "driver"
-  version: 1
-
-  entities {
-    name: "driver_id"
-    value_type: INT64
-  }
+  spec {
+    project: "your_project_name"
+    name: "driver"
+    version: 1
+
+    entities {
+      name: "driver_id"
+      value_type: INT64
+    }
 
-  features {
-    name: "city"
-    value_type: STRING
-  }
+    features {
+      name: "city"
+      value_type: STRING
+    }
 
-  source {
-    type: KAFKA
-    kafka_source_config {
-      bootstrap_servers: "localhost:9092"
+    source {
+      type: KAFKA
+      kafka_source_config {
+        bootstrap_servers: "localhost:9092"
+        topic: "your-kafka-topic"
+      }
     }
   }
 }
 '
+```
 
+Verify that the FeatureSet has been registered correctly.
+
+```text
 # To check that the FeatureSet has been registered correctly.
 # You should also see logs from Feast Core of the ingestion job being started
-grpc_cli call localhost:6565 GetFeatureSets ''
+grpc_cli call localhost:6565 GetFeatureSet '
+  project: "your_project_name"
+  name: "driver"
+'
 ```
 
-#### Ingestion and Population of Feature Values
+Or alternatively, list all feature sets
+
+```text
+grpc_cli call localhost:6565 ListFeatureSets '
+  filter {
+    project: "your_project_name"
+    feature_set_name: "driver"
+    feature_set_version: "1"
+  }
+'
+```
+
+#### 2.4.6 Ingestion and Population of Feature Values
 
 ```text
 # Produce FeatureRow messages to Kafka so it will be ingested by Feast
 # and written to the registered stores.
 # Make sure the value here is the topic assigned to the feature set
 # ... producer.send("feast-driver-features" ...)
-# 
+#
 # Install Python SDK to help writing FeatureRow messages to Kafka
-cd $FEAST_HOME/sdk/python
+cd $FEAST_HOMEDIR/sdk/python
 pip3 install -e .
 pip3 install pendulum
 
@@ -194,24 +502,33 @@ timestamp.FromJsonString(
 )
 row.event_timestamp.CopyFrom(timestamp)
 
-# The format is [FEATURE_NAME]:[VERSION]
-row.feature_set = "driver:1"
+# The format is [PROJECT_NAME]/[FEATURE_NAME]:[VERSION]
+row.feature_set = "your_project_name/driver:1"
 
-producer.send("feast-driver-features", row.SerializeToString())
+producer.send("your-kafka-topic", row.SerializeToString())
 producer.flush()
 logger.info(row)
 EOF
+```
+
+#### 2.4.7 Retrieval from Feast Serving
+
+Ensure that Feast Serving returns results for the feature value for the specific driver
 
-# Check that the ingested feature rows can be retrieved from Feast serving
+```text
 grpc_cli call localhost:6566 GetOnlineFeatures '
-feature_sets {
-  name: "driver"
+features {
+  project: "your_project_name"
+  name: "city"
   version: 1
+  max_age {
+    seconds: 3600
+  }
 }
-entity_dataset {
-  entity_names: "driver_id"
-  entity_dataset_rows {
-    entity_ids {
+entity_rows {
+  fields {
+    key: "driver_id"
+    value {
       int64_val: 1234
     }
   }
@@ -219,92 +536,32 @@ entity_dataset {
 '
 ```
 
-## Development
-
-Notes:
-
-* Use of Lombok is being phased out, prefer to use [Google Auto](https://github.com/google/auto) in new code.
-
-### Running Unit Tests
-
 ```text
-$ mvn test
-```
-
-### Running Integration Tests
-
-_Note: integration suite isn't yet separated from unit._
-
-```text
-$ mvn verify
-```
-
-### Running Components Locally
-
-The `core` and `serving` modules are Spring Boot applications. These may be run as usual for [the Spring Boot Maven plugin](https://docs.spring.io/spring-boot/docs/current/maven-plugin/index.html):
-
-```text
-$ mvn --projects core spring-boot:run
-
-# Or for short:
-$ mvn -pl core spring-boot:run
+field_values {
+  fields {
+    key: "driver_id"
+    value {
+      int64_val: 1234
+    }
+  }
+  fields {
+    key: "your_project_name/city:1"
+    value {
+      string_val: "JAKARTA"
+    }
+  }
+}
 ```
 
-Note that you should execute `mvn` from the Feast repository root directory, as there are intermodule dependencies that Maven will not resolve if you `cd` to subdirectories to run.
-
-#### Running From IntelliJ
-
-Compiling and running tests in IntelliJ should work as usual.
-
-Running the Spring Boot apps may work out of the box in IDEA Ultimate, which has built-in support for Spring Boot projects, but the Community Edition needs a bit of help:
-
-The Spring Boot Maven plugin automatically puts dependencies with `provided` scope on the runtime classpath when using `spring-boot:run`, such as its embedded Tomcat server. The "Play" buttons in the gutter or right-click menu of a `main()` method [do not do this](https://stackoverflow.com/questions/30237768/run-spring-boots-main-using-ide).
-
-A solution to this is:
-
-1. Open `View > Tool Windows > Maven`
-2. Drill down to e.g. `Feast Core > Plugins > spring-boot:run`, right-click and `Create 'feast-core [spring-boot'…`
-3. In the dialog that pops up, check the `Resolve Workspace artifacts` box
-4. Click `OK`. You should now be able to select this run configuration for the Play button in the main toolbar, keyboard shortcuts, etc.
-
-#### Tips for Running Postgres, Redis and Kafka with Docker
-
-This guide assumes you are running Docker service on a bridge network \(which is usually the case if you're running Linux\). Otherwise, you may need to use different network options than shown below.
-
-> `--net host` usually only works as expected when you're running Docker service in bridge networking mode.
-
-```text
-# Start Postgres
-docker run --name postgres --rm -it -d --net host -e POSTGRES_DB=postgres -e POSTGRES_USER=postgres \
--e POSTGRES_PASSWORD=password postgres:12-alpine
-
-# Start Redis
-docker run --name redis --rm -it --net host -d redis:5-alpine
-
-# Start Zookeeper (needed by Kafka)
-docker run --rm \
-  --net=host \
-  --name=zookeeper \
-  --env=ZOOKEEPER_CLIENT_PORT=2181 \
-  --detach confluentinc/cp-zookeeper:5.2.1
+#### 2.4.8 Summary
 
-# Start Kafka
-docker run --rm \
-  --net=host \
-  --name=kafka \
-  --env=KAFKA_ZOOKEEPER_CONNECT=localhost:2181 \
-  --env=KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \
-  --env=KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
-  --detach confluentinc/cp-kafka:5.2.1
-```
-
-## Code reviews
+If you have made it to this point successfully you should have a functioning Feast deployment, at the very least using the Apache Beam DirectRunner for ingestion jobs and Redis for online serving.
 
-Code submission to Feast \(including submission from project maintainers\) requires review and approval. Please submit a **pull request** to initiate the code review process. We use [prow](https://github.com/kubernetes/test-infra/tree/master/prow) to manage the testing and reviewing of pull requests. Please refer to [config.yaml](../.prow/config.yaml) for details on the test jobs.
+It is important to note that most of the functionality demonstrated above is already available in a more abstracted form in the Python SDK \(Feast management, data ingestion, feature retrieval\) and the Java/Go SDKs \(feature retrieval\). However, it is useful to understand these internals from a development standpoint.
 
-## Code conventions
+## 3. Style guide
 
-### Java
+### 3.1 Java
 
 We conform to the [Google Java Style Guide](https://google.github.io/styleguide/javaguide.html). Maven can helpfully take care of that for you before you commit:
 
@@ -319,13 +576,17 @@ $ mvn spotless:check  # Check is automatic upon `mvn verify`
 $ mvn verify -Dspotless.check.skip
 ```
 
-If you're using IntelliJ, you can import [these code style settings](https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml) if you'd like to use the IDE's reformat function as you work.
+If you're using IntelliJ, you can import [these code style settings](https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml) if you'd like to use the IDE's reformat function as you develop.
 
-### Go
+### 3.2 Go
 
 Make sure you apply `go fmt`.
 
-## Release process
+### 3.3 Python
+
+We use [Python Black](https://github.com/psf/black) to format our Python code prior to submission.
+
+## 4. Release process
 
 Feast uses [semantic versioning](https://semver.org/).
 
diff --git a/docs/getting-help.md b/docs/getting-help.md
index d8180ab7842..597a782d606 100644
--- a/docs/getting-help.md
+++ b/docs/getting-help.md
@@ -1,23 +1,36 @@
 # Getting Help
 
-## Chat
+### Chat
 
-* Come and chat with us in the [\#Feast Slack channel in the Kubeflow workspace](https://join.slack.com/t/kubeflow/shared_invite/enQtNDg5MTM4NTQyNjczLTdkNTVhMjg1ZTExOWI0N2QyYTQ2MTIzNTJjMWRiOTFjOGRlZWEzODc1NzMwNTMwM2EzNjY1MTFhODczNjk4MTk) and catch up on all things Feast!
+* Come and say hello in [\#Feast](https://join.slack.com/t/kubeflow/shared_invite/enQtNDg5MTM4NTQyNjczLTdkNTVhMjg1ZTExOWI0N2QyYTQ2MTIzNTJjMWRiOTFjOGRlZWEzODc1NzMwNTMwM2EzNjY1MTFhODczNjk4MTk) over in the Kubeflow Slack.
 
-## GitHub
+### GitHub
 
 * Feast's GitHub repo can be [found here](https://github.com/gojek/feast/).
 * Found a bug or need a feature? [Create an issue on GitHub](https://github.com/gojek/feast/issues/new)
 
-## Mailing list
+### Community Call
 
-### Feast discussion
+We have a community call every 2 weeks. Alternating between two times.
+
+* 11 am \(UTC + 8\)
+* 5 pm \(UTC + 8\)
+
+Please join the [**feast-dev**](getting-help.md#feast-development) mailing list to receive the the calendar invitation.
+
+### Mailing list
+
+#### Feast discussion
 
 * Google Group: [https://groups.google.com/d/forum/feast-discuss](https://groups.google.com/d/forum/feast-discuss)
 * Mailing List: [feast-discuss@googlegroups.com](mailto:feast-discuss@googlegroups.com)
 
-### Feast development
+#### Feast development
 
 * Google Group: [https://groups.google.com/d/forum/feast-dev](https://groups.google.com/d/forum/feast-dev)
 * Mailing List: [feast-dev@googlegroups.com](mailto:feast-dev@googlegroups.com)
 
+### Google Drive
+
+The Feast community also maintains a [Google Drive](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) with documents like RFCs, meeting notes, or roadmaps. Please join one of the above mailing lists \(feast-dev or feast-discuss\) to gain access to the drive.
+
diff --git a/docs/getting-started/installing-feast.md b/docs/getting-started/installing-feast.md
deleted file mode 100644
index 0b8b42f26f6..00000000000
--- a/docs/getting-started/installing-feast.md
+++ /dev/null
@@ -1,424 +0,0 @@
-# Installing Feast
-
-## Overview
-
-This installation guide will demonstrate three ways of installing Feast:
-
-* \*\*\*\*[**Docker Compose \(Quickstart\):**](installing-feast.md#docker-compose) Fastest way to get Feast up and running. Provides a pre-installed Jupyter Notebook with the Feast Python SDK and sample code.
-* [**Minikube**](installing-feast.md#minikube)**:** This installation has no external dependencies, but does not have a historical feature store installed. It allows users to quickly get a feel for Feast.
-* [**Google Kubernetes Engine:**](installing-feast.md#google-kubernetes-engine) This guide installs a single cluster Feast installation on Google's GKE. It has Google Cloud specific dependencies like BigQuery, Dataflow, and Google Cloud Storage.
-
-## Docker Compose \(Quickstart\)
-
-### Overview
-
-A docker compose file is provided to quickly test Feast with the official docker images. There is no hard dependency on GCP, unless batch serving is required. Once you have set up Feast using Docker Compose, you will be able to:
-
-* Create, register, and manage feature sets
-* Ingest feature data into Feast
-* Retrieve features for online serving
-
-{% hint style="info" %}
-The docker compose setup uses Direct Runner for the Apache Beam jobs. Running Beam with the Direct Runner means it does not need a dedicated runner like Flink or Dataflow, but this comes at the cost of performance. We recommend the use of a full runner when running Feast with very large workloads.
-{% endhint %}
-
-### 0. Requirements
-
-* [Docker compose](https://docs.docker.com/compose/install/) should be installed.
-* TCP ports 6565, 6566, 8888, and 9094 should not be in use. Otherwise, modify the port mappings in  `infra/docker-compose/docker-compose.yml` to use unoccupied ports.
-* \(for batch serving only\) For batch serving you will also need a [GCP service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) that has access to GCS and BigQuery. Port 6567 will be used for the batch serving endpoint.
-* \(for batch serving only\) [Google Cloud SDK ](https://cloud.google.com/sdk/install)installed, authenticated, and configured to the project you will use.
-
-### 1. Step-by-step guide \(Online serving only\)
-
-Clone the [Feast repository](https://github.com/gojek/feast/) and navigate to the `docker-compose` sub-directory:
-
-```bash
-git clone https://github.com/gojek/feast.git && \
-cd feast && export FEAST_HOME_DIR=$(pwd) && \
-cd infra/docker-compose
-```
-
-Make a copy of the `.env.sample` file:
-
-```bash
-cp .env.sample .env
-```
-
-Start Feast:
-
-```javascript
-docker-compose up -d
-```
-
-A Jupyter notebook is now available to use Feast:
-
-[http://localhost:8888/notebooks/feast-notebooks/feast-quickstart.ipynb](http://localhost:8888/notebooks/feast-notebooks/feast-quickstart.ipynb)
-
-### 2. Step-by-step guide \(Batch and online serving\)
-
-Clone the [Feast repository](https://github.com/gojek/feast/) and navigate to the `docker-compose` sub-directory:
-
-```bash
-git clone https://github.com/gojek/feast.git && \
-cd feast && export FEAST_HOME_DIR=$(pwd) && \
-cd infra/docker-compose
-```
-
-Create a [service account ](https://cloud.google.com/iam/docs/creating-managing-service-accounts)from the GCP console and copy it to the `gcp-service-accounts` folder:
-
-```javascript
-cp my-service-account.json ${FEAST_HOME_DIR}/infra/docker-compose/gcp-service-accounts
-```
-
-Create a Google Cloud Storage bucket. Make sure that your service account above has read/write permissions to this bucket:
-
-```bash
-gsutil mb gs://my-feast-staging-bucket
-```
-
-Make a copy of the `.env.sample` file:
-
-```bash
-cp .env.sample .env
-```
-
-Customize the `.env` file based on your environment. At the very least you have to modify:
-
-* **FEAST\_CORE\_GCP\_SERVICE\_ACCOUNT\_KEY:** This should be your service account file name without the .json extension.
-* **FEAST\_BATCH\_SERVING\_GCP\_SERVICE\_ACCOUNT\_KEY:** This should be your service account file name without the .json extension.
-* **FEAST\_JUPYTER\_GCP\_SERVICE\_ACCOUNT\_KEY:** This should be your service account file name without the .json extension.
-* **FEAST\_JOB\_STAGING\_LOCATION:** Google Cloud Storage bucket that Feast will use to stage data exports and batch retrieval requests.
-
-We will also need to customize the `bq-store.yml` file inside `infra/docker-compose/serving/` to configure the BigQuery storage configuration as well as the feature sets that the store subscribes to. At a minimum you will need to set:
-
-* **project\_id:** This is you GCP project id.
-* **dataset\_id:** This is the name of the BigQuery dataset that tables will be created in. Each feature set will have one table in BigQuery.
-
-Start Feast:
-
-```javascript
-docker-compose -f docker-compose.yml -f docker-compose.batch.yml up -d
-```
-
-A Jupyter notebook is now available to use Feast:
-
-[http://localhost:8888/notebooks/feast-notebooks](http://localhost:8888/tree/feast-notebooks)
-
-## Minikube
-
-### Overview
-
-This guide will install Feast into [Minikube](https://github.com/kubernetes/minikube). Once Feast is installed you will be able to:
-
-* Define and register features.
-* Load feature data from both batch and streaming sources.
-* Retrieve features for online serving.
-
-{% hint style="warning" %}
-This Minikube installation guide is for demonstration purposes only. It is not meant for production use, and does not install a historical feature store.
-{% endhint %}
-
-### 0. Requirements
-
-The following software should be installed prior to starting:
-
-1. [Minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/) should be installed.
-2. [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) installed and configured to work with Minikube.
-3. [Helm](https://helm.sh/3) \(2.16.0 or greater\). Helm 3 has not been tested yet.
-
-### 1. Set up Minikube
-
-Start Minikube. Note the minimum cpu and memory below:
-
-```bash
-minikube start --cpus=3 --memory=4096 --kubernetes-version='v1.15.5'
-```
-
-Set up your Feast environmental variables
-
-```bash
-export FEAST_IP=$(minikube ip)
-export FEAST_CORE_URL=${FEAST_IP}:32090
-export FEAST_SERVING_URL=${FEAST_IP}:32091
-```
-
-### 2. Install Feast with Helm
-
-Clone the [Feast repository](https://github.com/gojek/feast/) and navigate to the `charts` sub-directory:
-
-```bash
-git clone https://github.com/gojek/feast.git && \
-cd feast && export FEAST_HOME_DIR=$(pwd) && \
-cd infra/charts/feast
-```
-
-Copy the `values-demo.yaml` file for your installation:
-
-```bash
-cp values-demo.yaml my-feast-values.yaml
-```
-
-Update all occurrences of the domain `feast.example.com` inside of `my-feast-values.yaml` with your Minikube IP. This is to allow external access to the services in the cluster. You can find your Minikube IP by running the following command `minikube ip`, or simply replace the text from the command line:
-
-```bash
-sed -i "s/feast.example.com/${FEAST_IP}/g" my-feast-values.yaml
-```
-
-Install Tiller:
-
-```bash
-helm init
-```
-
-Install the Feast Helm chart:
-
-```bash
-helm install --name feast -f my-feast-values.yaml .
-```
-
-Ensure that the system comes online. This will take a few minutes
-
-```bash
-watch kubectl get pods
-```
-
-```bash
-NAME                                           READY   STATUS      RESTARTS   AGE
-pod/feast-feast-core-666fd46db4-l58l6          1/1     Running     0          5m
-pod/feast-feast-serving-online-84d99ddcbd      1/1     Running     0          6m
-pod/feast-kafka-0                              1/1     Running     0          3m
-pod/feast-kafka-1                              1/1     Running     0          4m
-pod/feast-kafka-2                              1/1     Running     0          4m
-pod/feast-postgresql-0                         1/1     Running     0          5m
-pod/feast-redis-master-0                       1/1     Running     0          5m
-pod/feast-zookeeper-0                          1/1     Running     0          5m
-pod/feast-zookeeper-1                          1/1     Running     0          5m
-pod/feast-zookeeper-2                          1/1     Running     0          5m
-```
-
-### 3. Connect to Feast with the Python SDK
-
-Install the Python SDK using pip:
-
-```bash
-pip install -e ${FEAST_HOME_DIR}/sdk/python
-```
-
-Configure the Feast Python SDK:
-
-```bash
-feast config set core_url ${FEAST_CORE_URL}
-feast config set serving_url ${FEAST_SERVING_URL}
-```
-
-That's it! You can now start to use Feast!
-
-## Google Kubernetes Engine
-
-### Overview 
-
-This guide will install Feast into a Kubernetes cluster on GCP. It assumes that all of your services will run within a single K8s cluster. Once Feast is installed you will be able to:
-
-* Define and register features.
-* Load feature data from both batch and streaming sources.
-* Retrieve features for model training.
-* Retrieve features for online serving.
-
-{% hint style="info" %}
-This guide requires [Google Cloud Platform](https://cloud.google.com/) for installation.
-
-* [BigQuery](https://cloud.google.com/bigquery/) is used for storing historical features.
-* [Cloud Dataflow](https://cloud.google.com/dataflow/) is used for running data ingestion jobs.
-* [Google Cloud Storage](https://cloud.google.com/storage/) is used for intermediate data storage.
-{% endhint %}
-
-### 0. Requirements 
-
-1. [Google Cloud SDK ](https://cloud.google.com/sdk/install)installed, authenticated, and configured to the project you will use.
-2. [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) installed.
-3. [Helm](https://helm.sh/3) \(2.16.0 or greater\) installed on your local machine with Tiller installed in your cluster. Helm 3 has not been tested yet.
-
-### 1. Set up GCP
-
-First define the environmental variables that we will use throughout this installation. Please customize these to reflect your environment.
-
-```bash
-export FEAST_GCP_PROJECT_ID=my-gcp-project
-export FEAST_GCP_REGION=us-central1
-export FEAST_GCP_ZONE=us-central1-a
-export FEAST_BIGQUERY_DATASET_ID=feast
-export FEAST_GCS_BUCKET=${FEAST_GCP_PROJECT_ID}_feast_bucket
-export FEAST_GKE_CLUSTER_NAME=feast
-export FEAST_S_ACCOUNT_NAME=feast-sa
-```
-
-Create a Google Cloud Storage bucket for Feast to stage data during exports:
-
-```bash
-gsutil mb gs://${FEAST_GCS_BUCKET}
-```
-
-Create a BigQuery dataset for storing historical features:
-
-```bash
-bq mk ${FEAST_BIGQUERY_DATASET_ID}
-```
-
-Create the service account that Feast will run as:
-
-```bash
-gcloud iam service-accounts create ${FEAST_SERVICE_ACCOUNT_NAME}
-
-gcloud projects add-iam-policy-binding ${FEAST_GCP_PROJECT_ID} \
-  --member serviceAccount:${FEAST_S_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com \
-  --role roles/editor
-
-gcloud iam service-accounts keys create key.json --iam-account \
-${FEAST_S_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com
-```
-
-Ensure that [Dataflow API is enabled](https://console.cloud.google.com/apis/api/dataflow.googleapis.com/overview):
-
-```bash
-gcloud services enable dataflow.googleapis.com
-```
-
-### 2. Set up a Kubernetes \(GKE\) cluster
-
-{% hint style="warning" %}
-Provisioning a GKE cluster can expose your services publicly. This guide does not cover securing access to the cluster.
-{% endhint %}
-
-Create a GKE cluster:
-
-```bash
-gcloud container clusters create ${FEAST_GKE_CLUSTER_NAME} \
-    --machine-type n1-standard-4
-```
-
-Create a secret in the GKE cluster based on your local key `key.json`:
-
-```bash
-kubectl create secret generic feast-gcp-service-account --from-file=key.json
-```
-
-For this guide we will use `NodePort` for exposing Feast services. In order to do so, we must find an internal IP of at least one GKE node.
-
-```bash
-export FEAST_IP=$(kubectl describe nodes | grep InternalIP | awk '{print $2}' | head -n 1)
-export FEAST_CORE_URL=${FEAST_IP}:32090
-export FEAST_ONLINE_SERVING_URL=${FEAST_IP}:32091
-export FEAST_BATCH_SERVING_URL=${FEAST_IP}:32092
-```
-
-Confirm that you are able to access this node:
-
-```bash
-ping $FEAST_IP
-```
-
-```bash
-PING 10.123.114.11 (10.203.164.22) 56(84) bytes of data.
-64 bytes from 10.123.114.11: icmp_seq=1 ttl=63 time=54.2 ms
-64 bytes from 10.123.114.11: icmp_seq=2 ttl=63 time=51.2 ms
-```
-
-### 3. Set up Helm
-
-Run the following command to provide Tiller with authorization to install Feast:
-
-```bash
-kubectl apply -f - <
+
+This guide will install Feast into a Kubernetes cluster on GCP. It assumes that all of your services will run within a single Kubernetes cluster. Once Feast is installed you will be able to:
+
+* Define and register features.
+* Load feature data from both batch and streaming sources.
+* Retrieve features for model training.
+* Retrieve features for online serving.
+
+{% hint style="info" %}
+This guide requires [Google Cloud Platform](https://cloud.google.com/) for installation.
+
+* [BigQuery](https://cloud.google.com/bigquery/) is used for storing historical features.
+* [Google Cloud Storage](https://cloud.google.com/storage/) is used for intermediate data storage.
+{% endhint %}
+
+## 0. Requirements
+
+1. [Google Cloud SDK ](https://cloud.google.com/sdk/install)installed, authenticated, and configured to the project you will use.
+2. [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) installed.
+3. [Helm](https://helm.sh/3) \(2.16.0 or greater\) installed on your local machine with Tiller installed in your cluster. Helm 3 has not been tested yet.
+
+## 1. Set up GCP
+
+First define the environmental variables that we will use throughout this installation. Please customize these to reflect your environment.
+
+```bash
+export FEAST_GCP_PROJECT_ID=my-gcp-project
+export FEAST_GCP_REGION=us-central1
+export FEAST_GCP_ZONE=us-central1-a
+export FEAST_BIGQUERY_DATASET_ID=feast
+export FEAST_GCS_BUCKET=${FEAST_GCP_PROJECT_ID}_feast_bucket
+export FEAST_GKE_CLUSTER_NAME=feast
+export FEAST_SERVICE_ACCOUNT_NAME=feast-sa
+```
+
+Create a Google Cloud Storage bucket for Feast to stage batch data exports:
+
+```bash
+gsutil mb gs://${FEAST_GCS_BUCKET}
+```
+
+Create the service account that Feast will run as:
+
+```bash
+gcloud iam service-accounts create ${FEAST_SERVICE_ACCOUNT_NAME}
+
+gcloud projects add-iam-policy-binding ${FEAST_GCP_PROJECT_ID} \
+  --member serviceAccount:${FEAST_SERVICE_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com \
+  --role roles/editor
+
+gcloud iam service-accounts keys create key.json --iam-account \
+${FEAST_SERVICE_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com
+```
+
+## 2. Set up a Kubernetes \(GKE\) cluster
+
+{% hint style="warning" %}
+Provisioning a GKE cluster can expose your services publicly. This guide does not cover securing access to the cluster.
+{% endhint %}
+
+Create a GKE cluster:
+
+```bash
+gcloud container clusters create ${FEAST_GKE_CLUSTER_NAME} \
+    --machine-type n1-standard-4
+```
+
+Create a secret in the GKE cluster based on your local key `key.json`:
+
+```bash
+kubectl create secret generic feast-gcp-service-account --from-file=key.json
+```
+
+For this guide we will use `NodePort` for exposing Feast services. In order to do so, we must find an External IP of at least one GKE node. This should be a public IP.
+
+```bash
+export FEAST_IP=$(kubectl describe nodes | grep ExternalIP | awk '{print $2}' | head -n 1)
+export FEAST_CORE_URL=${FEAST_IP}:32090
+export FEAST_ONLINE_SERVING_URL=${FEAST_IP}:32091
+export FEAST_BATCH_SERVING_URL=${FEAST_IP}:32092
+```
+
+Add firewall rules to open up ports on your Google Cloud Platform project:
+
+```bash
+gcloud compute firewall-rules create feast-core-port --allow tcp:32090
+gcloud compute firewall-rules create feast-online-port --allow tcp:32091
+gcloud compute firewall-rules create feast-batch-port --allow tcp:32092
+gcloud compute firewall-rules create feast-redis-port --allow tcp:32101
+gcloud compute firewall-rules create feast-kafka-ports --allow tcp:31090-31095
+```
+
+## 3. Set up Helm
+
+Run the following command to provide Tiller with authorization to install Feast:
+
+```bash
+kubectl apply -f - <8888/tcp                           feast_jupyter_1
+8e49dbe81b92        gcr.io/kf-feast/feast-serving:latest   "java -Xms1024m -Xmx…"   2 minutes ago       Up 5 seconds        0.0.0.0:6567->6567/tcp                           feast_batch-serving_1
+b859494bd33a        gcr.io/kf-feast/feast-serving:latest   "java -jar /opt/feas…"   2 minutes ago       Up About a minute   0.0.0.0:6566->6566/tcp                           feast_online-serving_1
+5c4962811767        gcr.io/kf-feast/feast-core:latest      "java -jar /opt/feas…"   2 minutes ago       Up 2 minutes        0.0.0.0:6565->6565/tcp                           feast_core_1
+1ba7239e0ae0        confluentinc/cp-kafka:5.2.1            "/etc/confluent/dock…"   2 minutes ago       Up 2 minutes        0.0.0.0:9092->9092/tcp, 0.0.0.0:9094->9094/tcp   feast_kafka_1
+e2779672735c        confluentinc/cp-zookeeper:5.2.1        "/etc/confluent/dock…"   2 minutes ago       Up 2 minutes        2181/tcp, 2888/tcp, 3888/tcp                     feast_zookeeper_1
+39ac26f5c709        postgres:12-alpine                     "docker-entrypoint.s…"   2 minutes ago       Up 2 minutes        5432/tcp                                         feast_db_1
+3c4ee8616096        redis:5-alpine                         "docker-entrypoint.s…"   2 minutes ago       Up 2 minutes        0.0.0.0:6379->6379/tcp                           feast_redis_1
+```
+
+### Google Kubernetes Engine
+
+All services should either be in a `running` state or `complete`state:
+
+```text
+kubectl get pods
+```
+
+```text
+NAME                                                READY   STATUS      RESTARTS   AGE
+feast-feast-core-5ff566f946-4wlbh                   1/1     Running     1          32m
+feast-feast-serving-batch-848d74587b-96hq6          1/1     Running     2          32m
+feast-feast-serving-online-df69755d5-fml8v          1/1     Running     2          32m
+feast-kafka-0                                       1/1     Running     1          32m
+feast-kafka-1                                       1/1     Running     0          30m
+feast-kafka-2                                       1/1     Running     0          29m
+feast-kafka-config-3e860262-zkzr8                   0/1     Completed   0          32m
+feast-postgresql-0                                  1/1     Running     0          32m
+feast-prometheus-statsd-exporter-554db85b8d-r4hb8   1/1     Running     0          32m
+feast-redis-master-0                                1/1     Running     0          32m
+feast-zookeeper-0                                   1/1     Running     0          32m
+feast-zookeeper-1                                   1/1     Running     0          32m
+feast-zookeeper-2                                   1/1     Running     0          31m
+```
+
+## How can I verify that I can connect to all services?
+
+First find the `IP:Port` combination of your services.
+
+### **Docker Compose \(from inside the docker cluster\)**
+
+You will probably need to connect using the hostnames of services and standard Feast ports:
+
+```bash
+export FEAST_CORE_URL=core:6565
+export FEAST_ONLINE_SERVING_URL=online-serving:6566
+export FEAST_BATCH_SERVING_URL=batch-serving:6567
+```
+
+### **Docker Compose \(from outside the docker cluster\)**
+
+You will probably need to connect using `localhost` and standard ports:
+
+```bash
+export FEAST_CORE_URL=localhost:6565
+export FEAST_ONLINE_SERVING_URL=localhost:6566
+export FEAST_BATCH_SERVING_URL=localhost:6567
+```
+
+### **Google Kubernetes Engine \(GKE\)**
+
+You will need to find the external IP of one of the nodes as well as the NodePorts. Please make sure that your firewall is open for these ports:
+
+```bash
+export FEAST_IP=$(kubectl describe nodes | grep ExternalIP | awk '{print $2}' | head -n 1)
+export FEAST_CORE_URL=${FEAST_IP}:32090
+export FEAST_ONLINE_SERVING_URL=${FEAST_IP}:32091
+export FEAST_BATCH_SERVING_URL=${FEAST_IP}:32092
+```
+
+`netcat`, `telnet`, or even `curl` can be used to test whether all services are available and ports are open, but `grpc_cli` is the most powerful. It can be installed from [here](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md).
+
+### Testing Feast Core:
+
+```bash
+grpc_cli ls ${FEAST_CORE_URL} feast.core.CoreService
+```
+
+```text
+GetFeastCoreVersion
+GetFeatureSet
+ListFeatureSets
+ListStores
+ApplyFeatureSet
+UpdateStore
+CreateProject
+ArchiveProject
+ListProjects
+```
+
+### Testing Feast Batch Serving and Online Serving
+
+```bash
+grpc_cli ls ${FEAST_BATCH_SERVING_URL} feast.serving.ServingService
+```
+
+```text
+GetFeastServingInfo
+GetOnlineFeatures
+GetBatchFeatures
+GetJob
+```
+
+```bash
+grpc_cli ls ${FEAST_ONLINE_SERVING_URL} feast.serving.ServingService
+```
+
+```text
+GetFeastServingInfo
+GetOnlineFeatures
+GetBatchFeatures
+GetJob
+```
+
+## How can I print logs from the Feast Services?
+
+Feast will typically have three services that you need to monitor if something goes wrong.
+
+* Feast Core
+* Feast Serving \(Online\)
+* Feast Serving \(Batch\)
+
+In order to print the logs from these services, please run the commands below.
+
+### Docker Compose
+
+```text
+ docker logs -f feast_core_1
+```
+
+```text
+docker logs -f feast_batch-serving_1
+```
+
+```text
+docker logs -f feast_online-serving_1
+```
+
+### Google Kubernetes Engine
+
+```text
+kubectl logs $(kubectl get pods | grep feast-core | awk '{print $1}')
+```
+
+```text
+kubectl logs $(kubectl get pods | grep feast-serving-batch | awk '{print $1}')
+```
+
+```text
+kubectl logs $(kubectl get pods | grep feast-serving-online | awk '{print $1}')
+```
+
diff --git a/docs/roadmap.md b/docs/roadmap.md
new file mode 100644
index 00000000000..b423a0fe12e
--- /dev/null
+++ b/docs/roadmap.md
@@ -0,0 +1,51 @@
+# Roadmap
+
+## Feast 0.5
+
+[Discussion](https://github.com/gojek/feast/issues/527)
+
+#### New functionality
+
+1. Streaming statistics and validation \(M1 from [Feature Validation RFC](https://docs.google.com/document/d/1TPmd7r4mniL9Y-V_glZaWNo5LMXLshEAUpYsohojZ-8/edit)\)
+2. Batch statistics and validation \(M2 from [Feature Validation RFC](https://docs.google.com/document/d/1TPmd7r4mniL9Y-V_glZaWNo5LMXLshEAUpYsohojZ-8/edit)\)
+3. Support for Redis Clusters \([\#502](https://github.com/gojek/feast/issues/502)\)
+4. User authentication & authorization \([\#504](https://github.com/gojek/feast/issues/504)\)
+5. Add feature or feature set descriptions \([\#463](https://github.com/gojek/feast/issues/463)\)
+6. Redis Cluster Support \([\#478](https://github.com/gojek/feast/issues/478)\)
+7. Job management API  ([\#302](https://github.com/gojek/feast/issues/302)\)
+
+#### Technical debt, refactoring, or housekeeping
+1. Clean up and document all configuration options ([\#525](https://github.com/gojek/feast/issues/525)\)
+2. Externalize storage interfaces ([\#402](https://github.com/gojek/feast/issues/402)\)
+3. Reduce memory usage in Redis \([\#515](https://github.com/gojek/feast/issues/515)\)
+4. Support for handling out of order ingestion \([\#273](https://github.com/gojek/feast/issues/273)\)
+5. Remove feature versions and enable automatic data migration \([\#386](https://github.com/gojek/feast/issues/386)\) \([\#462](https://github.com/gojek/feast/issues/462)\)
+6. Tracking of batch ingestion by with dataset\_id/job\_id \([\#461](https://github.com/gojek/feast/issues/461)\)
+7. Write Beam metrics after ingestion to store (not prior) \([\#489](https://github.com/gojek/feast/issues/489)\)
+
+## Feast 0.6
+
+#### New functionality
+
+1. Extended discovery API/SDK \(needs to be scoped
+   1. Resource listing
+   2. Schemas, statistics, metrics
+   3. Entities as a higher-level concept \([\#405](https://github.com/gojek/feast/issues/405)\)
+   4. Add support for discovery based on annotations/labels/tags for easier filtering and discovery
+2. Add support for default values \(needs to be scoped\)
+3. Add support for audit logs \(needs to be scoped\)
+4. Support for an open source warehouse store or connector  \(needs to be scoped\)
+
+#### Technical debt, refactoring, or housekeeping
+
+1. Move all non-registry functionality out of Feast Core and make it optional \(needs to be scoped\)
+   1. Allow Feast serving to use its own local feature sets \(files\)
+   2. Move job management to Feast serving
+   3. Move stream management \(topic generation\) out of Feast core
+2. Remove feature set versions from Feast \(not just retrieval API\) \(needs to be scoped\)
+   1. Allow for auto-migration of data in Feast
+   2. Implement interface for adding a managed data store
+3. Multi-store support for serving \(batch and online\) \(needs to be scoped\)
+
+
+
diff --git a/examples/basic/basic.ipynb b/examples/basic/basic.ipynb
index 6a83e6a08b5..b9893011d97 100644
--- a/examples/basic/basic.ipynb
+++ b/examples/basic/basic.ipynb
@@ -2,12 +2,10 @@
  "cells": [
   {
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "# Feast Basic Customer Transactions Example"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
+   ]
   },
   {
    "cell_type": "markdown",
@@ -17,15 +15,78 @@
     "1. Create a synthetic customer feature dataset\n",
     "2. Register a feature set to represent these features in Feast\n",
     "3. Ingest these features into Feast\n",
-    "4. Create a feature query and retrieve historical feature data\n",
-    "5. Create a feature query and retrieve online feature data"
+    "4. Create a feature query and retrieve online feature data\n",
+    "5. Create a feature query and retrieve historical feature data"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### 1. Clone Feast and install all dependencies"
+    "### 0. Configuration"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "# Feast Core acts as the central feature registry\n",
+    "FEAST_CORE_URL = os.getenv('FEAST_CORE_URL', 'core:6565')\n",
+    "\n",
+    "# Feast Online Serving allows for the retrieval of real-time feature data\n",
+    "FEAST_ONLINE_SERVING_URL = os.getenv('FEAST_ONLINE_SERVING_URL', 'online-serving:6566')\n",
+    "\n",
+    "# Feast Batch Serving allows for the retrieval of historical feature data\n",
+    "FEAST_BATCH_SERVING_URL = os.getenv('FEAST_BATCH_SERVING_URL', 'batch-serving:6567')\n",
+    "\n",
+    "# PYTHON_REPOSITORY_PATH is the path to the Python SDK inside the Feast Git Repo\n",
+    "PYTHON_REPOSITORY_PATH = os.getenv('PYTHON_REPOSITORY_PATH', '../../')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 1. Install Feast SDK"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Install from PyPi"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install --ignore-installed --upgrade feast"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "(Alternative) Install from local repository"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "os.environ['PYTHON_SDK_PATH'] = os.path.join(PYTHON_REPOSITORY_PATH, 'sdk/python')\n",
+    "sys.path.append(os.environ['PYTHON_SDK_PATH'])"
    ]
   },
   {
@@ -34,9 +95,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!git clone https://github.com/gojek/feast.git \\\n",
-    "&& cd feast/sdk/python/ &&  pip install --upgrade --quiet -e . \\\n",
-    "&& pip install --quiet --upgrade pandas numpy protobuf"
+    "!echo $PYTHON_SDK_PATH"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!python -m pip install --ignore-installed --upgrade -e  ${PYTHON_SDK_PATH}"
    ]
   },
   {
@@ -48,7 +116,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -68,34 +136,57 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### 3. Configure Feast services and connect the Feast client"
+    "### 3. Configure Feast services and connect the Feast client\n",
+    "\n",
+    "Connect to Feast Core and Feast Online Serving"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "CORE_URL = 'localhost:6565'\n",
-    "ONLINE_SERVING_URL = 'localhost:6566'\n",
-    "BATCH_SERVING_URL = 'localhost:6567'"
+    "client = Client(core_url=FEAST_CORE_URL, serving_url=FEAST_ONLINE_SERVING_URL)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Create a project workspace"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "client = Client(core_url=CORE_URL, serving_url=BATCH_SERVING_URL) # Connect to Feast Core"
+    "client.create_project('customer_project')"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### 4. Create synthetic customer features"
+    "Set the active project"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client.set_project('customer_project')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 4. Create customer features"
    ]
   },
   {
@@ -107,19 +198,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "days = [datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0).replace(tzinfo=utc) \\\n",
-    "        - timedelta(day) for day in range(31)]\n",
+    "        - timedelta(day) for day in range(3)][::-1]\n",
     "\n",
     "customers = [1001, 1002, 1003, 1004, 1005]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -132,7 +223,7 @@
     "    }\n",
     ")\n",
     "\n",
-    "print(customer_features.head(10))"
+    "print(customer_features.head(500))"
    ]
   },
   {
@@ -147,21 +238,19 @@
    "metadata": {},
    "source": [
     "Now we will create a feature set for these features. Feature sets are essentially a schema that represent\n",
-    "feature values. Feature sets allow Feast to both identify feature values and their structure. \n",
-    "\n",
-    "In this case we need to define any entity columns as well as the maximum age. The entity column in this case is \"customer_id\". Max age is set to 1 day (defined in seconds). This means that for each feature query during retrieval, the serving API will only retrieve features up to a maximum of 1 day per provided timestamp and entity combination. "
+    "feature values. Feature sets allow Feast to both identify feature values and their structure. The following feature set contains no features yet."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "customer_fs = FeatureSet(\n",
     "    \"customer_transactions\",\n",
-    "    max_age=Duration(seconds=86400),\n",
-    "    entities=[Entity(name='customer_id', dtype=ValueType.INT64)]\n",
+    "    entities=[Entity(name='customer_id', dtype=ValueType.INT64)],\n",
+    "    max_age=Duration(seconds=432000)    \n",
     ")"
    ]
   },
@@ -169,12 +258,12 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Here we are automatically inferring the schema from the provided dataset"
+    "Here we are automatically inferring the schema from the provided dataset. The two features from the dataset will be added to the feature set"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -197,7 +286,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -213,7 +302,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -230,7 +319,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -241,7 +330,65 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### 8. Create a batch retrieval query"
+    "### 8. Retrieve online features"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The process of retrieving features from the online API is very similar to that of the batch API. The only major difference is that users do not have to provide timestamps (only the latest features are returned, as long as they are within the maximum age window)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The example below retrieves online features for a single customer: \"1001\". It is possible to retrieve any features from feast, even outside of the current project."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "online_features = client.get_online_features(\n",
+    "    feature_refs=[\n",
+    "        f\"daily_transactions\",\n",
+    "        f\"total_transactions\",\n",
+    "    ],\n",
+    "    entity_rows=[\n",
+    "        GetOnlineFeaturesRequest.EntityRow(\n",
+    "            fields={\n",
+    "                \"customer_id\": Value(\n",
+    "                    int64_val=1001)\n",
+    "            }\n",
+    "        )\n",
+    "    ],\n",
+    ")\n",
+    "print(online_features)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    " "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### The following section requires Google Cloud Platform (Google Cloud Storage and BigQuery)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 9. Create a batch retrieval query"
    ]
   },
   {
@@ -250,12 +397,12 @@
    "source": [
     "In order to retrieve historical feature data, the user must provide an entity_rows dataframe. This dataframe contains a combination of timestamps and entities. In this case, the user must provide both customer_ids and timestamps. \n",
     "\n",
-    "We will randomly generate timestamps over the last 30 days, and assign customer_ids to them. When these entity rows are sent to the Feast Serving API to retrieve feature values, along with a list of feature ids, Feast is then able to attach the correct feature values to each entity row. The one exception is if the feature values fall outside of the maximum age window."
+    "We will randomly generate timestamps over the last 30 days, and assign customer_ids to them. When these entity rows are sent to the Feast Serving API to retrieve feature values, along with a list of feature ids, Feast is then able to attach the correct feature values to each entity row. "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -275,76 +422,55 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### 9. Retrieve historical/batch features"
+    "### 10. Retrieve historical/batch features"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 32,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "job = client.get_batch_features(\n",
-    "                            feature_ids=[\n",
-    "                                f\"customer_transactions:{customer_fs.version}:daily_transactions\", \n",
-    "                                f\"customer_transactions:{customer_fs.version}:total_transactions\", \n",
-    "                               ],\n",
-    "                            entity_rows=entity_rows\n",
-    "                         )\n",
-    "df = job.to_dataframe()\n",
-    "print(df.head(10))"
+    "Next we will create a new client object, but this time we will configure it to connect to the Batch Serving Service. This service will allow us to retrieve historical feature data."
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
+   "outputs": [],
    "source": [
-    "### 10. Retrieve online features"
+    "batch_client = Client(core_url=FEAST_CORE_URL, serving_url=FEAST_BATCH_SERVING_URL)\n",
+    "batch_client.set_project(\"customer_project\")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The process of retrieving features from the online API is very similar to that of the batch API. The only major difference is that users do not have to provide timestamps (only the latest features are returned, as long as they are within the maximum age window)"
+    "By calling the `get_batch_features` method we are able to retrieve a `job` object for the exporting of feature data. For every entity and timestamp combination in `entity_rows` we will be receiving a row with feature values joined to it."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
-   "metadata": {},
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [],
    "source": [
-    "online_client = Client(core_url=CORE_URL, serving_url=ONLINE_SERVING_URL)"
+    "job = batch_client.get_batch_features(\n",
+    "                            feature_refs=[\n",
+    "                                f\"customer_project/daily_transactions\", \n",
+    "                                f\"customer_project/total_transactions\", \n",
+    "                               ],\n",
+    "                            entity_rows=entity_rows\n",
+    "                         )"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The example below retrieves online features for a single customer: \"1001\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 37,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "online_features = online_client.get_online_features(\n",
-    "    feature_ids=[\n",
-    "        f\"customer_transactions:{customer_fs.version}:daily_transactions\",\n",
-    "        f\"customer_transactions:{customer_fs.version}:total_transactions\",\n",
-    "    ],\n",
-    "    entity_rows=[\n",
-    "        GetOnlineFeaturesRequest.EntityRow(\n",
-    "            fields={\n",
-    "                \"customer_id\": Value(\n",
-    "                    int64_val=1001)\n",
-    "            }\n",
-    "        )\n",
-    "    ],\n",
-    ")"
+    "Once the job is complete, it is possible to retrieve the exported data (from Google Cloud Storage) and load it into memory as a Pandas Dataframe."
    ]
   },
   {
@@ -353,7 +479,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "print(online_features)"
+    "df = job.to_dataframe()\n",
+    "print(df.head(10))"
    ]
   }
  ],
@@ -378,13 +505,13 @@
   "pycharm": {
    "stem_cell": {
     "cell_type": "raw",
-    "source": [],
     "metadata": {
      "collapsed": false
-    }
+    },
+    "source": []
    }
   }
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
\ No newline at end of file
+}
diff --git a/go.mod b/go.mod
index 8dc819493e4..5cac45f363e 100644
--- a/go.mod
+++ b/go.mod
@@ -7,8 +7,8 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang/mock v1.2.0
-	github.com/golang/protobuf v1.3.2
-	github.com/google/go-cmp v0.3.0
+	github.com/golang/protobuf v1.4.1
+	github.com/google/go-cmp v0.4.0
 	github.com/huandu/xstrings v1.2.0 // indirect
 	github.com/lyft/protoc-gen-validate v0.1.0 // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index 8ded1a626ec..4ef895cc929 100644
--- a/go.sum
+++ b/go.sum
@@ -129,12 +129,22 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -389,6 +399,7 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
@@ -408,6 +419,14 @@ google.golang.org/grpc v1.19.1 h1:TrBcJ1yqAl1G++wO39nD/qtgpsW9/1+QGrluyMGEYgM=
 google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0 h1:cJv5/xdbk1NnMPR1VP9+HU6gupuG9MLBoH1r6RHZ2MY=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml
index 80d17aeb6ba..c8f328548a9 100644
--- a/infra/charts/feast/Chart.yaml
+++ b/infra/charts/feast/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 description: A Helm chart to install Feast on kubernetes
 name: feast
-version: 0.3.2
+version: 0.4.4
diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md
index 0463a9a3f89..e93b687f191 100644
--- a/infra/charts/feast/README.md
+++ b/infra/charts/feast/README.md
@@ -36,10 +36,10 @@ helm repo add feast-charts https://feast-charts.storage.googleapis.com
 helm repo update
 ```
 
-Install Feast release with minimal features, without batch serving and persistency:
+Install Feast release with minimal features, without batch serving and persistence:
 ```bash
 RELEASE_NAME=demo
-helm install feast-charts/feast --name $RELEASE_NAME --version 0.3.2 -f values-demo.yaml
+helm install feast-charts/feast --name $RELEASE_NAME -f values-demo.yaml
 ```
 
 Install Feast release for typical use cases, with batch and online serving:
@@ -60,7 +60,7 @@ PROJECT_ID=google-cloud-project-id
 DATASET_ID=bigquery-dataset-id
 
 # Install the Helm release using default values.yaml
-helm install feast-charts/feast --name feast --version 0.3.2 \
+helm install feast-charts/feast --name feast \
   --set feast-serving-batch."application\.yaml".feast.jobs.staging-location=$STAGING_LOCATION \
   --set feast-serving-batch."store\.yaml".bigquery_config.project_id=$PROJECT_ID \
   --set feast-serving-batch."store\.yaml".bigquery_config.dataset_id=$DATASET_ID
@@ -81,17 +81,26 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-core.kafka.topics[0].name` |  Default topic name in Kafka| `feast`
 | `feast-core.kafka.topics[0].replicationFactor` |  No of replication factor for the topic| `1`
 | `feast-core.kafka.topics[0].partitions` |  No of partitions for the topic | `1`
+| `feast-core.prometheus-statsd-exporter.enabled` | Flag to install Prometheus StatsD Exporter | `false`
+| `feast-core.prometheus-statsd-exporter.*` | Refer to this [link](charts/feast-core/charts/prometheus-statsd-exporter/values.yaml  |
 | `feast-core.replicaCount` | No of pods to create | `1`
 | `feast-core.image.repository` | Repository for Feast Core Docker image | `gcr.io/kf-feast/feast-core`
-| `feast-core.image.tag` | Tag for Feast Core Docker image | `0.3.2`
+| `feast-core.image.tag` | Tag for Feast Core Docker image | `0.4.4`
 | `feast-core.image.pullPolicy` | Image pull policy for Feast Core Docker image | `IfNotPresent`
+| `feast-core.prometheus.enabled` | Add annotations to enable Prometheus scraping | `false`
 | `feast-core.application.yaml` | Configuration for Feast Core application | Refer to this [link](charts/feast-core/values.yaml) 
 | `feast-core.springConfigMountPath` | Directory to mount application.yaml | `/etc/feast/feast-core`
 | `feast-core.gcpServiceAccount.useExistingSecret` | Flag to use existing secret for GCP service account | `false`
 | `feast-core.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account`
 | `feast-core.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json`
 | `feast-core.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts`
+| `feast-core.gcpProjectId` | Project ID to set `GOOGLE_CLOUD_PROJECT` to change default project used by SDKs | `""`
+| `feast-core.jarPath` | Path to Jar file in the Docker image | `/opt/feast/feast-core.jar`
 | `feast-core.jvmOptions` | Options for the JVM | `[]`
+| `feast-core.logLevel` | Application logging level | `warn`
+| `feast-core.logType` | Application logging type (`JSON` or `Console`) | `JSON`
+| `feast-core.springConfigProfiles` | Map of profile name to file content for additional Spring profiles | `{}`
+| `feast-core.springConfigProfilesActive` | CSV of profiles to enable from `springConfigProfiles` | `""`
 | `feast-core.livenessProbe.enabled` | Flag to enable liveness probe | `true`
 | `feast-core.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60`
 | `feast-core.livenessProbe.periodSeconds` | How often to perform the probe | `10`
@@ -109,6 +118,7 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-core.grpc.port` | Kubernetes Service port for GRPC request| `6565`
 | `feast-core.grpc.targetPort` | Container port for GRPC request| `6565`
 | `feast-core.resources` | CPU and memory allocation for the pod | `{}`
+| `feast-core.ingress` | See *Ingress Parameters* [below](#ingress-parameters) | `{}`
 | `feast-serving-online.enabled` | Flag to install Feast Online Serving | `true`
 | `feast-serving-online.redis.enabled` | Flag to install Redis in Feast Serving | `false`
 | `feast-serving-online.redis.usePassword` | Flag to use password to access Redis | `false`
@@ -116,8 +126,9 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-serving-online.core.enabled` | Flag for Feast Serving to use Feast Core in the same Helm release | `true`
 | `feast-serving-online.replicaCount` | No of pods to create  | `1`
 | `feast-serving-online.image.repository` | Repository for Feast Serving Docker image | `gcr.io/kf-feast/feast-serving`
-| `feast-serving-online.image.tag` | Tag for Feast Serving Docker image | `0.3.2`
+| `feast-serving-online.image.tag` | Tag for Feast Serving Docker image | `0.4.4`
 | `feast-serving-online.image.pullPolicy` | Image pull policy for Feast Serving Docker image | `IfNotPresent`
+| `feast-serving-online.prometheus.enabled` | Add annotations to enable Prometheus scraping | `true`
 | `feast-serving-online.application.yaml` | Application configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) 
 | `feast-serving-online.store.yaml` | Store configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) 
 | `feast-serving-online.springConfigMountPath` | Directory to mount application.yaml and store.yaml | `/etc/feast/feast-serving`
@@ -125,7 +136,13 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-serving-online.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account`
 | `feast-serving-online.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json`
 | `feast-serving-online.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts`
+| `feast-serving-online.gcpProjectId` | Project ID to set `GOOGLE_CLOUD_PROJECT` to change default project used by SDKs | `""`
+| `feast-serving-online.jarPath` | Path to Jar file in the Docker image | `/opt/feast/feast-serving.jar`
 | `feast-serving-online.jvmOptions` | Options for the JVM | `[]`
+| `feast-serving-online.logLevel` | Application logging level | `warn`
+| `feast-serving-online.logType` | Application logging type (`JSON` or `Console`) | `JSON`
+| `feast-serving-online.springConfigProfiles` | Map of profile name to file content for additional Spring profiles | `{}`
+| `feast-serving-online.springConfigProfilesActive` | CSV of profiles to enable from `springConfigProfiles` | `""`
 | `feast-serving-online.livenessProbe.enabled` | Flag to enable liveness probe | `true`
 | `feast-serving-online.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60`
 | `feast-serving-online.livenessProbe.periodSeconds` | How often to perform the probe | `10`
@@ -143,6 +160,7 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-serving-online.grpc.port` | Kubernetes Service port for GRPC request| `6566`
 | `feast-serving-online.grpc.targetPort` | Container port for GRPC request| `6566`
 | `feast-serving-online.resources` | CPU and memory allocation for the pod | `{}`
+| `feast-serving-online.ingress` | See *Ingress Parameters* [below](#ingress-parameters) | `{}`
 | `feast-serving-batch.enabled` | Flag to install Feast Batch Serving | `true`
 | `feast-serving-batch.redis.enabled` | Flag to install Redis in Feast Serving | `false`
 | `feast-serving-batch.redis.usePassword` | Flag to use password to access Redis | `false`
@@ -150,8 +168,9 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-serving-batch.core.enabled` | Flag for Feast Serving to use Feast Core in the same Helm release | `true`
 | `feast-serving-batch.replicaCount` | No of pods to create  | `1`
 | `feast-serving-batch.image.repository` | Repository for Feast Serving Docker image | `gcr.io/kf-feast/feast-serving`
-| `feast-serving-batch.image.tag` | Tag for Feast Serving Docker image | `0.3.2`
+| `feast-serving-batch.image.tag` | Tag for Feast Serving Docker image | `0.4.4`
 | `feast-serving-batch.image.pullPolicy` | Image pull policy for Feast Serving Docker image | `IfNotPresent`
+| `feast-serving-batch.prometheus.enabled` | Add annotations to enable Prometheus scraping | `true`
 | `feast-serving-batch.application.yaml` | Application configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) 
 | `feast-serving-batch.store.yaml` | Store configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) 
 | `feast-serving-batch.springConfigMountPath` | Directory to mount application.yaml and store.yaml | `/etc/feast/feast-serving`
@@ -159,7 +178,13 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-serving-batch.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account`
 | `feast-serving-batch.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json`
 | `feast-serving-batch.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts`
+| `feast-serving-batch.gcpProjectId` | Project ID to set `GOOGLE_CLOUD_PROJECT` to change default project used by SDKs | `""`
+| `feast-serving-batch.jarPath` | Path to Jar file in the Docker image | `/opt/feast/feast-serving.jar`
 | `feast-serving-batch.jvmOptions` | Options for the JVM | `[]`
+| `feast-serving-batch.logLevel` | Application logging level | `warn`
+| `feast-serving-batch.logType` | Application logging type (`JSON` or `Console`) | `JSON`
+| `feast-serving-batch.springConfigProfiles` | Map of profile name to file content for additional Spring profiles | `{}`
+| `feast-serving-batch.springConfigProfilesActive` | CSV of profiles to enable from `springConfigProfiles` | `""`
 | `feast-serving-batch.livenessProbe.enabled` | Flag to enable liveness probe | `true`
 | `feast-serving-batch.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60`
 | `feast-serving-batch.livenessProbe.periodSeconds` | How often to perform the probe | `10`
@@ -176,4 +201,51 @@ The following table lists the configurable parameters of the Feast chart and the
 | `feast-serving-batch.http.targetPort` | Container port for HTTP request | `8080`
 | `feast-serving-batch.grpc.port` | Kubernetes Service port for GRPC request| `6566`
 | `feast-serving-batch.grpc.targetPort` | Container port for GRPC request| `6566`
-| `feast-serving-batch.resources` | CPU and memory allocation for the pod | `{}`
\ No newline at end of file
+| `feast-serving-batch.resources` | CPU and memory allocation for the pod | `{}`
+| `feast-serving-batch.ingress` | See *Ingress Parameters* [below](#ingress-parameters) | `{}`
+
+## Ingress Parameters
+
+The following table lists the configurable parameters of the ingress section for each Feast module.
+
+Note, there are two ingresses available for each module - `grpc` and `http`.
+
+| Parameter                     | Description | Default
+| ----------------------------- | ----------- | -------
+| `ingress.grcp.enabled`        | Enables an ingress (endpoint) for the gRPC server | `false`
+| `ingress.grcp.*`              | See below |
+| `ingress.http.enabled`        | Enables an ingress (endpoint) for the HTTP server | `false`
+| `ingress.http.*`              | See below |
+| `ingress.*.class`             | Value for `kubernetes.io/ingress.class` | `nginx`
+| `ingress.*.hosts`             | List of host-names for the ingress | `[]`
+| `ingress.*.annotations`       | Additional ingress annotations | `{}`
+| `ingress.*.https.enabled`     | Add a tls section to the ingress | `true`
+| `ingress.*.https.secretNames` | Map of hostname to TLS secret name | `{}` If not specified, defaults to `domain-tld-tls` e.g. `feast.example.com` uses secret `example-com-tls`
+| `ingress.*.auth.enabled`      | Enable auth on the ingress (only applicable for `nginx` type | `false`
+| `ingress.*.auth.signinHost`   | External hostname of the OAuth2 proxy to use | First item in `ingress.hosts`, replacing the sub-domain with 'auth' e.g. `feast.example.com` uses `auth.example.com`
+| `ingress.*.auth.authUrl`      | Internal URI to internal auth endpoint | `http://auth-server.auth-ns.svc.cluster.local/auth`
+| `ingress.*.whitelist`         | Subnet masks to whitelist (i.e. value for `nginx.ingress.kubernetes.io/whitelist-source-range`) | `"""`
+
+To enable all the ingresses will a config like the following (while also adding the hosts etc):
+
+```yaml
+feast-core:
+  ingress:
+    grpc:
+      enabled: true
+    http:
+      enabled: true
+feast-serving-online:
+  ingress:
+    grpc:
+      enabled: true
+    http:
+      enabled: true
+feast-serving-batch:
+  ingress:
+    grpc:
+      enabled: true
+    http:
+      enabled: true
+```
+
diff --git a/infra/charts/feast/charts/feast-core/Chart.yaml b/infra/charts/feast/charts/feast-core/Chart.yaml
index efda76d3026..86d0699b9ac 100644
--- a/infra/charts/feast/charts/feast-core/Chart.yaml
+++ b/infra/charts/feast/charts/feast-core/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 description: A Helm chart for core component of Feast
 name: feast-core
-version: 0.3.2
+version: 0.4.4
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/.helmignore b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/.helmignore
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/.helmignore
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/.helmignore
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/Chart.yaml b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/Chart.yaml
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/Chart.yaml
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/Chart.yaml
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/README.md b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/README.md
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/README.md
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/README.md
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/NOTES.txt b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/NOTES.txt
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/templates/NOTES.txt
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/NOTES.txt
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/_helpers.tpl b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/_helpers.tpl
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/templates/_helpers.tpl
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/_helpers.tpl
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/config.yaml b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/config.yaml
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/templates/config.yaml
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/config.yaml
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/deployment.yaml b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/deployment.yaml
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/templates/deployment.yaml
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/deployment.yaml
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/pvc.yaml b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/pvc.yaml
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/templates/pvc.yaml
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/pvc.yaml
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/service.yaml b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/service.yaml
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/templates/service.yaml
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/service.yaml
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/templates/serviceaccount.yaml b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/serviceaccount.yaml
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/templates/serviceaccount.yaml
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/templates/serviceaccount.yaml
diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/values.yaml b/infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/values.yaml
similarity index 100%
rename from infra/charts/feast/charts/prometheus-statsd-exporter/values.yaml
rename to infra/charts/feast/charts/feast-core/charts/prometheus-statsd-exporter/values.yaml
diff --git a/infra/charts/feast/charts/feast-core/requirements.yaml b/infra/charts/feast/charts/feast-core/requirements.yaml
index efe9fec508a..ef1e39a7d0f 100644
--- a/infra/charts/feast/charts/feast-core/requirements.yaml
+++ b/infra/charts/feast/charts/feast-core/requirements.yaml
@@ -6,4 +6,10 @@ dependencies:
 - name: kafka
   version: 0.20.1
   repository: "@incubator"
-  condition: kafka.enabled
\ No newline at end of file
+  condition: kafka.enabled
+- name: common
+  version: 0.0.5
+  repository: "@incubator"
+- name: prometheus-statsd-exporter
+  version: 0.1.2
+  condition: prometheus-statsd-exporter.enabled
\ No newline at end of file
diff --git a/infra/charts/feast/charts/feast-core/templates/_ingress.yaml b/infra/charts/feast/charts/feast-core/templates/_ingress.yaml
new file mode 100644
index 00000000000..5bed6df0470
--- /dev/null
+++ b/infra/charts/feast/charts/feast-core/templates/_ingress.yaml
@@ -0,0 +1,68 @@
+{{- /*
+This takes an array of three values:
+- the top context
+- the feast component
+- the service protocol
+- the ingress context
+*/ -}}
+{{- define "feast.ingress" -}}
+{{- $top := (index . 0) -}}
+{{- $component := (index . 1) -}}
+{{- $protocol := (index . 2) -}}
+{{- $ingressValues := (index . 3) -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+{{ include "feast.ingress.metadata" . }}
+spec:
+  rules:
+  {{- range $host := $ingressValues.hosts }}
+  - host: {{ $host }}
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: {{ include (printf "feast-%s.fullname" $component) $top }}
+          servicePort: {{ index $top.Values "service" $protocol "port" }}
+  {{- end }}
+{{- if $ingressValues.https.enabled }}
+  tls:
+    {{- range $host := $ingressValues.hosts }}
+    - secretName: {{ index $ingressValues.https.secretNames $host | default (splitList "." $host | rest | join "-" | printf "%s-tls") }}
+      hosts:
+      - {{ $host }}
+    {{- end }}
+{{- end -}}
+{{- end -}}
+
+{{- define "feast.ingress.metadata" -}}
+{{- $commonMetadata := fromYaml (include "common.metadata" (first .)) }}
+{{- $overrides := fromYaml (include "feast.ingress.metadata-overrides" .) -}}
+{{- toYaml (merge $overrides $commonMetadata) -}}
+{{- end -}}
+
+{{- define "feast.ingress.metadata-overrides" -}}
+{{- $top := (index . 0) -}}
+{{- $component := (index . 1) -}}
+{{- $protocol := (index . 2) -}}
+{{- $ingressValues := (index . 3) -}}
+{{- $commonFullname := include "common.fullname" $top }}
+metadata:
+  name: {{ $commonFullname }}-{{ $component }}-{{ $protocol }}
+  annotations:
+    kubernetes.io/ingress.class: {{ $ingressValues.class | quote }}
+    {{- if (and (eq $ingressValues.class "nginx") $ingressValues.auth.enabled) }}
+    nginx.ingress.kubernetes.io/auth-url: {{ $ingressValues.auth.authUrl | quote }}
+    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-email, x-auth-request-user"
+    nginx.ingress.kubernetes.io/auth-signin: "https://{{ $ingressValues.auth.signinHost | default (splitList "." (index $ingressValues.hosts 0) | rest | join "." | printf "auth.%s")}}/oauth2/start?rd=/r/$host/$request_uri"
+    {{- end }}
+    {{- if (and (eq $ingressValues.class "nginx") $ingressValues.whitelist) }}
+    nginx.ingress.kubernetes.io/whitelist-source-range: {{ $ingressValues.whitelist | quote -}}
+    {{- end }}
+    {{- if (and (eq $ingressValues.class "nginx") (eq $protocol "grpc") ) }}
+    # TODO: Allow choice of GRPC/GRPCS
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
+    {{- if $ingressValues.annotations -}}
+    {{ include "common.annote" $ingressValues.annotations | indent 4 }}
+    {{- end }}
+{{- end -}}
diff --git a/infra/charts/feast/charts/feast-core/templates/configmap.yaml b/infra/charts/feast/charts/feast-core/templates/configmap.yaml
index 68dc45c0571..da45cad5bdf 100644
--- a/infra/charts/feast/charts/feast-core/templates/configmap.yaml
+++ b/infra/charts/feast/charts/feast-core/templates/configmap.yaml
@@ -11,22 +11,43 @@ metadata:
     heritage: {{ .Release.Service }}
 data:
   application.yaml: |
-{{- $config := index .Values "application.yaml"}}
+{{- toYaml (index .Values "application.yaml") | nindent 4 }}
 
 {{- if .Values.postgresql.enabled }}
-{{- $datasource := dict "url" (printf "jdbc:postgresql://%s:%s/%s" (printf "%s-postgresql" .Release.Name) (.Values.postgresql.service.port | toString) (.Values.postgresql.postgresqlDatabase)) "driverClassName" "org.postgresql.Driver" }}
-{{- $newConfig := dict "spring" (dict "datasource" $datasource) }}
-{{- $config := mergeOverwrite $config $newConfig }}
+  application-bundled-postgresql.yaml: |
+    spring:
+      datasource:
+        url: {{ printf "jdbc:postgresql://%s:%s/%s" (printf "%s-postgresql" .Release.Name) (.Values.postgresql.service.port | toString) (.Values.postgresql.postgresqlDatabase) }}
+        driverClassName: org.postgresql.Driver
 {{- end }}
 
-{{- if .Values.kafka.enabled }}
-{{- $topic := index .Values.kafka.topics 0 }}
-{{- $options := dict "topic" $topic.name "replicationFactor" $topic.replicationFactor "partitions" $topic.partitions }}
-{{- if not .Values.kafka.external.enabled }}
-{{- $_ := set $options "bootstrapServers" (printf "%s:9092" (printf "%s-kafka" .Release.Name)) }}
+{{ if .Values.kafka.enabled }}
+  {{- $topic := index .Values.kafka.topics 0 }}
+  application-bundled-kafka.yaml: |
+    feast:
+      stream:
+        type: kafka
+        options:
+          topic: {{ $topic.name | quote }}
+          replicationFactor: {{ $topic.replicationFactor }}
+          partitions: {{ $topic.partitions }}
+          {{- if not .Values.kafka.external.enabled }}
+          bootstrapServers: {{ printf "%s:9092" (printf "%s-kafka" .Release.Name) }}
+          {{- end }}
 {{- end }}
-{{- $newConfig := dict "feast" (dict "stream" (dict "type" "kafka" "options" $options))}}
-{{- $config := mergeOverwrite $config $newConfig }}
+
+{{- if (index .Values "prometheus-statsd-exporter" "enabled" )}}
+  application-bundled-statsd.yaml: |
+    feast:
+      jobs:
+        metrics:
+          enabled: true
+          type: statsd
+          host: prometheus-statsd-exporter
+          port: 9125
 {{- end }}
 
-{{- toYaml $config | nindent 4 }}
+{{- range $name, $content := .Values.springConfigProfiles }}
+  application-{{ $name }}.yaml: |
+{{- toYaml $content | nindent 4 }}
+{{- end }}
diff --git a/infra/charts/feast/charts/feast-core/templates/deployment.yaml b/infra/charts/feast/charts/feast-core/templates/deployment.yaml
index 02a533c2637..df834b6749e 100644
--- a/infra/charts/feast/charts/feast-core/templates/deployment.yaml
+++ b/infra/charts/feast/charts/feast-core/templates/deployment.yaml
@@ -18,6 +18,13 @@ spec:
       release: {{ .Release.Name }}
   template:
     metadata:
+      {{- if .Values.prometheus.enabled }}
+      annotations:
+      {{ $config := index .Values "application.yaml" }}
+        prometheus.io/path: /metrics
+        prometheus.io/port: "{{ $config.server.port }}"
+        prometheus.io/scrape: "true"
+      {{- end }}
       labels:
         app: {{ template "feast-core.name" . }}
         component: core
@@ -40,9 +47,9 @@ spec:
 
       containers:
       - name: {{ .Chart.Name }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        image: '{{ .Values.image.repository }}:{{ required "No .image.tag found. This must be provided as input."  .Values.image.tag }}'
         imagePullPolicy: {{ .Values.image.pullPolicy }}
-        
+
         volumeMounts:
         - name: {{ template "feast-core.fullname" . }}-config
           mountPath: "{{ .Values.springConfigMountPath }}"
@@ -53,31 +60,48 @@ spec:
         {{- end }}
 
         env:
+        - name: LOG_TYPE
+          value: {{ .Values.logType | quote }}
+        - name: LOG_LEVEL
+          value: {{ .Values.logLevel | quote }}
+
         {{- if .Values.postgresql.enabled }}
         - name: SPRING_DATASOURCE_USERNAME
-          value: {{ .Values.postgresql.postgresqlUsername }}
+          value: {{ .Values.postgresql.postgresqlUsername | quote }}
         - name: SPRING_DATASOURCE_PASSWORD
-          value: {{ .Values.postgresql.postgresqlPassword }}
+          value: {{ .Values.postgresql.postgresqlPassword | quote }}
         {{- end }}
 
         {{- if .Values.gcpServiceAccount.useExistingSecret }}
         - name: GOOGLE_APPLICATION_CREDENTIALS
           value: {{ .Values.gcpServiceAccount.mountPath }}/{{ .Values.gcpServiceAccount.existingSecret.key }}
         {{- end }}
+        {{- if .Values.gcpProjectId }}
+        - name: GOOGLE_CLOUD_PROJECT
+          value: {{ .Values.gcpProjectId | quote }}
+        {{- end }}
 
         command:
         - java
         {{- range .Values.jvmOptions }}
-        - {{ . }}
+        - {{ . | quote }}
+        {{- end }}
+        - -jar
+        - {{ .Values.jarPath | quote }}
+        - "--spring.config.location=file:{{ .Values.springConfigMountPath }}/"
+        {{- $profilesArray := splitList "," .Values.springConfigProfilesActive -}}
+        {{- $profilesArray = append $profilesArray (.Values.postgresql.enabled | ternary "bundled-postgresql" "") -}}
+        {{- $profilesArray = append $profilesArray (.Values.kafka.enabled | ternary "bundled-kafka" "") -}}
+        {{- $profilesArray = append $profilesArray (index .Values "prometheus-statsd-exporter" "enabled" | ternary "bundled-statsd" "") -}}
+        {{- $profilesArray = compact $profilesArray -}}
+        {{- if $profilesArray }}
+        - "--spring.profiles.active={{ join "," $profilesArray }}"
         {{- end }}
-        - -jar 
-        - /opt/feast/feast-core.jar
-        - "--spring.config.location=file:{{ .Values.springConfigMountPath }}/application.yaml"
 
         ports:
         - name: http
           containerPort: {{ .Values.service.http.targetPort }}
-        - name: grpc 
+        - name: grpc
           containerPort: {{ .Values.service.grpc.targetPort }}
 
         {{- if .Values.livenessProbe.enabled }}
@@ -103,6 +127,6 @@ spec:
           timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
           failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
         {{- end }}
-        
+
         resources:
           {{- toYaml .Values.resources | nindent 10 }}
diff --git a/infra/charts/feast/charts/feast-core/templates/ingress.yaml b/infra/charts/feast/charts/feast-core/templates/ingress.yaml
index 86fc2d3f175..7f453e1a75f 100644
--- a/infra/charts/feast/charts/feast-core/templates/ingress.yaml
+++ b/infra/charts/feast/charts/feast-core/templates/ingress.yaml
@@ -1,28 +1,7 @@
-{{- if .Values.ingress.enabled -}}
-{{- $fullName := include "feast-core.fullname" . -}}
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  name: {{ $fullName }}
-  labels:
-    app: {{ template "feast-core.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    component: core
-    heritage: {{ .Release.Service }}
-    release: {{ .Release.Name }}
-  annotations:
-{{- with .Values.ingress.annotations }}
-{{ toYaml . | indent 4 }}
+{{- if .Values.ingress.http.enabled -}}
+{{ template "feast.ingress" (list . "core" "http" .Values.ingress.http) }}
+{{- end }}
+---
+{{ if .Values.ingress.grpc.enabled -}}
+{{ template "feast.ingress" (list . "core" "grpc" .Values.ingress.grpc) }}
 {{- end }}
-spec:
-  rules:
-  {{- range .Values.ingress.hosts }}
-    - host: {{ .host | quote }}
-      http:
-        paths:
-        - path: /
-          backend:
-            serviceName: {{ $fullName }}
-            servicePort: {{ .port | quote }}
-  {{- end }}
-{{- end }}
\ No newline at end of file
diff --git a/infra/charts/feast/charts/feast-core/values.yaml b/infra/charts/feast/charts/feast-core/values.yaml
index 321d71c844d..077906dc35d 100644
--- a/infra/charts/feast/charts/feast-core/values.yaml
+++ b/infra/charts/feast/charts/feast-core/values.yaml
@@ -1,12 +1,15 @@
-# postgresql configures Postgresql that is installed as part of Feast Core.
+# ============================================================
+# Bundled PostgreSQL
+# ============================================================
+
 # Refer to https://github.com/helm/charts/tree/c42002a21abf8eff839ff1d2382152bde2bbe596/stable/postgresql
 # for additional configuration.
 postgresql:
   # enabled specifies whether Postgresql should be installed as part of Feast Core.
   #
-  # Feast Core requires a database to store data such as the created FeatureSets 
+  # Feast Core requires a database to store data such as the created FeatureSets
   # and job statuses. If enabled, the database and service port specified below
-  # will override "spring.datasource.url" value in application.yaml. The 
+  # will override "spring.datasource.url" value in application.yaml. The
   # username and password will also be set as environment variables that will
   # override "spring.datasource.username/password" in application.yaml.
   enabled: true
@@ -20,12 +23,15 @@ postgresql:
     # port is the TCP port that Postgresql will listen to
     port: 5432
 
-# kafka configures Kafka that is installed as part of Feast Core.
+# ============================================================
+# Bundled Kafka
+# ============================================================
+
 # Refer to https://github.com/helm/charts/tree/c42002a21abf8eff839ff1d2382152bde2bbe596/incubator/kafka
 # for additional configuration.
 kafka:
   # enabled specifies whether Kafka should be installed as part of Feast Core.
-  # 
+  #
   # Feast Core requires a Kafka instance to be set as the default source for
   # FeatureRows. If enabled, "feast.stream" option in application.yaml will
   # be overridden by this installed Kafka configuration.
@@ -36,22 +42,38 @@ kafka:
     replicationFactor: 1
     partitions: 1
 
+
+# ============================================================
+# Bundled Prometheus StatsD Exporter
+# ============================================================
+
+prometheus-statsd-exporter:
+  enabled: false
+
+# ============================================================
+# Feast Core
+# ============================================================
+
 # replicaCount is the number of pods that will be created.
 replicaCount: 1
 
 # image configures the Docker image for Feast Core
 image:
   repository: gcr.io/kf-feast/feast-core
-  tag: 0.3.2
   pullPolicy: IfNotPresent
 
+# Add prometheus scraping annotations to the Pod metadata.
+# If enabled, you must also ensure server.port is specified under application.yaml
+prometheus:
+  enabled: false
+
 # application.yaml is the main configuration for Feast Core application.
-# 
+#
 # Feast Core is a Spring Boot app which uses this yaml configuration file.
 # Refer to https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/core/src/main/resources/application.yml
 # for a complete list and description of the configuration.
 #
-# Note that some properties defined in application.yaml may be overriden by 
+# Note that some properties defined in application.yaml may be overriden by
 # Helm under certain conditions. For example, if postgresql and kafka dependencies
 # are enabled.
 application.yaml:
@@ -97,7 +119,14 @@ application.yaml:
           host: localhost
           port: 8125
 
-# springConfigMountPath is the directory path where application.yaml will be 
+springConfigProfiles: {}
+#  db: |
+#    spring:
+#      datasource:
+#        driverClassName: org.postgresql.Driver
+#        url: jdbc:postgresql://${DB_HOST:127.0.0.1}:${DB_PORT:5432}/${DB_DATABASE:postgres}
+springConfigProfilesActive: ""
+# springConfigMountPath is the directory path where application.yaml will be
 # mounted in the container.
 springConfigMountPath: /etc/feast/feast-core
 
@@ -108,7 +137,7 @@ gcpServiceAccount:
   useExistingSecret: false
   existingSecret:
     # name is the secret name of the existing secret for the service account.
-    name: feast-gcp-service-account 
+    name: feast-gcp-service-account
     # key is the secret key of the existing secret for the service account.
     # key is normally derived from the file name of the JSON key file.
     key: key.json
@@ -116,19 +145,29 @@ gcpServiceAccount:
   # the value of "existingSecret.key" is file name of the service account file.
   mountPath: /etc/gcloud/service-accounts
 
-# jvmOptions are options that will be passed to the Java Virtual Machine (JVM) 
+# Project ID picked up by the Cloud SDK (e.g. BigQuery run against this project)
+gcpProjectId: ""
+
+# Path to Jar file in the Docker image.
+# If you are using gcr.io/kf-feast/feast-core this should not need to be changed
+jarPath: /opt/feast/feast-core.jar
+
+# jvmOptions are options that will be passed to the Java Virtual Machine (JVM)
 # running Feast Core.
-# 
+#
 # For example, it is good practice to set min and max heap size in JVM.
 # https://stackoverflow.com/questions/6902135/side-effect-for-increasing-maxpermsize-and-max-heap-size
 #
 # Refer to https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html
 # to see other JVM options that can be set.
 #
-# jvmOptions: 
-# - -Xms1024m 
+jvmOptions: []
+# - -Xms1024m
 # - -Xmx1024m
 
+logType: JSON
+logLevel: warn
+
 livenessProbe:
   enabled: true
   initialDelaySeconds: 60
@@ -163,12 +202,29 @@ service:
     # nodePort:
 
 ingress:
-  enabled: false
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-  hosts:
-  # - host: chart-example.local
-  #   port: http
+  grpc:
+    enabled: false
+    class: nginx
+    hosts: []
+    annotations: {}
+    https:
+      enabled: true
+      secretNames: {}
+    whitelist: ""
+    auth:
+      enabled: false
+  http:
+    enabled: false
+    class: nginx
+    hosts: []
+    annotations: {}
+    https:
+      enabled: true
+      secretNames: {}
+    whitelist: ""
+    auth:
+      enabled: false
+      authUrl: http://auth-server.auth-ns.svc.cluster.local/auth
 
 resources: {}
   # We usually recommend not to specify default resources and to leave this as a conscious
diff --git a/infra/charts/feast/charts/feast-serving/Chart.yaml b/infra/charts/feast/charts/feast-serving/Chart.yaml
index 8486275d94f..2e9cf89243d 100644
--- a/infra/charts/feast/charts/feast-serving/Chart.yaml
+++ b/infra/charts/feast/charts/feast-serving/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 description: A Helm chart for serving component of Feast
 name: feast-serving
-version: 0.3.2
+version: 0.4.4
diff --git a/infra/charts/feast/charts/feast-serving/requirements.yaml b/infra/charts/feast/charts/feast-serving/requirements.yaml
index fa4c1df4c10..2cee3f81494 100644
--- a/infra/charts/feast/charts/feast-serving/requirements.yaml
+++ b/infra/charts/feast/charts/feast-serving/requirements.yaml
@@ -3,3 +3,6 @@ dependencies:
   version: 9.5.0
   repository: "@stable"
   condition: redis.enabled
+- name: common
+  version: 0.0.5
+  repository: "@incubator"
diff --git a/infra/charts/feast/charts/feast-serving/templates/_helpers.tpl b/infra/charts/feast/charts/feast-serving/templates/_helpers.tpl
index 49abb6b8e50..ab670cc8cc7 100644
--- a/infra/charts/feast/charts/feast-serving/templates/_helpers.tpl
+++ b/infra/charts/feast/charts/feast-serving/templates/_helpers.tpl
@@ -43,3 +43,10 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
+
+{{/*
+Helpers
+*/}}
+{{- define "bq_store_and_no_job_options" -}}
+{{ and (eq (index .Values "store.yaml" "type") "BIGQUERY") (empty (index .Values "application.yaml" "feast" "jobs" "store-options")) }}
+{{- end -}}
diff --git a/infra/charts/feast/charts/feast-serving/templates/_ingress.yaml b/infra/charts/feast/charts/feast-serving/templates/_ingress.yaml
new file mode 100644
index 00000000000..5bed6df0470
--- /dev/null
+++ b/infra/charts/feast/charts/feast-serving/templates/_ingress.yaml
@@ -0,0 +1,68 @@
+{{- /*
+This takes an array of three values:
+- the top context
+- the feast component
+- the service protocol
+- the ingress context
+*/ -}}
+{{- define "feast.ingress" -}}
+{{- $top := (index . 0) -}}
+{{- $component := (index . 1) -}}
+{{- $protocol := (index . 2) -}}
+{{- $ingressValues := (index . 3) -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+{{ include "feast.ingress.metadata" . }}
+spec:
+  rules:
+  {{- range $host := $ingressValues.hosts }}
+  - host: {{ $host }}
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: {{ include (printf "feast-%s.fullname" $component) $top }}
+          servicePort: {{ index $top.Values "service" $protocol "port" }}
+  {{- end }}
+{{- if $ingressValues.https.enabled }}
+  tls:
+    {{- range $host := $ingressValues.hosts }}
+    - secretName: {{ index $ingressValues.https.secretNames $host | default (splitList "." $host | rest | join "-" | printf "%s-tls") }}
+      hosts:
+      - {{ $host }}
+    {{- end }}
+{{- end -}}
+{{- end -}}
+
+{{- define "feast.ingress.metadata" -}}
+{{- $commonMetadata := fromYaml (include "common.metadata" (first .)) }}
+{{- $overrides := fromYaml (include "feast.ingress.metadata-overrides" .) -}}
+{{- toYaml (merge $overrides $commonMetadata) -}}
+{{- end -}}
+
+{{- define "feast.ingress.metadata-overrides" -}}
+{{- $top := (index . 0) -}}
+{{- $component := (index . 1) -}}
+{{- $protocol := (index . 2) -}}
+{{- $ingressValues := (index . 3) -}}
+{{- $commonFullname := include "common.fullname" $top }}
+metadata:
+  name: {{ $commonFullname }}-{{ $component }}-{{ $protocol }}
+  annotations:
+    kubernetes.io/ingress.class: {{ $ingressValues.class | quote }}
+    {{- if (and (eq $ingressValues.class "nginx") $ingressValues.auth.enabled) }}
+    nginx.ingress.kubernetes.io/auth-url: {{ $ingressValues.auth.authUrl | quote }}
+    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-email, x-auth-request-user"
+    nginx.ingress.kubernetes.io/auth-signin: "https://{{ $ingressValues.auth.signinHost | default (splitList "." (index $ingressValues.hosts 0) | rest | join "." | printf "auth.%s")}}/oauth2/start?rd=/r/$host/$request_uri"
+    {{- end }}
+    {{- if (and (eq $ingressValues.class "nginx") $ingressValues.whitelist) }}
+    nginx.ingress.kubernetes.io/whitelist-source-range: {{ $ingressValues.whitelist | quote -}}
+    {{- end }}
+    {{- if (and (eq $ingressValues.class "nginx") (eq $protocol "grpc") ) }}
+    # TODO: Allow choice of GRPC/GRPCS
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+    {{- end }}
+    {{- if $ingressValues.annotations -}}
+    {{ include "common.annote" $ingressValues.annotations | indent 4 }}
+    {{- end }}
+{{- end -}}
diff --git a/infra/charts/feast/charts/feast-serving/templates/configmap.yaml b/infra/charts/feast/charts/feast-serving/templates/configmap.yaml
index 0ec80252c16..934216a9d5f 100644
--- a/infra/charts/feast/charts/feast-serving/templates/configmap.yaml
+++ b/infra/charts/feast/charts/feast-serving/templates/configmap.yaml
@@ -11,37 +11,43 @@ metadata:
     heritage: {{ .Release.Service }}
 data:
   application.yaml: |
-{{- $config := index .Values "application.yaml" }}
+{{- toYaml (index .Values "application.yaml") | nindent 4 }}
 
 {{- if .Values.core.enabled }}
-{{- $newConfig := dict "feast" (dict "core-host" (printf "%s-feast-core" .Release.Name)) }}
-{{- $config := mergeOverwrite $config $newConfig }}
+  application-bundled-core.yaml: |
+    feast:
+      core-host: {{ printf "%s-feast-core" .Release.Name }}
 {{- end }}
 
-{{- $store := index .Values "store.yaml" }}
-{{- if and (eq $store.type "BIGQUERY") (not (hasKey $config.feast.jobs "store-options")) }}
-{{- $jobStore := dict "host" (printf "%s-redis-headless" .Release.Name) "port" 6379 }}
-{{- $newConfig := dict "feast" (dict "jobs" (dict "store-options" $jobStore)) }}
-{{- $config := mergeOverwrite $config $newConfig }}
+{{- if eq (include "bq_store_and_no_job_options" .) "true" }}
+  application-bundled-redis.yaml: |
+    feast:
+      jobs:
+        store-options:
+          host: {{ printf "%s-redis-headless" .Release.Name }}
+          port: 6379
 {{- end }}
 
-{{- toYaml $config | nindent 4 }}
-  
   store.yaml: |
-{{- $config := index .Values "store.yaml"}}
+{{- $store := index .Values "store.yaml"}}
 
-{{- if and .Values.redis.enabled (eq $config.type "REDIS") }}
+{{- if and .Values.redis.enabled (eq $store.type "REDIS") }}
 
 {{- if eq .Values.redis.master.service.type "ClusterIP" }}
 {{- $newConfig := dict "redis_config" (dict "host" (printf "%s-redis-headless" .Release.Name) "port" .Values.redis.redisPort) }}
-{{- $config := mergeOverwrite $config $newConfig }}
+{{- $config := mergeOverwrite $store $newConfig }}
 {{- end }}
 
 {{- if and (eq .Values.redis.master.service.type "LoadBalancer") (not (empty .Values.redis.master.service.loadBalancerIP)) }}
 {{- $newConfig := dict "redis_config" (dict "host" .Values.redis.master.service.loadBalancerIP "port" .Values.redis.redisPort) }}
-{{- $config := mergeOverwrite $config $newConfig }}
+{{- $config := mergeOverwrite $store $newConfig }}
 {{- end }}
 
 {{- end }}
 
-{{- toYaml $config | nindent 4 }}
+{{- toYaml $store | nindent 4 }}
+
+{{- range $name, $content := .Values.springConfigProfiles }}
+  application-{{ $name }}.yaml: |
+{{- toYaml $content | nindent 4 }}
+{{- end }}
diff --git a/infra/charts/feast/charts/feast-serving/templates/deployment.yaml b/infra/charts/feast/charts/feast-serving/templates/deployment.yaml
index 5be636df96b..64dd3955d0c 100644
--- a/infra/charts/feast/charts/feast-serving/templates/deployment.yaml
+++ b/infra/charts/feast/charts/feast-serving/templates/deployment.yaml
@@ -47,9 +47,9 @@ spec:
 
       containers:
       - name: {{ .Chart.Name }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        image: '{{ .Values.image.repository }}:{{ required "No .image.tag found. This must be provided as input."  .Values.image.tag }}'
         imagePullPolicy: {{ .Values.image.pullPolicy }}
-        
+
         volumeMounts:
         - name: {{ template "feast-serving.fullname" . }}-config
           mountPath: "{{ .Values.springConfigMountPath }}"
@@ -60,24 +60,40 @@ spec:
         {{- end }}
 
         env:
+        - name: LOG_TYPE
+          value: {{ .Values.logType | quote }}
+        - name: LOG_LEVEL
+          value: {{ .Values.logLevel | quote }}
+
         {{- if .Values.gcpServiceAccount.useExistingSecret }}
         - name: GOOGLE_APPLICATION_CREDENTIALS
           value: {{ .Values.gcpServiceAccount.mountPath }}/{{ .Values.gcpServiceAccount.existingSecret.key }}
         {{- end }}
+        {{- if .Values.gcpProjectId }}
+        - name: GOOGLE_CLOUD_PROJECT
+          value: {{ .Values.gcpProjectId | quote }}
+        {{- end }}
 
         command:
         - java
         {{- range .Values.jvmOptions }}
-        - {{ . }}
+        - {{ . | quote }}
+        {{- end }}
+        - -jar
+        - {{ .Values.jarPath | quote }}
+        - "--spring.config.location=file:{{ .Values.springConfigMountPath }}/"
+        {{- $profilesArray := splitList "," .Values.springConfigProfilesActive -}}
+        {{- $profilesArray = append $profilesArray (.Values.core.enabled | ternary "bundled-core" "") -}}
+        {{- $profilesArray = append $profilesArray (eq (include "bq_store_and_no_job_options" .) "true" | ternary "bundled-redis" "") -}}
+        {{- $profilesArray = compact $profilesArray -}}
+        {{- if $profilesArray }}
+        - "--spring.profiles.active={{ join "," $profilesArray }}"
         {{- end }}
-        - -jar 
-        - /opt/feast/feast-serving.jar
-        - "--spring.config.location=file:{{ .Values.springConfigMountPath }}/application.yaml"
 
         ports:
         - name: http
           containerPort: {{ .Values.service.http.targetPort }}
-        - name: grpc 
+        - name: grpc
           containerPort: {{ .Values.service.grpc.targetPort }}
 
         {{- if .Values.livenessProbe.enabled }}
@@ -101,6 +117,6 @@ spec:
           timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
           failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
         {{- end }}
-        
+
         resources:
           {{- toYaml .Values.resources | nindent 10 }}
diff --git a/infra/charts/feast/charts/feast-serving/templates/ingress.yaml b/infra/charts/feast/charts/feast-serving/templates/ingress.yaml
index c6b4cb07a81..1bcd176147a 100644
--- a/infra/charts/feast/charts/feast-serving/templates/ingress.yaml
+++ b/infra/charts/feast/charts/feast-serving/templates/ingress.yaml
@@ -1,28 +1,7 @@
-{{- if .Values.ingress.enabled -}}
-{{- $fullName := include "feast-serving.fullname" . -}}
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  name: {{ $fullName }}
-  labels:
-    app: {{ template "feast-serving.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    component: serving
-    heritage: {{ .Release.Service }}
-    release: {{ .Release.Name }}
-  annotations:
-{{- with .Values.ingress.annotations }}
-{{ toYaml . | indent 4 }}
+{{- if .Values.ingress.http.enabled -}}
+{{ template "feast.ingress" (list . "serving" "http" .Values.ingress.http) }}
 {{- end }}
-spec:
-  rules:
-  {{- range .Values.ingress.hosts }}
-    - host: {{ .host | quote }}
-      http:
-        paths:
-          - path: /
-            backend:
-              serviceName: {{ $fullName }}
-              servicePort: {{ .port | quote }}
-  {{- end }}
+---
+{{ if .Values.ingress.grpc.enabled -}}
+{{ template "feast.ingress" (list . "serving" "grpc" .Values.ingress.grpc) }}
 {{- end }}
diff --git a/infra/charts/feast/charts/feast-serving/values.yaml b/infra/charts/feast/charts/feast-serving/values.yaml
index d489a48748d..8be09b639e9 100644
--- a/infra/charts/feast/charts/feast-serving/values.yaml
+++ b/infra/charts/feast/charts/feast-serving/values.yaml
@@ -3,23 +3,23 @@
 # for additional configuration
 redis:
   # enabled specifies whether Redis should be installed as part of Feast Serving.
-  # 
+  #
   # If enabled, "redis_config" in store.yaml will be overwritten by Helm
   # to the configuration in this Redis installation.
   enabled: false
   # usePassword specifies if password is required to access Redis. Note that
   # Feast 0.3 does not support Redis with password.
-  usePassword: false 
+  usePassword: false
   # cluster configuration for Redis.
   cluster:
     # enabled specifies if Redis should be installed in cluster mode.
     enabled: false
 
-# core configures Feast Core in the same parent feast chart that this Feast 
+# core configures Feast Core in the same parent feast chart that this Feast
 # Serving connects to.
 core:
   # enabled specifies that Feast Serving will use Feast Core installed
-  # in the same parent feast chart. If enabled, Helm will overwrite 
+  # in the same parent feast chart. If enabled, Helm will overwrite
   # "feast.core-host" in application.yaml with the correct value.
   enabled: true
 
@@ -29,7 +29,6 @@ replicaCount: 1
 # image configures the Docker image for Feast Serving
 image:
   repository: gcr.io/kf-feast/feast-serving
-  tag: 0.3.2
   pullPolicy: IfNotPresent
 
 # application.yaml is the main configuration for Feast Serving application.
@@ -38,7 +37,7 @@ image:
 # Refer to https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/serving/src/main/resources/application.yml
 # for a complete list and description of the configuration.
 #
-# Note that some properties defined in application.yaml may be overridden by 
+# Note that some properties defined in application.yaml may be overridden by
 # Helm under certain conditions. For example, if core is enabled, then
 # "feast.core-host" will be overridden. Also, if "type: BIGQUERY" is specified
 # in store.yaml, "feast.jobs.store-options" will be overridden as well with
@@ -56,6 +55,15 @@ application.yaml:
       config-path: /etc/feast/feast-serving/store.yaml
       redis-pool-max-size: 128
       redis-pool-max-idle: 64
+      cassandra-pool-core-local-connections: 1
+      cassandra-pool-max-local-connections: 1
+      cassandra-pool-core-remote-connections: 1
+      cassandra-pool-max-remote-connections: 1
+      cassandra-pool-max-requests-local-connection: 32768
+      cassandra-pool-max-requests-remote-connection: 2048
+      cassandra-pool-new-local-connection-threshold: 30000
+      cassandra-pool-new-remote-connection-threshold: 400
+      cassandra-pool-timeout-millis: 0
     jobs:
       staging-location: ""
       store-type: ""
@@ -67,19 +75,19 @@ application.yaml:
     port: 8080
 
 # store.yaml is the configuration for Feast Store.
-# 
+#
 # Refer to this link for description:
 # https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/protos/feast/core/Store.proto
 #
 # Use the correct store configuration depending on whether the installed
 # Feast Serving is "online" or "batch", by uncommenting the correct store.yaml.
 #
-# Note that if "redis.enabled: true" and "type: REDIS" in store.yaml, 
+# Note that if "redis.enabled: true" and "type: REDIS" in store.yaml,
 # Helm will override "redis_config" with configuration of Redis installed
 # in this chart.
-# 
+#
 # Note that if "type: BIGQUERY" in store.yaml, Helm assumes Feast Online serving
-# is also installed with Redis store. Helm will then override "feast.jobs.store-options" 
+# is also installed with Redis store. Helm will then override "feast.jobs.store-options"
 # in application.yaml with the installed Redis store configuration. This is
 # because in Feast 0.3, Redis job store is required.
 #
@@ -105,7 +113,14 @@ application.yaml:
 #     name: "*"
 #     version: "*"
 
-# springConfigMountPath is the directory path where application.yaml and 
+springConfigProfiles: {}
+#  db: |
+#    spring:
+#      datasource:
+#        driverClassName: org.postgresql.Driver
+#        url: jdbc:postgresql://${DB_HOST:127.0.0.1}:${DB_PORT:5432}/${DB_DATABASE:postgres}
+springConfigProfilesActive: ""
+# springConfigMountPath is the directory path where application.yaml and
 # store.yaml will be mounted in the container.
 springConfigMountPath: /etc/feast/feast-serving
 
@@ -116,7 +131,7 @@ gcpServiceAccount:
   useExistingSecret: false
   existingSecret:
     # name is the secret name of the existing secret for the service account.
-    name: feast-gcp-service-account 
+    name: feast-gcp-service-account
     # key is the secret key of the existing secret for the service account.
     # key is normally derived from the file name of the JSON key file.
     key: key.json
@@ -124,19 +139,29 @@ gcpServiceAccount:
   # the value of "existingSecret.key" is file name of the service account file.
   mountPath: /etc/gcloud/service-accounts
 
-# jvmOptions are options that will be passed to the Java Virtual Machine (JVM) 
+# Project ID picked up by the Cloud SDK (e.g. BigQuery run against this project)
+gcpProjectId: ""
+
+# Path to Jar file in the Docker image.
+# If using gcr.io/kf-feast/feast-serving this should not need to be changed.
+jarPath: /opt/feast/feast-serving.jar
+
+# jvmOptions are options that will be passed to the Java Virtual Machine (JVM)
 # running Feast Core.
-# 
+#
 # For example, it is good practice to set min and max heap size in JVM.
 # https://stackoverflow.com/questions/6902135/side-effect-for-increasing-maxpermsize-and-max-heap-size
 #
 # Refer to https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html
 # to see other JVM options that can be set.
 #
-# jvmOptions: 
-# - -Xms768m 
+jvmOptions: []
+# - -Xms768m
 # - -Xmx768m
 
+logType: JSON
+logLevel: warn
+
 livenessProbe:
   enabled: false
   initialDelaySeconds: 60
@@ -171,12 +196,29 @@ service:
     # nodePort:
 
 ingress:
-  enabled: false
-  annotations: {}
-    # kubernetes.io/ingress.class: nginx
-  hosts:
-  # - host: chart-example.local
-  #   port: http
+  grpc:
+    enabled: false
+    class: nginx
+    hosts: []
+    annotations: {}
+    https:
+      enabled: true
+      secretNames: {}
+    whitelist: ""
+    auth:
+      enabled: false
+  http:
+    enabled: false
+    class: nginx
+    hosts: []
+    annotations: {}
+    https:
+      enabled: true
+      secretNames: {}
+    whitelist: ""
+    auth:
+      enabled: false
+      authUrl: http://auth-server.auth-ns.svc.cluster.local/auth
 
 prometheus:
   enabled: true
@@ -186,6 +228,7 @@ resources: {}
   # choice for the user. This also increases chances charts run on environments with little
   # resources, such as Minikube. If you do want to specify resources, uncomment the following
   # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  #
   # limits:
   #   cpu: 100m
   #   memory: 128Mi
diff --git a/infra/charts/feast/requirements.lock b/infra/charts/feast/requirements.lock
index 8afd9521573..e441790dc76 100644
--- a/infra/charts/feast/requirements.lock
+++ b/infra/charts/feast/requirements.lock
@@ -1,12 +1,6 @@
 dependencies:
-- name: feast-core
-  repository: ""
-  version: 0.3.2
-- name: feast-serving
-  repository: ""
-  version: 0.3.2
-- name: feast-serving
-  repository: ""
-  version: 0.3.2
-digest: sha256:7ee4cd271cbd4ace44817dd12ba65f490a8e3529adf199604a2c2bdad9c2fac3
-generated: "2019-11-27T13:35:41.334054+08:00"
+- name: common
+  repository: https://kubernetes-charts-incubator.storage.googleapis.com
+  version: 0.0.5
+digest: sha256:935bfb09e9ed90ff800826a7df21adaabe3225511c3ad78df44e1a5a60e93f14
+generated: 2019-12-10T14:47:49.57569Z
diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml
index ed280d64b6e..1fa1826965a 100644
--- a/infra/charts/feast/requirements.yaml
+++ b/infra/charts/feast/requirements.yaml
@@ -1,12 +1,12 @@
 dependencies:
 - name: feast-core
-  version: 0.3.2
+  version: 0.4.4
   condition: feast-core.enabled
 - name: feast-serving
   alias: feast-serving-batch
-  version: 0.3.2
+  version: 0.4.4
   condition: feast-serving-batch.enabled
 - name: feast-serving
   alias: feast-serving-online
-  version: 0.3.2
-  condition: feast-serving-online.enabled
+  version: 0.4.4
+  condition: feast-serving-online.enabled
\ No newline at end of file
diff --git a/infra/charts/feast/values-demo.yaml b/infra/charts/feast/values-demo.yaml
index fad4bc0afb0..2cb5ccbe741 100644
--- a/infra/charts/feast/values-demo.yaml
+++ b/infra/charts/feast/values-demo.yaml
@@ -1,7 +1,7 @@
 # The following are values for installing Feast for demonstration purpose:
 # - Persistence is disabled since for demo purpose data is not expected
 #   to be durable
-# - Only online serving (no batch serving) is installed to remove dependency 
+# - Only online serving (no batch serving) is installed to remove dependency
 #   on Google Cloud services. Batch serving requires BigQuery dependency.
 # - Replace all occurrences of "feast.example.com" with the domain name or
 #   external IP pointing to your cluster
@@ -68,4 +68,17 @@ feast-serving-online:
         version: "*"
 
 feast-serving-batch:
-  enabled: false
+#  enabled: false
+  enabled: true
+  store.yaml:
+    name: bigquery
+    type: BIGQUERY
+    bigquery_config:
+      project_id: PROJECT_ID
+      dataset_id: DATASET_ID
+    subscriptions:
+    - project: "*"
+      name: "*"
+      version: "*"
+  redis:
+    enabled: false
\ No newline at end of file
diff --git a/infra/charts/feast/values.yaml b/infra/charts/feast/values.yaml
index ebc8c802a16..fde03f9ad71 100644
--- a/infra/charts/feast/values.yaml
+++ b/infra/charts/feast/values.yaml
@@ -2,29 +2,27 @@
 # - Feast Core
 # - Feast Serving Online
 # - Feast Serving Batch
-# 
+# - Prometheus StatsD Exporter
+#
 # The configuration for different components can be referenced from:
 # - charts/feast-core/values.yaml
 # - charts/feast-serving/values.yaml
+# - charts/prometheus-statsd-exporter/values.yaml
 #
 # Note that "feast-serving-online" and "feast-serving-batch" are
 # aliases to "feast-serving" chart since in typical scenario two instances
-# of Feast Serving: online and batch will be deployed. Both described 
+# of Feast Serving: online and batch will be deployed. Both described
 # using the same chart "feast-serving".
 #
-# The following are default values for typical Feast deployment, but not
-# for production setting. Refer to "values-production.yaml" for recommended
-# values in production environment.
-# 
-# Note that the import job by default uses DirectRunner 
+# Note that the import job by default uses DirectRunner
 # https://beam.apache.org/documentation/runners/direct/
 # in this configuration since it allows Feast to run in more environments
 # (unlike DataflowRunner which requires Google Cloud services).
-# 
-# A secret containing Google Cloud service account JSON key is required 
-# in this configuration. 
+#
+# A secret containing Google Cloud service account JSON key is required
+# in this configuration.
 # https://cloud.google.com/iam/docs/creating-managing-service-accounts
-# 
+#
 # The Google Cloud service account must have the following roles:
 # - bigquery.dataEditor
 # - bigquery.jobUser
@@ -32,33 +30,40 @@
 # Assuming a service account JSON key file has been downloaded to
 # (please name the file key.json):
 # /home/user/key.json
-# 
+#
 # Run the following command to create the secret in your Kubernetes cluster:
 #
 # kubectl create secret generic feast-gcp-service-account \
 #   --from-file=/home/user/key.json
 #
+# Replace every instance of EXTERNAL_IP with the external IP of your GKE cluster
 
 # ============================================================
 # Feast Core
 # ============================================================
 
 feast-core:
-  # enabled specifies whether to install Feast Core component.
+  # If enabled specifies whether to install Feast Core component.
   #
   # Normally, this is set to "false" when Feast users need access to low latency
   # Feast Serving, by deploying multiple instances of Feast Serving closest
   # to the client. These instances of Feast Serving however can still use
   # the same shared Feast Core.
   enabled: true
-  # jvmOptions are options that will be passed to the Java Virtual Machine (JVM) 
+
+  # Specify which image tag to use. Keep this consistent for all components
+  image:
+    tag: "0.4.4"
+
+  # jvmOptions are options that will be passed to the Java Virtual Machine (JVM)
   # running Feast Core.
   #
   # For example, it is good practice to set min and max heap size in JVM.
   # https://stackoverflow.com/questions/6902135/side-effect-for-increasing-maxpermsize-and-max-heap-size
-  jvmOptions: 
+  jvmOptions:
   - -Xms1024m
   - -Xmx1024m
+
   # resources that should be allocated to Feast Core.
   resources:
     requests:
@@ -66,20 +71,46 @@ feast-core:
       memory: 1024Mi
     limits:
       memory: 2048Mi
+
   # gcpServiceAccount is the Google service account that Feast Core will use.
   gcpServiceAccount:
-    # useExistingSecret specifies Feast to use an existing secret containing 
+    # useExistingSecret specifies Feast to use an existing secret containing
     # Google Cloud service account JSON key file.
-    # 
+    #
     # This is the only supported option for now to use a service account JSON.
     # Feast admin is expected to create this secret before deploying Feast.
     useExistingSecret: true
     existingSecret:
       # name is the secret name of the existing secret for the service account.
-      name: feast-gcp-service-account 
+      name: feast-gcp-service-account
       # key is the secret key of the existing secret for the service account.
       # key is normally derived from the file name of the JSON key file.
       key: key.json
+  # Setting service.type to NodePort exposes feast-core service at a static port
+  service:
+    type: NodePort
+    grpc:
+      # this is the port that is exposed outside of the cluster
+      nodePort: 32090
+  # Make kafka externally accessible using NodePort
+  # Please set EXTERNAL_IP to your cluster's external IP
+  kafka:
+    external:
+      enabled: true
+      type: NodePort
+      domain: EXTERNAL_IP
+    configurationOverrides:
+      "advertised.listeners": |-
+        EXTERNAL://EXTERNAL_IP:$((31090 + ${KAFKA_BROKER_ID}))
+      "listener.security.protocol.map": |-
+        PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
+  application.yaml:
+    feast:
+      stream:
+        options:
+          # Point to one of your Kafka brokers
+          # Please set EXTERNAL_IP to your cluster's external IP
+          bootstrapServers: EXTERNAL_IP:31090
 
 # ============================================================
 # Feast Serving Online
@@ -88,14 +119,22 @@ feast-core:
 feast-serving-online:
   # enabled specifies whether to install Feast Serving Online component.
   enabled: true
+  # Specify what image tag to use. Keep this consistent for all components
+  image:
+    tag: "0.4.4"
   # redis.enabled specifies whether Redis should be installed as part of Feast Serving.
-  # 
+  #
   # If enabled is set to "false", Feast admin has to ensure there is an
   # existing Redis running outside Feast, that Feast Serving can connect to.
+  # master.service.type set to NodePort exposes Redis to outside of the cluster
   redis:
     enabled: true
+    master:
+      service:
+        nodePort: 32101
+        type: NodePort
   # jvmOptions are options that will be passed to the Feast Serving JVM.
-  jvmOptions: 
+  jvmOptions:
   - -Xms1024m
   - -Xmx1024m
   # resources that should be allocated to Feast Serving.
@@ -105,23 +144,28 @@ feast-serving-online:
       memory: 1024Mi
     limits:
       memory: 2048Mi
+  # Make service accessible to outside of cluster using NodePort
+  service:
+    type: NodePort
+    grpc:
+      nodePort: 32091
   # store.yaml is the configuration for Feast Store.
-  # 
+  #
   # Refer to this link for more description:
   # https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/protos/feast/core/Store.proto
   store.yaml:
     name: redis
     type: REDIS
     redis_config:
-      # If redis.enabled is set to false, Feast admin should uncomment and 
-      # set the host value to an "existing" Redis instance Feast will use as 
-      # online Store. 
-      # 
-      # Else, if redis.enabled is set to true, no additional configuration is
-      # required.
+      # If redis.enabled is set to false, Feast admin should uncomment and
+      # set the host value to an "existing" Redis instance Feast will use as
+      # online Store. Also use the correct port for that existing instance.
       #
+      # Else, if redis.enabled is set to true, replace EXTERNAL_IP with your
+      # cluster's external IP.
       # host: redis-host
-      port: 6379
+      host: EXTERNAL_IP
+      port: 32101
     subscriptions:
     - name: "*"
       project: "*"
@@ -134,14 +178,17 @@ feast-serving-online:
 feast-serving-batch:
   # enabled specifies whether to install Feast Serving Batch component.
   enabled: true
+  # Specify what image tag to use. Keep this consistent for all components
+  image:
+    tag: "0.4.4"
   # redis.enabled specifies whether Redis should be installed as part of Feast Serving.
-  # 
+  #
   # This is usually set to "false" for Feast Serving Batch because the default
   # store is BigQuery.
   redis:
     enabled: false
   # jvmOptions are options that will be passed to the Feast Serving JVM.
-  jvmOptions: 
+  jvmOptions:
   - -Xms1024m
   - -Xmx1024m
   # resources that should be allocated to Feast Serving.
@@ -151,17 +198,22 @@ feast-serving-batch:
       memory: 1024Mi
     limits:
       memory: 2048Mi
+  # Make service accessible to outside of cluster using NodePort
+  service:
+    type: NodePort
+    grpc:
+      nodePort: 32092
   # gcpServiceAccount is the service account that Feast Serving will use.
   gcpServiceAccount:
-    # useExistingSecret specifies Feast to use an existing secret containing 
+    # useExistingSecret specifies Feast to use an existing secret containing
     # Google Cloud service account JSON key file.
-    # 
+    #
     # This is the only supported option for now to use a service account JSON.
     # Feast admin is expected to create this secret before deploying Feast.
     useExistingSecret: true
     existingSecret:
       # name is the secret name of the existing secret for the service account.
-      name: feast-gcp-service-account 
+      name: feast-gcp-service-account
       # key is the secret key of the existing secret for the service account.
       # key is normally derived from the file name of the JSON key file.
       key: key.json
@@ -172,28 +224,33 @@ feast-serving-batch:
   # for a complete list and description of the configuration.
   application.yaml:
     feast:
-      jobs: 
-        # staging-location specifies the URI to store intermediate files for 
+      jobs:
+        # staging-location specifies the URI to store intermediate files for
         # batch serving (required if using BigQuery as Store).
-        # 
-        # Please set the value to an "existing" Google Cloud Storage URI that 
+        #
+        # Please set the value to an "existing" Google Cloud Storage URI that
         # Feast serving has write access to.
-        staging-location: gs://bucket/path
-        # Type of store to store job metadata. 
+        staging-location: gs://YOUR_BUCKET_NAME/serving/batch
+        # Type of store to store job metadata.
         #
-        # This default configuration assumes that Feast Serving Online is 
+        # This default configuration assumes that Feast Serving Online is
         # enabled as well. So Feast Serving Batch will share the same
         # Redis instance to store job statuses.
         store-type: REDIS
+        # Default to use the internal hostname of the redis instance deployed by Online service,
+        # otherwise use externally exposed by setting EXTERNAL_IP to your cluster's external IP
+        # store-options:
+          # host: EXTERNAL_IP
+          # port: 32101
   # store.yaml is the configuration for Feast Store.
-  # 
+  #
   # Refer to this link for more description:
   # https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/protos/feast/core/Store.proto
   store.yaml:
     name: bigquery
     type: BIGQUERY
     bigquery_config:
-      # project_id specifies the Google Cloud Project. Please set this to the 
+      # project_id specifies the Google Cloud Project. Please set this to the
       # project id you are using BigQuery in.
       project_id: PROJECT_ID
       # dataset_id specifies an "existing" BigQuery dataset Feast Serving Batch
diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample
index e14bde27728..c8652e8fe0c 100644
--- a/infra/docker-compose/.env.sample
+++ b/infra/docker-compose/.env.sample
@@ -1,19 +1,21 @@
+# General
 COMPOSE_PROJECT_NAME=feast
-
 FEAST_VERSION=latest
 
+# Feast Core
 FEAST_CORE_IMAGE=gcr.io/kf-feast/feast-core
-FEAST_CORE_CONFIG=direct-runner
-FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY=placeholder
+FEAST_CORE_CONFIG=direct-runner.yml
+FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
 
+# Feast Serving
 FEAST_SERVING_IMAGE=gcr.io/kf-feast/feast-serving
-FEAST_ONLINE_SERVING_CONFIG=online-serving
-FEAST_ONLINE_STORE_CONFIG=redis-store
-FEAST_BATCH_SERVING_CONFIG=batch-serving
-FEAST_BATCH_STORE_CONFIG=bq-store
-FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY=placeholder
-FEAST_JOB_STAGING_LOCATION=gs://your-gcp-project/bucket
+FEAST_ONLINE_SERVING_CONFIG=online-serving.yml
+FEAST_ONLINE_STORE_CONFIG=redis-store.yml
+FEAST_BATCH_SERVING_CONFIG=batch-serving.yml
+FEAST_BATCH_STORE_CONFIG=bq-store.yml
+FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
+FEAST_JOB_STAGING_LOCATION=gs://your-gcs-bucket/staging
 
-FEAST_JUPYTER_IMAGE=gcr.io/kf-feast/feast-jupyter
-FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY=placeholder
+# Jupyter
+FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY=placeholder.json
 
diff --git a/infra/docker-compose/docker-compose.batch.yml b/infra/docker-compose/docker-compose.batch.yml
deleted file mode 100644
index c00ac9475bd..00000000000
--- a/infra/docker-compose/docker-compose.batch.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-version: "3.7"
-
-services:
-  batch-serving:
-    image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
-    volumes:
-      - ./serving/${FEAST_BATCH_SERVING_CONFIG}.yml:/etc/feast/application.yml
-      - ./serving/${FEAST_BATCH_STORE_CONFIG}.yml:/etc/feast/store.yml
-      - ./gcp-service-accounts/${FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY}.json:/etc/gcloud/service-accounts/key.json
-    depends_on:
-      - core
-      - redis
-    ports:
-      - 6567:6567
-    restart: on-failure
-    environment:
-      GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
-      FEAST_JOB_STAGING_LOCATION: ${FEAST_JOB_STAGING_LOCATION}
-    command:
-      - "java"
-      - "-Xms1024m"
-      - "-Xmx1024m"
-      - "-jar"
-      - "/opt/feast/feast-serving.jar"
-      - "--spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml"
\ No newline at end of file
diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml
index a224500ca0a..a796e5fa44e 100644
--- a/infra/docker-compose/docker-compose.yml
+++ b/infra/docker-compose/docker-compose.yml
@@ -4,8 +4,8 @@ services:
   core:
     image: ${FEAST_CORE_IMAGE}:${FEAST_VERSION}
     volumes:
-      - ./core/${FEAST_CORE_CONFIG}.yml:/etc/feast/application.yml
-      - ./gcp-service-accounts/${FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY}.json:/etc/gcloud/service-accounts/key.json
+      - ./core/${FEAST_CORE_CONFIG}:/etc/feast/application.yml
+      - ./gcp-service-accounts/${FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY}:/etc/gcloud/service-accounts/key.json
     environment:
       DB_HOST: db
       GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
@@ -24,8 +24,8 @@ services:
   online-serving:
     image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
     volumes:
-      - ./serving/${FEAST_ONLINE_SERVING_CONFIG}.yml:/etc/feast/application.yml
-      - ./serving/${FEAST_ONLINE_STORE_CONFIG}.yml:/etc/feast/store.yml
+      - ./serving/${FEAST_ONLINE_SERVING_CONFIG}:/etc/feast/application.yml
+      - ./serving/${FEAST_ONLINE_STORE_CONFIG}:/etc/feast/store.yml
     depends_on:
       - core
       - redis
@@ -38,12 +38,34 @@ services:
       - /opt/feast/feast-serving.jar
       - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml
 
+  batch-serving:
+    image: ${FEAST_SERVING_IMAGE}:${FEAST_VERSION}
+    volumes:
+      - ./serving/${FEAST_BATCH_SERVING_CONFIG}:/etc/feast/application.yml
+      - ./serving/${FEAST_BATCH_STORE_CONFIG}:/etc/feast/store.yml
+      - ./gcp-service-accounts/${FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY}:/etc/gcloud/service-accounts/key.json
+    depends_on:
+      - core
+      - redis
+    ports:
+      - 6567:6567
+    restart: on-failure
+    environment:
+      GOOGLE_APPLICATION_CREDENTIALS: /etc/gcloud/service-accounts/key.json
+      FEAST_JOB_STAGING_LOCATION: ${FEAST_JOB_STAGING_LOCATION}
+    command:
+      - "java"
+      - "-Xms1024m"
+      - "-Xmx1024m"
+      - "-jar"
+      - "/opt/feast/feast-serving.jar"
+      - "--spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml"
+
   jupyter:
-    image: ${FEAST_JUPYTER_IMAGE}:${FEAST_VERSION}
+    image: jupyter/datascience-notebook:latest
     volumes:
-      - ./jupyter/notebooks:/home/jovyan/feast-notebooks
-      - ./jupyter/features:/home/jovyan/features
-      - ./gcp-service-accounts/${FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY}.json:/etc/gcloud/service-accounts/key.json
+      - ../../:/home/jovyan/feast
+      - ./gcp-service-accounts/${FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY}:/etc/gcloud/service-accounts/key.json
     depends_on:
       - core
       - online-serving
@@ -59,6 +81,8 @@ services:
 
   redis:
     image: redis:5-alpine
+    ports:
+      - "6379:6379"
 
   kafka:
     image: confluentinc/cp-kafka:5.2.1
@@ -70,7 +94,8 @@ services:
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
     ports:
-      - 9094:9092
+      - "9092:9092"
+      - "9094:9094"
 
     depends_on:
       - zookeeper
@@ -81,4 +106,8 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
 
   db:
-    image: postgres:12-alpine
\ No newline at end of file
+    image: postgres:12-alpine
+    environment:
+      POSTGRES_PASSWORD: password
+    ports:
+      - "5432:5342"
\ No newline at end of file
diff --git a/infra/docker-compose/jupyter/features/cust_trans_fs.yaml b/infra/docker-compose/jupyter/features/cust_trans_fs.yaml
deleted file mode 100644
index eb21ce9b35b..00000000000
--- a/infra/docker-compose/jupyter/features/cust_trans_fs.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-name: customer_transactions
-kind: feature_set
-entities:
-- name: customer_id
-  valueType: INT64
-features:
-- name: daily_transactions
-  valueType: FLOAT
-- name: total_transactions
-  valueType: FLOAT
-maxAge: 3600s
\ No newline at end of file
diff --git a/infra/docker-compose/jupyter/features/cust_trans_fs_updated.yaml b/infra/docker-compose/jupyter/features/cust_trans_fs_updated.yaml
deleted file mode 100644
index 8293d04b881..00000000000
--- a/infra/docker-compose/jupyter/features/cust_trans_fs_updated.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-name: customer_transactions
-kind: feature_set
-entities:
-- name: customer_id
-  valueType: INT64
-features:
-- name: daily_transactions
-  valueType: FLOAT
-- name: total_transactions
-  valueType: FLOAT
-- name: discounts
-  valueType: FLOAT
-maxAge: 3600s
\ No newline at end of file
diff --git a/infra/docker-compose/jupyter/notebooks/feast-batch-serving.ipynb b/infra/docker-compose/jupyter/notebooks/feast-batch-serving.ipynb
deleted file mode 100644
index c288093f07b..00000000000
--- a/infra/docker-compose/jupyter/notebooks/feast-batch-serving.ipynb
+++ /dev/null
@@ -1,504 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Feast Batch Serving\n",
-    "This is an extension to `feast-quickstart` notebook to demonstrate the batch serving capability of Feast.\n",
-    "\n",
-    "## Prerequisite\n",
-    "- A running Feast Serving service with store configuration that supports batch retrieval. (eg. BigQuery store)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Data Preparation\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import feast\n",
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "from datetime import datetime, timedelta\n",
-    "from feast.serving.ServingService_pb2 import GetOnlineFeaturesRequest\n",
-    "from feast.types.Value_pb2 import Value as Value\n",
-    "from feast.client import Client\n",
-    "from feast.feature_set import FeatureSet"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "client = feast.Client(core_url=\"core:6565\", serving_url=\"batch-serving:6567\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "cust_trans_fs = FeatureSet.from_yaml(\"../features/cust_trans_fs.yaml\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Feature set updated/created: \"customer_transactions:1\".\n"
-     ]
-    }
-   ],
-   "source": [
-    "client.apply(cust_trans_fs)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
datetimecustomer_iddaily_transactionstotal_transactions
02019-12-06 02:17:46.899904100002.797627175.978266
12019-12-06 02:17:46.899915100014.931632153.871975
22019-12-06 02:17:46.899922100020.206628108.558844
32019-12-06 02:17:46.899929100032.354937119.549455
42019-12-06 02:17:46.899937100047.171423115.345183
\n", - "
" - ], - "text/plain": [ - " datetime customer_id daily_transactions \\\n", - "0 2019-12-06 02:17:46.899904 10000 2.797627 \n", - "1 2019-12-06 02:17:46.899915 10001 4.931632 \n", - "2 2019-12-06 02:17:46.899922 10002 0.206628 \n", - "3 2019-12-06 02:17:46.899929 10003 2.354937 \n", - "4 2019-12-06 02:17:46.899937 10004 7.171423 \n", - "\n", - " total_transactions \n", - "0 175.978266 \n", - "1 153.871975 \n", - "2 108.558844 \n", - "3 119.549455 \n", - "4 115.345183 " - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "offset = 10000\n", - "nr_of_customers = 5\n", - "customer_df = pd.DataFrame(\n", - " {\n", - " \"datetime\": [datetime.utcnow() for _ in range(nr_of_customers)],\n", - " \"customer_id\": [offset + inc for inc in range(nr_of_customers)],\n", - " \"daily_transactions\": [np.random.uniform(0, 10) for _ in range(nr_of_customers)],\n", - " \"total_transactions\": [np.random.uniform(100, 200) for _ in range(nr_of_customers)],\n", - " }\n", - ")\n", - "customer_df" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 5/5 [00:00<00:00, 7.24rows/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Ingested 5 rows into customer_transactions:1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "client.ingest(cust_trans_fs, dataframe=customer_df)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Batch Retrieval\n", - "Batch retrieval takes a dataframe containing the entities column and event timestamp as an input. The result would be the outer join of the input and the features. The input dataframe needs to have a column named `datetime` as event timestamp. No results will be returned if the difference between the feature ingestion timestamp and the `event_timestamp` is greater than the `maxAge` parameter specified in the feature set." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
customer_transactions_v1_feature_timestampcustomer_idevent_timestampcustomer_transactions_v1_daily_transactionscustomer_transactions_v1_total_transactions
02019-12-06 02:17:46+00:00100012019-12-06 02:17:55.612449+00:004.931632153.871980
12019-12-06 02:17:46+00:00100042019-12-06 02:17:55.612449+00:007.171423115.345184
22019-12-06 02:17:46+00:00100002019-12-06 02:17:55.612449+00:002.797627175.978270
32019-12-06 02:17:46+00:00100022019-12-06 02:17:55.612449+00:000.206628108.558846
42019-12-06 02:17:46+00:00100032019-12-06 02:17:55.612449+00:002.354937119.549450
\n", - "
" - ], - "text/plain": [ - " customer_transactions_v1_feature_timestamp customer_id \\\n", - "0 2019-12-06 02:17:46+00:00 10001 \n", - "1 2019-12-06 02:17:46+00:00 10004 \n", - "2 2019-12-06 02:17:46+00:00 10000 \n", - "3 2019-12-06 02:17:46+00:00 10002 \n", - "4 2019-12-06 02:17:46+00:00 10003 \n", - "\n", - " event_timestamp \\\n", - "0 2019-12-06 02:17:55.612449+00:00 \n", - "1 2019-12-06 02:17:55.612449+00:00 \n", - "2 2019-12-06 02:17:55.612449+00:00 \n", - "3 2019-12-06 02:17:55.612449+00:00 \n", - "4 2019-12-06 02:17:55.612449+00:00 \n", - "\n", - " customer_transactions_v1_daily_transactions \\\n", - "0 4.931632 \n", - "1 7.171423 \n", - "2 2.797627 \n", - "3 0.206628 \n", - "4 2.354937 \n", - "\n", - " customer_transactions_v1_total_transactions \n", - "0 153.871980 \n", - "1 115.345184 \n", - "2 175.978270 \n", - "3 108.558846 \n", - "4 119.549450 " - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "entity_df = customer_df[[\"customer_id\"]].assign(datetime=datetime.utcnow())\n", - "feature_ids=[\n", - " \"customer_transactions:1:daily_transactions\",\n", - " \"customer_transactions:1:total_transactions\",\n", - "]\n", - "batch_job = client.get_batch_features(feature_ids, entity_df)\n", - "batch_job.to_dataframe()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
customer_transactions_v1_feature_timestampcustomer_idevent_timestampcustomer_transactions_v1_daily_transactionscustomer_transactions_v1_total_transactions
0None100002020-01-05 02:18:43.900732+00:00NoneNone
1None100012020-01-05 02:18:43.900732+00:00NoneNone
2None100022020-01-05 02:18:43.900732+00:00NoneNone
3None100032020-01-05 02:18:43.900732+00:00NoneNone
4None100042020-01-05 02:18:43.900732+00:00NoneNone
\n", - "
" - ], - "text/plain": [ - " customer_transactions_v1_feature_timestamp customer_id \\\n", - "0 None 10000 \n", - "1 None 10001 \n", - "2 None 10002 \n", - "3 None 10003 \n", - "4 None 10004 \n", - "\n", - " event_timestamp \\\n", - "0 2020-01-05 02:18:43.900732+00:00 \n", - "1 2020-01-05 02:18:43.900732+00:00 \n", - "2 2020-01-05 02:18:43.900732+00:00 \n", - "3 2020-01-05 02:18:43.900732+00:00 \n", - "4 2020-01-05 02:18:43.900732+00:00 \n", - "\n", - " customer_transactions_v1_daily_transactions \\\n", - "0 None \n", - "1 None \n", - "2 None \n", - "3 None \n", - "4 None \n", - "\n", - " customer_transactions_v1_total_transactions \n", - "0 None \n", - "1 None \n", - "2 None \n", - "3 None \n", - "4 None " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "stale_entity_df = customer_df[[\"customer_id\"]].assign(datetime=datetime.utcnow() + timedelta(days=30))\n", - "feature_ids=[\n", - " \"customer_transactions:1:daily_transactions\",\n", - " \"customer_transactions:1:total_transactions\",\n", - "]\n", - "batch_job = client.get_batch_features(feature_ids, stale_entity_df)\n", - "batch_job.to_dataframe()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/infra/docker-compose/jupyter/notebooks/feast-quickstart.ipynb b/infra/docker-compose/jupyter/notebooks/feast-quickstart.ipynb deleted file mode 100644 index b89e59b1e49..00000000000 --- a/infra/docker-compose/jupyter/notebooks/feast-quickstart.ipynb +++ /dev/null @@ -1,569 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Feast Quick Start\n", - "This is a quick example to demonstrate:\n", - "- Register a feature set on Feast\n", - "- Ingest features into Feast\n", - "- Retrieve the ingested features from Feast\n", - "- Update a feature" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import feast\n", - "import numpy as np\n", - "import pandas as pd\n", - "from datetime import datetime\n", - "from feast.serving.ServingService_pb2 import GetOnlineFeaturesRequest\n", - "from feast.types.Value_pb2 import Value as Value\n", - "from feast.client import Client\n", - "from feast.feature_set import FeatureSet" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, instantiate the client.\n", - "Feast endpoints can be set via the following environmental variables: `FEAST_CORE_URL`, `FEAST_SERVING_URL`.\n", - "Alternatively, they can also be passed in explicitly as follows:\n", - " \n", - "`client = feast.Client(core_url=core:6565, serving_url=online-serving:6566)`" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "client = feast.Client()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Register a feature set\n", - "\n", - "Let's create and register our first feature set. Below is an example of a basic customer transactions feature set that has been exported to YAML:\n", - "```\n", - "name: customer_transactions\n", - "kind: feature_set\n", - "entities:\n", - "- name: customer_id\n", - " valueType: INT64\n", - "features:\n", - "- name: daily_transactions\n", - " valueType: FLOAT\n", - "- name: total_transactions\n", - " valueType: FLOAT\n", - "maxAge: 3600s \n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "cust_trans_fs = FeatureSet.from_yaml(\"../features/cust_trans_fs.yaml\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Feature set updated/created: \"customer_transactions:1\".\n" - ] - } - ], - "source": [ - "client.apply(cust_trans_fs)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Ingest features into Feast\n", - "The dataframe below contains the features and entities of the above feature set." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
datetimecustomer_iddaily_transactionstotal_transactions
02019-11-26 12:03:47.320634100005.178112110.670651
12019-11-26 12:03:47.320644100010.268114195.393913
22019-11-26 12:03:47.320651100021.486614136.929052
32019-11-26 12:03:47.320658100039.676433166.022999
42019-11-26 12:03:47.320665100045.928573165.687951
\n", - "
" - ], - "text/plain": [ - " datetime customer_id daily_transactions \\\n", - "0 2019-11-26 12:03:47.320634 10000 5.178112 \n", - "1 2019-11-26 12:03:47.320644 10001 0.268114 \n", - "2 2019-11-26 12:03:47.320651 10002 1.486614 \n", - "3 2019-11-26 12:03:47.320658 10003 9.676433 \n", - "4 2019-11-26 12:03:47.320665 10004 5.928573 \n", - "\n", - " total_transactions \n", - "0 110.670651 \n", - "1 195.393913 \n", - "2 136.929052 \n", - "3 166.022999 \n", - "4 165.687951 " - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "offset = 10000\n", - "nr_of_customers = 5\n", - "customer_df = pd.DataFrame(\n", - " {\n", - " \"datetime\": [datetime.utcnow() for _ in range(nr_of_customers)],\n", - " \"customer_id\": [offset + inc for inc in range(nr_of_customers)],\n", - " \"daily_transactions\": [np.random.uniform(0, 10) for _ in range(nr_of_customers)],\n", - " \"total_transactions\": [np.random.uniform(100, 200) for _ in range(nr_of_customers)],\n", - " }\n", - ")\n", - "customer_df" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 0%| | 0/5 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
datetimecustomer_iddaily_transactionstotal_transactionsdiscounts
02019-11-26 12:03:47.320634100005.178112110.6706518.389938
12019-11-26 12:03:47.320644100010.268114195.3939130.430047
22019-11-26 12:03:47.320651100021.486614136.9290527.408917
32019-11-26 12:03:47.320658100039.676433166.0229991.192721
42019-11-26 12:03:47.320665100045.928573165.6879512.051037
\n", - "" - ], - "text/plain": [ - " datetime customer_id daily_transactions \\\n", - "0 2019-11-26 12:03:47.320634 10000 5.178112 \n", - "1 2019-11-26 12:03:47.320644 10001 0.268114 \n", - "2 2019-11-26 12:03:47.320651 10002 1.486614 \n", - "3 2019-11-26 12:03:47.320658 10003 9.676433 \n", - "4 2019-11-26 12:03:47.320665 10004 5.928573 \n", - "\n", - " total_transactions discounts \n", - "0 110.670651 8.389938 \n", - "1 195.393913 0.430047 \n", - "2 136.929052 7.408917 \n", - "3 166.022999 1.192721 \n", - "4 165.687951 2.051037 " - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "discounts = [np.random.uniform(0, 10) for _ in range(nr_of_customers)]\n", - "customer_df_updated = customer_df.assign(discounts=discounts)\n", - "customer_df_updated" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 0%| | 0/5 [00:00 - - org.xolstice.maven.plugins - protobuf-maven-plugin - org.apache.maven.plugins maven-shade-plugin @@ -91,21 +87,9 @@ - org.glassfish - javax.el - 3.0.0 - - - - javax.validation - validation-api - 2.0.1.Final - - - - org.hibernate.validator - hibernate-validator - 6.0.13.Final + dev.feast + datatypes-java + ${project.version} @@ -120,15 +104,6 @@ provided - - io.grpc - grpc-stub - - - - com.google.cloud - google-cloud-storage - com.google.cloud google-cloud-bigquery @@ -148,27 +123,6 @@ mockito-core - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - com.fasterxml.jackson.module - jackson-module-jsonSchema - - com.google.protobuf protobuf-java @@ -214,22 +168,19 @@ - redis.clients - jedis + io.lettuce + lettuce-core - org.slf4j - slf4j-api + org.apache.beam + beam-sdks-java-io-cassandra + ${org.apache.beam.version} - - - ch.qos.logback - logback-classic - 1.2.3 - runtime + org.slf4j + slf4j-api @@ -245,6 +196,13 @@ test + + org.cassandraunit + cassandra-unit-shaded + 3.11.2.0 + test + + com.google.guava guava @@ -255,5 +213,12 @@ 2.8.1 + + + org.apache.commons + commons-math3 + 3.6.1 + + diff --git a/ingestion/src/main/java/feast/ingestion/ImportJob.java b/ingestion/src/main/java/feast/ingestion/ImportJob.java index 41af5f9bb40..c4973ce3cae 100644 --- a/ingestion/src/main/java/feast/ingestion/ImportJob.java +++ b/ingestion/src/main/java/feast/ingestion/ImportJob.java @@ -22,7 +22,9 @@ import feast.core.FeatureSetProto.FeatureSet; import feast.core.SourceProto.Source; import feast.core.StoreProto.Store; +import feast.ingestion.options.BZip2Decompressor; import feast.ingestion.options.ImportOptions; +import feast.ingestion.options.StringListStreamConverter; import feast.ingestion.transform.ReadFromSource; import feast.ingestion.transform.ValidateFeatureRows; import feast.ingestion.transform.WriteFailedElementToBigQuery; @@ -33,6 +35,7 @@ import feast.ingestion.utils.StoreUtil; import feast.ingestion.values.FailedElement; import feast.types.FeatureRowProto.FeatureRow; +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -57,15 +60,14 @@ public class ImportJob { * @param args arguments to be passed to Beam pipeline * @throws InvalidProtocolBufferException if options passed to the pipeline are invalid */ - public static void main(String[] args) throws InvalidProtocolBufferException { + public static void main(String[] args) throws IOException { ImportOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().create().as(ImportOptions.class); runPipeline(options); } @SuppressWarnings("UnusedReturnValue") - public static PipelineResult runPipeline(ImportOptions options) - throws InvalidProtocolBufferException { + public static PipelineResult runPipeline(ImportOptions options) throws IOException { /* * Steps: * 1. Read messages from Feast Source as FeatureRow @@ -80,8 +82,10 @@ public static PipelineResult runPipeline(ImportOptions options) log.info("Starting import job with settings: \n{}", options.toString()); - List featureSets = - SpecUtil.parseFeatureSetSpecJsonList(options.getFeatureSetJson()); + BZip2Decompressor> decompressor = + new BZip2Decompressor<>(new StringListStreamConverter()); + List featureSetJson = decompressor.decompress(options.getFeatureSetJson()); + List featureSets = SpecUtil.parseFeatureSetSpecJsonList(featureSetJson); List stores = SpecUtil.parseStoreJsonList(options.getStoreJson()); for (Store store : stores) { diff --git a/ingestion/src/main/java/feast/ingestion/options/BZip2Compressor.java b/ingestion/src/main/java/feast/ingestion/options/BZip2Compressor.java new file mode 100644 index 00000000000..b7e4e6ee0af --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/options/BZip2Compressor.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.options; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; + +public class BZip2Compressor implements OptionCompressor { + + private final OptionByteConverter byteConverter; + + public BZip2Compressor(OptionByteConverter byteConverter) { + this.byteConverter = byteConverter; + } + /** + * Compress pipeline option using BZip2 + * + * @param option Pipeline option value + * @return BZip2 compressed option value + * @throws IOException + */ + @Override + public byte[] compress(T option) throws IOException { + ByteArrayOutputStream compressedStream = new ByteArrayOutputStream(); + try (BZip2CompressorOutputStream bzip2Output = + new BZip2CompressorOutputStream(compressedStream)) { + bzip2Output.write(byteConverter.toByte(option)); + } + + return compressedStream.toByteArray(); + } +} diff --git a/ingestion/src/main/java/feast/ingestion/options/BZip2Decompressor.java b/ingestion/src/main/java/feast/ingestion/options/BZip2Decompressor.java new file mode 100644 index 00000000000..ce49c1be6e6 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/options/BZip2Decompressor.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.options; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; + +public class BZip2Decompressor implements OptionDecompressor { + + private final InputStreamConverter inputStreamConverter; + + public BZip2Decompressor(InputStreamConverter inputStreamConverter) { + this.inputStreamConverter = inputStreamConverter; + } + + @Override + public T decompress(byte[] compressed) throws IOException { + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(compressed); + BZip2CompressorInputStream bzip2Input = new BZip2CompressorInputStream(inputStream)) { + return inputStreamConverter.readStream(bzip2Input); + } + } +} diff --git a/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java b/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java index b299bb47e55..1fa127d6629 100644 --- a/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java +++ b/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java @@ -26,18 +26,19 @@ /** Options passed to Beam to influence the job's execution environment */ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions, DirectOptions { + @Required @Description( - "JSON string representation of the FeatureSet that the import job will process." + "JSON string representation of the FeatureSet that the import job will process, in BZip2 binary format." + "FeatureSet follows the format in feast.core.FeatureSet proto." + "Mutliple FeatureSetSpec can be passed by specifying '--featureSet={...}' multiple times" + "The conversion of Proto message to JSON should follow this mapping:" + "https://developers.google.com/protocol-buffers/docs/proto3#json" + "Please minify and remove all insignificant whitespace such as newline in the JSON string" + "to prevent error when parsing the options") - List getFeatureSetJson(); + byte[] getFeatureSetJson(); - void setFeatureSetJson(List featureSetJson); + void setFeatureSetJson(byte[] featureSetJson); @Required @Description( @@ -64,8 +65,7 @@ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions, */ void setDeadLetterTableSpec(String deadLetterTableSpec); - // TODO: expound - @Description("MetricsAccumulator exporter type to instantiate.") + @Description("MetricsAccumulator exporter type to instantiate. Supported type: statsd") @Default.String("none") String getMetricsExporterType(); @@ -83,4 +83,14 @@ public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions, int getStatsdPort(); void setStatsdPort(int StatsdPort); + + @Description( + "Fixed window size in seconds (default 60) to apply before aggregating the numerical value of " + + "features and exporting the aggregated values as metrics. Refer to " + + "feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java" + + "for the metric nameas and types used.") + @Default.Integer(60) + int getWindowSizeInSecForFeatureValueMetric(); + + void setWindowSizeInSecForFeatureValueMetric(int seconds); } diff --git a/ingestion/src/main/java/feast/ingestion/options/InputStreamConverter.java b/ingestion/src/main/java/feast/ingestion/options/InputStreamConverter.java new file mode 100644 index 00000000000..e2fef732368 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/options/InputStreamConverter.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.options; + +import java.io.IOException; +import java.io.InputStream; + +public interface InputStreamConverter { + + /** + * Used in conjunction with {@link OptionDecompressor} to decompress the pipeline option + * + * @param inputStream Input byte stream in compressed format + * @return Decompressed pipeline option value + */ + T readStream(InputStream inputStream) throws IOException; +} diff --git a/ingestion/src/main/java/feast/ingestion/options/OptionByteConverter.java b/ingestion/src/main/java/feast/ingestion/options/OptionByteConverter.java new file mode 100644 index 00000000000..ff5a41a627d --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/options/OptionByteConverter.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.options; + +import java.io.IOException; + +public interface OptionByteConverter { + + /** + * Used in conjunction with {@link OptionCompressor} to compress the pipeline option + * + * @param option Pipeline option value + * @return byte representation of the pipeline option value, without compression. + */ + byte[] toByte(T option) throws IOException; +} diff --git a/ingestion/src/main/java/feast/ingestion/options/OptionCompressor.java b/ingestion/src/main/java/feast/ingestion/options/OptionCompressor.java new file mode 100644 index 00000000000..b2345fc3eb1 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/options/OptionCompressor.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.options; + +import java.io.IOException; + +public interface OptionCompressor { + + /** + * Compress pipeline option into bytes format. This is necessary as some Beam runner has + * limitation in terms of pipeline option size. + * + * @param option Pipeline option value + * @return Compressed values of the option, as byte array + */ + byte[] compress(T option) throws IOException; +} diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto b/ingestion/src/main/java/feast/ingestion/options/OptionDecompressor.java similarity index 58% rename from ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto rename to ingestion/src/main/java/feast/ingestion/options/OptionDecompressor.java index 9730b49ec3b..affeafdaa0b 100644 --- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceKey.proto +++ b/ingestion/src/main/java/feast/ingestion/options/OptionDecompressor.java @@ -1,5 +1,6 @@ /* - * Copyright 2018 The Feast Authors + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,13 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package feast.ingestion.options; -syntax = "proto3"; +import java.io.IOException; -option java_package = "feast_ingestion.types"; -option java_outer_classname = "CoalesceKeyProto"; +public interface OptionDecompressor { -message CoalesceKey { - string entityName = 1; - string entityKey = 2; -} \ No newline at end of file + /** + * Decompress pipeline option from byte array. + * + * @param compressed Compressed pipeline option value + * @return Decompressed pipeline option + */ + T decompress(byte[] compressed) throws IOException; +} diff --git a/ingestion/src/main/java/feast/ingestion/options/StringListStreamConverter.java b/ingestion/src/main/java/feast/ingestion/options/StringListStreamConverter.java new file mode 100644 index 00000000000..d7277f3c7d6 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/options/StringListStreamConverter.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.options; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.List; +import java.util.stream.Collectors; + +public class StringListStreamConverter implements InputStreamConverter> { + + /** + * Convert Input byte stream to newline separated strings + * + * @param inputStream Input byte stream + * @return List of string + */ + @Override + public List readStream(InputStream inputStream) throws IOException { + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); + List stringList = reader.lines().collect(Collectors.toList()); + reader.close(); + return stringList; + } +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapper.java b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapper.java new file mode 100644 index 00000000000..65d3130e3c3 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapper.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.transform; + +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.mapping.Mapper.Option; +import feast.store.serving.cassandra.CassandraMutation; +import java.io.Serializable; +import java.util.Iterator; +import java.util.concurrent.Future; +import org.apache.beam.sdk.io.cassandra.Mapper; + +/** A {@link Mapper} that supports writing {@code CassandraMutation}s with the Beam Cassandra IO. */ +public class CassandraMutationMapper implements Mapper, Serializable { + + private com.datastax.driver.mapping.Mapper mapper; + private Boolean tracing; + + CassandraMutationMapper( + com.datastax.driver.mapping.Mapper mapper, Boolean tracing) { + this.mapper = mapper; + this.tracing = tracing; + } + + @Override + public Iterator map(ResultSet resultSet) { + throw new UnsupportedOperationException("Only supports write operations"); + } + + @Override + public Future deleteAsync(CassandraMutation entityClass) { + throw new UnsupportedOperationException("Only supports write operations"); + } + + /** + * Saves records to Cassandra with: - Cassandra's internal write time set to the timestamp of the + * record. Cassandra will not override an existing record with the same partition key if the write + * time is older - Expiration of the record + * + * @param entityClass Cassandra's object mapper + */ + @Override + public Future saveAsync(CassandraMutation entityClass) { + return mapper.saveAsync( + entityClass, + Option.timestamp(entityClass.getWriteTime()), + Option.ttl(entityClass.getTtl()), + Option.consistencyLevel(ConsistencyLevel.ONE), + Option.tracing(tracing)); + } +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapperFactory.java b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapperFactory.java new file mode 100644 index 00000000000..18596564b80 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/transform/CassandraMutationMapperFactory.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.transform; + +import com.datastax.driver.core.Session; +import com.datastax.driver.mapping.MappingManager; +import feast.store.serving.cassandra.CassandraMutation; +import org.apache.beam.sdk.io.cassandra.Mapper; +import org.apache.beam.sdk.transforms.SerializableFunction; + +public class CassandraMutationMapperFactory implements SerializableFunction { + + private transient MappingManager mappingManager; + private Class entityClass; + private boolean tracing; + + public CassandraMutationMapperFactory(Class entityClass, Boolean tracing) { + this.entityClass = entityClass; + this.tracing = tracing; + } + + @Override + public Mapper apply(Session session) { + if (mappingManager == null) { + this.mappingManager = new MappingManager(session); + } + + return new CassandraMutationMapper(mappingManager.mapper(entityClass), this.tracing); + } +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java b/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java index 778540595a2..6087cee79c8 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java +++ b/ingestion/src/main/java/feast/ingestion/transform/WriteToStore.java @@ -16,24 +16,31 @@ */ package feast.ingestion.transform; +import com.datastax.driver.core.ConsistencyLevel; +import com.datastax.driver.core.Session; import com.google.api.services.bigquery.model.TableDataInsertAllResponse.InsertErrors; import com.google.api.services.bigquery.model.TableRow; import com.google.auto.value.AutoValue; import feast.core.FeatureSetProto.FeatureSet; import feast.core.StoreProto.Store; import feast.core.StoreProto.Store.BigQueryConfig; -import feast.core.StoreProto.Store.RedisConfig; +import feast.core.StoreProto.Store.CassandraConfig; import feast.core.StoreProto.Store.StoreType; import feast.ingestion.options.ImportOptions; import feast.ingestion.utils.ResourceUtil; import feast.ingestion.values.FailedElement; import feast.store.serving.bigquery.FeatureRowToTableRow; import feast.store.serving.bigquery.GetTableDestination; +import feast.store.serving.cassandra.CassandraMutation; +import feast.store.serving.cassandra.FeatureRowToCassandraMutationDoFn; import feast.store.serving.redis.FeatureRowToRedisMutationDoFn; import feast.store.serving.redis.RedisCustomIO; import feast.types.FeatureRowProto.FeatureRow; import java.io.IOException; +import java.util.Arrays; import java.util.Map; +import org.apache.beam.sdk.io.cassandra.CassandraIO; +import org.apache.beam.sdk.io.cassandra.Mapper; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.Method; @@ -47,6 +54,7 @@ import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TypeDescriptors; @@ -88,16 +96,14 @@ public PDone expand(PCollection input) { switch (storeType) { case REDIS: - RedisConfig redisConfig = getStore().getRedisConfig(); - PCollection redisWriteResult = input - .apply( - "FeatureRowToRedisMutation", - ParDo.of(new FeatureRowToRedisMutationDoFn(getFeatureSets()))) - .apply( - "WriteRedisMutationToRedis", - RedisCustomIO.write(redisConfig)); + PCollection redisWriteResult = + input + .apply( + "FeatureRowToRedisMutation", + ParDo.of(new FeatureRowToRedisMutationDoFn(getFeatureSets()))) + .apply("WriteRedisMutationToRedis", RedisCustomIO.write(getStore())); if (options.getDeadLetterTableSpec() != null) { - redisWriteResult.apply( + redisWriteResult.apply( WriteFailedElementToBigQuery.newBuilder() .setTableSpec(options.getDeadLetterTableSpec()) .setJsonSchema(ResourceUtil.getDeadletterTableSchemaJson()) @@ -152,6 +158,29 @@ public void processElement(ProcessContext context) { .build()); } break; + case CASSANDRA: + CassandraConfig cassandraConfig = getStore().getCassandraConfig(); + log.info("Tracing Enabled: {}", cassandraConfig.getTracing()); + SerializableFunction mapperFactory = + new CassandraMutationMapperFactory( + CassandraMutation.class, cassandraConfig.getTracing()); + input + .apply( + "Create CassandraMutation from FeatureRow", + ParDo.of( + new FeatureRowToCassandraMutationDoFn( + getFeatureSets(), + cassandraConfig.getDefaultTtl(), + cassandraConfig.getVersionless()))) + .apply( + CassandraIO.write() + .withHosts(Arrays.asList(cassandraConfig.getBootstrapHosts().split(","))) + .withPort(cassandraConfig.getPort()) + .withKeyspace(cassandraConfig.getKeyspace()) + .withEntity(CassandraMutation.class) + .withMapperFactoryFn(mapperFactory) + .withConsistencyLevel(String.valueOf(ConsistencyLevel.ALL))); + break; default: log.error("Store type '{}' is not supported. No Feature Row will be written.", storeType); break; diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java index 7d61a62f3fc..c31d3c535e9 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java @@ -24,10 +24,8 @@ import feast.types.FieldProto; import feast.types.ValueProto.Value.ValCase; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.values.TupleTag; @@ -111,10 +109,7 @@ public void processElement(ProcessContext context) { } context.output(getFailureTag(), failedElement.build()); } else { - featureRow = featureRow.toBuilder() - .clearFields() - .addAllFields(fields) - .build(); + featureRow = featureRow.toBuilder().clearFields().addAllFields(fields).build(); context.output(getSuccessTag(), featureRow); } } diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java new file mode 100644 index 00000000000..a4ed07b5052 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java @@ -0,0 +1,325 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.transform.metrics; + +import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.FEATURE_SET_NAME_TAG_KEY; +import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.FEATURE_SET_PROJECT_TAG_KEY; +import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.FEATURE_SET_VERSION_TAG_KEY; +import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.FEATURE_TAG_KEY; +import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.INGESTION_JOB_NAME_KEY; +import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.METRIC_PREFIX; +import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.STORE_TAG_KEY; + +import com.google.auto.value.AutoValue; +import com.timgroup.statsd.NonBlockingStatsDClient; +import com.timgroup.statsd.StatsDClient; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import feast.types.ValueProto.Value; +import java.util.ArrayList; +import java.util.DoubleSummaryStatistics; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.values.KV; +import org.apache.commons.math3.stat.descriptive.rank.Percentile; +import org.slf4j.Logger; + +/** + * WriteFeatureValueMetricsDoFn accepts key value of FeatureSetRef(str) to FeatureRow(List) and + * writes a histogram of the numerical values of each feature to StatsD. + * + *

The histogram of the numerical values is represented as the following in StatsD: + * + *

    + *
  • gauge of feature_value_min + *
  • gauge of feature_value_max + *
  • gauge of feature_value_mean + *
  • gauge of feature_value_percentile_50 + *
  • gauge of feature_value_percentile_90 + *
  • gauge of feature_value_percentile_95 + *
+ * + *

StatsD timing/histogram metric type is not used since it does not support negative values. + */ +@AutoValue +public abstract class WriteFeatureValueMetricsDoFn + extends DoFn>, Void> { + + abstract String getStoreName(); + + abstract String getStatsdHost(); + + abstract int getStatsdPort(); + + static Builder newBuilder() { + return new AutoValue_WriteFeatureValueMetricsDoFn.Builder(); + } + + @AutoValue.Builder + abstract static class Builder { + + abstract Builder setStoreName(String storeName); + + abstract Builder setStatsdHost(String statsdHost); + + abstract Builder setStatsdPort(int statsdPort); + + abstract WriteFeatureValueMetricsDoFn build(); + } + + private static final Logger log = + org.slf4j.LoggerFactory.getLogger(WriteFeatureValueMetricsDoFn.class); + private StatsDClient statsDClient; + public static String GAUGE_NAME_FEATURE_VALUE_MIN = "feature_value_min"; + public static String GAUGE_NAME_FEATURE_VALUE_MAX = "feature_value_max"; + public static String GAUGE_NAME_FEATURE_VALUE_MEAN = "feature_value_mean"; + public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25 = "feature_value_percentile_25"; + public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_50 = "feature_value_percentile_50"; + public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_90 = "feature_value_percentile_90"; + public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95 = "feature_value_percentile_95"; + public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99 = "feature_value_percentile_99"; + + @Setup + public void setup() { + // Note that exception may be thrown during StatsD client instantiation but no exception + // will be thrown when sending metrics (mimicking the UDP protocol behaviour). + // https://jar-download.com/artifacts/com.datadoghq/java-dogstatsd-client/2.1.1/documentation + // https://github.com/DataDog/java-dogstatsd-client#unix-domain-socket-support + try { + statsDClient = new NonBlockingStatsDClient(METRIC_PREFIX, getStatsdHost(), getStatsdPort()); + } catch (Exception e) { + log.error("StatsD client cannot be started: " + e.getMessage()); + } + } + + @Teardown + public void tearDown() { + if (statsDClient != null) { + statsDClient.close(); + } + } + + @ProcessElement + public void processElement( + ProcessContext context, + @Element KV> featureSetRefToFeatureRows) { + if (statsDClient == null) { + return; + } + + String featureSetRef = featureSetRefToFeatureRows.getKey(); + if (featureSetRef == null) { + return; + } + String[] colonSplits = featureSetRef.split(":"); + if (colonSplits.length != 2) { + log.error( + "Skip writing feature value metrics because the feature set reference '{}' does not" + + "follow the required format /:", + featureSetRef); + return; + } + String[] slashSplits = colonSplits[0].split("/"); + if (slashSplits.length != 2) { + log.error( + "Skip writing feature value metrics because the feature set reference '{}' does not" + + "follow the required format /:", + featureSetRef); + return; + } + String projectName = slashSplits[0]; + String featureSetName = slashSplits[1]; + String version = colonSplits[1]; + + Map featureNameToStats = new HashMap<>(); + Map> featureNameToValues = new HashMap<>(); + for (FeatureRow featureRow : featureSetRefToFeatureRows.getValue()) { + for (Field field : featureRow.getFieldsList()) { + updateStats(featureNameToStats, featureNameToValues, field); + } + } + + for (Entry entry : featureNameToStats.entrySet()) { + String featureName = entry.getKey(); + DoubleSummaryStatistics stats = entry.getValue(); + String[] tags = { + STORE_TAG_KEY + ":" + getStoreName(), + FEATURE_SET_PROJECT_TAG_KEY + ":" + projectName, + FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, + FEATURE_SET_VERSION_TAG_KEY + ":" + version, + FEATURE_TAG_KEY + ":" + featureName, + INGESTION_JOB_NAME_KEY + ":" + context.getPipelineOptions().getJobName() + }; + + // stats can return non finite values when there is no element + // or there is an element that is not a number. Metric should only be sent for finite values. + if (Double.isFinite(stats.getMin())) { + if (stats.getMin() < 0) { + // StatsD gauge will asssign a delta instead of the actual value, if there is a sign in + // the value. E.g. if the value is negative, a delta will be assigned. For this reason, + // the gauge value is set to zero beforehand. + // https://github.com/statsd/statsd/blob/master/docs/metric_types.md#gauges + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MIN, 0, tags); + } + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MIN, stats.getMin(), tags); + } + if (Double.isFinite(stats.getMax())) { + if (stats.getMax() < 0) { + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MAX, 0, tags); + } + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MAX, stats.getMax(), tags); + } + if (Double.isFinite(stats.getAverage())) { + if (stats.getAverage() < 0) { + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MEAN, 0, tags); + } + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MEAN, stats.getAverage(), tags); + } + + // For percentile calculation, Percentile class from commons-math3 from Apache is used. + // Percentile requires double[], hence the conversion below. + if (!featureNameToValues.containsKey(featureName)) { + continue; + } + List valueList = featureNameToValues.get(featureName); + if (valueList == null || valueList.size() < 1) { + continue; + } + double[] values = new double[valueList.size()]; + for (int i = 0; i < values.length; i++) { + values[i] = valueList.get(i); + } + + double p25 = new Percentile().evaluate(values, 25); + if (p25 < 0) { + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25, 0, tags); + } + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25, p25, tags); + + double p50 = new Percentile().evaluate(values, 50); + if (p50 < 0) { + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_50, 0, tags); + } + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_50, p50, tags); + + double p90 = new Percentile().evaluate(values, 90); + if (p90 < 0) { + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_90, 0, tags); + } + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_90, p90, tags); + + double p95 = new Percentile().evaluate(values, 95); + if (p95 < 0) { + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95, 0, tags); + } + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95, p95, tags); + + double p99 = new Percentile().evaluate(values, 99); + if (p99 < 0) { + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99, 0, tags); + } + statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99, p99, tags); + } + } + + // Update stats and values array for the feature represented by the field. + // If the field contains non-numerical or non-boolean value, the stats and values array + // won't get updated because we are only concerned with numerical value in metrics data. + // For boolean value, true and false are treated as numerical value of 1 of 0 respectively. + private void updateStats( + Map featureNameToStats, + Map> featureNameToValues, + Field field) { + if (featureNameToStats == null || featureNameToValues == null || field == null) { + return; + } + + String featureName = field.getName(); + if (!featureNameToStats.containsKey(featureName)) { + featureNameToStats.put(featureName, new DoubleSummaryStatistics()); + } + if (!featureNameToValues.containsKey(featureName)) { + featureNameToValues.put(featureName, new ArrayList<>()); + } + + Value value = field.getValue(); + DoubleSummaryStatistics stats = featureNameToStats.get(featureName); + List values = featureNameToValues.get(featureName); + + switch (value.getValCase()) { + case INT32_VAL: + stats.accept(value.getInt32Val()); + values.add(((double) value.getInt32Val())); + break; + case INT64_VAL: + stats.accept(value.getInt64Val()); + values.add((double) value.getInt64Val()); + break; + case DOUBLE_VAL: + stats.accept(value.getDoubleVal()); + values.add(value.getDoubleVal()); + break; + case FLOAT_VAL: + stats.accept(value.getFloatVal()); + values.add((double) value.getFloatVal()); + break; + case BOOL_VAL: + stats.accept(value.getBoolVal() ? 1 : 0); + values.add(value.getBoolVal() ? 1d : 0d); + break; + case INT32_LIST_VAL: + for (Integer val : value.getInt32ListVal().getValList()) { + stats.accept(val); + values.add(((double) val)); + } + break; + case INT64_LIST_VAL: + for (Long val : value.getInt64ListVal().getValList()) { + stats.accept(val); + values.add(((double) val)); + } + break; + case DOUBLE_LIST_VAL: + for (Double val : value.getDoubleListVal().getValList()) { + stats.accept(val); + values.add(val); + } + break; + case FLOAT_LIST_VAL: + for (Float val : value.getFloatListVal().getValList()) { + stats.accept(val); + values.add(((double) val)); + } + break; + case BOOL_LIST_VAL: + for (Boolean val : value.getBoolListVal().getValList()) { + stats.accept(val ? 1 : 0); + values.add(val ? 1d : 0d); + } + break; + case BYTES_VAL: + case BYTES_LIST_VAL: + case STRING_VAL: + case STRING_LIST_VAL: + case VAL_NOT_SET: + default: + } + } +} diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java index 43f314aa861..10322ac812f 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteMetricsTransform.java @@ -21,11 +21,16 @@ import feast.ingestion.values.FailedElement; import feast.types.FeatureRowProto.FeatureRow; import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.GroupByKey; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.windowing.FixedWindows; +import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollectionTuple; import org.apache.beam.sdk.values.PDone; import org.apache.beam.sdk.values.TupleTag; +import org.joda.time.Duration; @AutoValue public abstract class WriteMetricsTransform extends PTransform { @@ -79,6 +84,42 @@ public PDone expand(PCollectionTuple input) { .setStoreName(getStoreName()) .build())); + // 1. Apply a fixed window + // 2. Group feature row by feature set reference + // 3. Calculate min, max, mean, percentiles of numerical values of features in the window + // and + // 4. Send the aggregate value to StatsD metric collector. + // + // NOTE: window is applied here so the metric collector will not be overwhelmed with + // metrics data. And for metric data, only statistic of the values are usually required + // vs the actual values. + input + .get(getSuccessTag()) + .apply( + "FixedWindow", + Window.into( + FixedWindows.of( + Duration.standardSeconds( + options.getWindowSizeInSecForFeatureValueMetric())))) + .apply( + "ConvertTo_FeatureSetRefToFeatureRow", + ParDo.of( + new DoFn>() { + @ProcessElement + public void processElement(ProcessContext c, @Element FeatureRow featureRow) { + c.output(KV.of(featureRow.getFeatureSet(), featureRow)); + } + })) + .apply("GroupByFeatureSetRef", GroupByKey.create()) + .apply( + "WriteFeatureValueMetrics", + ParDo.of( + WriteFeatureValueMetricsDoFn.newBuilder() + .setStatsdHost(options.getStatsdHost()) + .setStatsdPort(options.getStatsdPort()) + .setStoreName(getStoreName()) + .build())); + return PDone.in(input.getPipeline()); case "none": default: diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java index db2d1acd6d8..2cd1ee94ecc 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java @@ -31,13 +31,13 @@ public abstract class WriteRowMetricsDoFn extends DoFn { private static final Logger log = org.slf4j.LoggerFactory.getLogger(WriteRowMetricsDoFn.class); - private final String METRIC_PREFIX = "feast_ingestion"; - private final String STORE_TAG_KEY = "feast_store"; - private final String FEATURE_SET_PROJECT_TAG_KEY = "feast_project_name"; - private final String FEATURE_SET_NAME_TAG_KEY = "feast_featureSet_name"; - private final String FEATURE_SET_VERSION_TAG_KEY = "feast_featureSet_version"; - private final String FEATURE_TAG_KEY = "feast_feature_name"; - private final String INGESTION_JOB_NAME_KEY = "ingestion_job_name"; + public static final String METRIC_PREFIX = "feast_ingestion"; + public static final String STORE_TAG_KEY = "feast_store"; + public static final String FEATURE_SET_PROJECT_TAG_KEY = "feast_project_name"; + public static final String FEATURE_SET_NAME_TAG_KEY = "feast_featureSet_name"; + public static final String FEATURE_SET_VERSION_TAG_KEY = "feast_featureSet_version"; + public static final String FEATURE_TAG_KEY = "feast_feature_name"; + public static final String INGESTION_JOB_NAME_KEY = "ingestion_job_name"; public abstract String getStoreName(); diff --git a/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java b/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java index 7af98fb8f00..3e5ff76cac0 100644 --- a/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java +++ b/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java @@ -18,6 +18,14 @@ import static feast.types.ValueProto.ValueType; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.KeyspaceMetadata; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.schemabuilder.Create; +import com.datastax.driver.core.schemabuilder.KeyspaceOptions; +import com.datastax.driver.core.schemabuilder.SchemaBuilder; +import com.datastax.driver.mapping.MappingManager; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.DatasetId; @@ -40,17 +48,24 @@ import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.FeatureSetProto.FeatureSpec; import feast.core.StoreProto.Store; +import feast.core.StoreProto.Store.CassandraConfig; import feast.core.StoreProto.Store.RedisConfig; import feast.core.StoreProto.Store.StoreType; +import feast.store.serving.cassandra.CassandraMutation; import feast.types.ValueProto.ValueType.Enum; +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisConnectionException; +import io.lettuce.core.RedisURI; +import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.exceptions.JedisConnectionException; // TODO: Create partitioned table by default @@ -113,6 +128,9 @@ public static void setupStore(Store store, FeatureSet featureSet) { store.getBigqueryConfig().getDatasetId(), BigQueryOptions.getDefaultInstance().getService()); break; + case CASSANDRA: + StoreUtil.setupCassandra(store.getCassandraConfig()); + break; default: log.warn("Store type '{}' is unsupported", storeType); break; @@ -239,15 +257,86 @@ public static void setupBigQuery( * @param redisConfig Plase refer to feast.core.Store proto */ public static void checkRedisConnection(RedisConfig redisConfig) { - JedisPool jedisPool = new JedisPool(redisConfig.getHost(), redisConfig.getPort()); + RedisClient redisClient = + RedisClient.create(RedisURI.create(redisConfig.getHost(), redisConfig.getPort())); try { - jedisPool.getResource(); - } catch (JedisConnectionException e) { + redisClient.connect(); + } catch (RedisConnectionException e) { throw new RuntimeException( String.format( "Failed to connect to Redis at host: '%s' port: '%d'. Please check that your Redis is running and accessible from Feast.", redisConfig.getHost(), redisConfig.getPort())); } - jedisPool.close(); + redisClient.shutdown(); + } + + /** + * Ensures Cassandra is accessible, else throw a RuntimeException. Creates Cassandra keyspace and + * table if it does not already exist + * + * @param cassandraConfig Please refer to feast.core.Store proto + */ + public static void setupCassandra(CassandraConfig cassandraConfig) { + List contactPoints = + Arrays.stream(cassandraConfig.getBootstrapHosts().split(",")) + .map(host -> new InetSocketAddress(host, cassandraConfig.getPort())) + .collect(Collectors.toList()); + Cluster cluster = Cluster.builder().addContactPointsWithPorts(contactPoints).build(); + Session session; + + try { + String keyspace = cassandraConfig.getKeyspace(); + KeyspaceMetadata keyspaceMetadata = cluster.getMetadata().getKeyspace(keyspace); + if (keyspaceMetadata == null) { + log.info("Creating keyspace '{}'", keyspace); + Map replicationOptions = + cassandraConfig.getReplicationOptionsMap().entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + KeyspaceOptions createKeyspace = + SchemaBuilder.createKeyspace(keyspace) + .ifNotExists() + .with() + .replication(replicationOptions); + session = cluster.newSession(); + session.execute(createKeyspace); + } + + session = cluster.connect(keyspace); + // Currently no support for creating table from entity mapper: + // https://datastax-oss.atlassian.net/browse/JAVA-569 + Create createTable = + SchemaBuilder.createTable(keyspace, cassandraConfig.getTableName()) + .ifNotExists() + .addPartitionKey(CassandraMutation.ENTITIES, DataType.text()) + .addClusteringColumn(CassandraMutation.FEATURE, DataType.text()) + .addColumn(CassandraMutation.VALUE, DataType.blob()); + log.info("Create Cassandra table if not exists.."); + session.execute(createTable); + + validateCassandraTable(session); + + session.close(); + } catch (RuntimeException e) { + throw new RuntimeException( + String.format( + "Failed to connect to Cassandra at bootstrap hosts: '%s' port: '%s'. Please check that your Cassandra is running and accessible from Feast.", + contactPoints.stream() + .map(InetSocketAddress::getHostName) + .collect(Collectors.joining(",")), + cassandraConfig.getPort()), + e); + } + cluster.close(); + } + + private static void validateCassandraTable(Session session) { + try { + new MappingManager(session).mapper(CassandraMutation.class).getTableMetadata(); + } catch (RuntimeException e) { + throw new RuntimeException( + String.format( + "Table created does not match the datastax object mapper: %s", + CassandraMutation.class.getSimpleName())); + } } } diff --git a/ingestion/src/main/java/feast/ingestion/utils/ValueUtil.java b/ingestion/src/main/java/feast/ingestion/utils/ValueUtil.java new file mode 100644 index 00000000000..87a327e7726 --- /dev/null +++ b/ingestion/src/main/java/feast/ingestion/utils/ValueUtil.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.utils; + +import feast.types.ValueProto.Value; + +/** + * Utility class for converting {@link Value} of different types to a string for storing as key in + * data stores + */ +public class ValueUtil { + + public static String toString(Value value) { + String strValue; + switch (value.getValCase()) { + case BYTES_VAL: + strValue = value.getBytesVal().toString(); + break; + case STRING_VAL: + strValue = value.getStringVal(); + break; + case INT32_VAL: + strValue = String.valueOf(value.getInt32Val()); + break; + case INT64_VAL: + strValue = String.valueOf(value.getInt64Val()); + break; + case DOUBLE_VAL: + strValue = String.valueOf(value.getDoubleVal()); + break; + case FLOAT_VAL: + strValue = String.valueOf(value.getFloatVal()); + break; + case BOOL_VAL: + strValue = String.valueOf(value.getBoolVal()); + break; + default: + throw new IllegalArgumentException( + String.format("toString method not supported for type %s", value.getValCase())); + } + return strValue; + } +} diff --git a/ingestion/src/main/java/feast/retry/BackOffExecutor.java b/ingestion/src/main/java/feast/retry/BackOffExecutor.java index 7e38a3cf706..344c65ac424 100644 --- a/ingestion/src/main/java/feast/retry/BackOffExecutor.java +++ b/ingestion/src/main/java/feast/retry/BackOffExecutor.java @@ -1,38 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package feast.retry; +import java.io.Serializable; import org.apache.beam.sdk.util.BackOff; import org.apache.beam.sdk.util.BackOffUtils; import org.apache.beam.sdk.util.FluentBackoff; import org.apache.beam.sdk.util.Sleeper; import org.joda.time.Duration; -import java.io.IOException; -import java.io.Serializable; - public class BackOffExecutor implements Serializable { - private static FluentBackoff backoff; + private final Integer maxRetries; + private final Duration initialBackOff; - public BackOffExecutor(Integer maxRetries, Duration initialBackOff) { - backoff = FluentBackoff.DEFAULT - .withMaxRetries(maxRetries) - .withInitialBackoff(initialBackOff); - } + public BackOffExecutor(Integer maxRetries, Duration initialBackOff) { + this.maxRetries = maxRetries; + this.initialBackOff = initialBackOff; + } + + public void execute(Retriable retriable) throws Exception { + FluentBackoff backoff = + FluentBackoff.DEFAULT.withMaxRetries(maxRetries).withInitialBackoff(initialBackOff); + execute(retriable, backoff); + } - public void execute(Retriable retriable) throws Exception { - Sleeper sleeper = Sleeper.DEFAULT; - BackOff backOff = backoff.backoff(); - while(true) { - try { - retriable.execute(); - break; - } catch (Exception e) { - if(retriable.isExceptionRetriable(e) && BackOffUtils.next(sleeper, backOff)) { - retriable.cleanUpAfterFailure(); - } else { - throw e; - } - } + private void execute(Retriable retriable, FluentBackoff backoff) throws Exception { + Sleeper sleeper = Sleeper.DEFAULT; + BackOff backOff = backoff.backoff(); + while (true) { + try { + retriable.execute(); + break; + } catch (Exception e) { + if (retriable.isExceptionRetriable(e) && BackOffUtils.next(sleeper, backOff)) { + retriable.cleanUpAfterFailure(); + } else { + throw e; } + } } + } } diff --git a/ingestion/src/main/java/feast/retry/Retriable.java b/ingestion/src/main/java/feast/retry/Retriable.java index 8fd76fedbb1..30676fe8208 100644 --- a/ingestion/src/main/java/feast/retry/Retriable.java +++ b/ingestion/src/main/java/feast/retry/Retriable.java @@ -1,7 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package feast.retry; public interface Retriable { - void execute(); - Boolean isExceptionRetriable(Exception e); - void cleanUpAfterFailure(); + void execute() throws Exception; + + Boolean isExceptionRetriable(Exception e); + + void cleanUpAfterFailure(); } diff --git a/ingestion/src/main/java/feast/store/serving/cassandra/CassandraMutation.java b/ingestion/src/main/java/feast/store/serving/cassandra/CassandraMutation.java new file mode 100644 index 00000000000..d6f57056ded --- /dev/null +++ b/ingestion/src/main/java/feast/store/serving/cassandra/CassandraMutation.java @@ -0,0 +1,131 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.store.serving.cassandra; + +import com.datastax.driver.mapping.annotations.ClusteringColumn; +import com.datastax.driver.mapping.annotations.Computed; +import com.datastax.driver.mapping.annotations.PartitionKey; +import com.datastax.driver.mapping.annotations.Table; +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.ingestion.utils.ValueUtil; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.stream.Collectors; +import org.apache.beam.sdk.coders.AvroCoder; +import org.apache.beam.sdk.coders.DefaultCoder; + +/** + * Cassandra's object mapper that handles basic CRUD operations in Cassandra tables More info: + * https://docs.datastax.com/en/developer/java-driver/3.1/manual/object_mapper/ + */ +@DefaultCoder(value = AvroCoder.class) +@Table(name = "feature_store") +public final class CassandraMutation implements Serializable { + + public static final String ENTITIES = "entities"; + public static final String FEATURE = "feature"; + public static final String VALUE = "value"; + + @PartitionKey private final String entities; + + @ClusteringColumn private final String feature; + + private final ByteBuffer value; + + @Computed(value = "writetime(value)") + private final long writeTime; + + @Computed(value = "ttl(value)") + private final int ttl; + + // NoArgs constructor is needed when using Beam's CassandraIO withEntity and specifying this + // class, + // it looks for an init() method + CassandraMutation() { + this.entities = null; + this.feature = null; + this.value = null; + this.writeTime = 0; + this.ttl = 0; + } + + CassandraMutation(String entities, String feature, ByteBuffer value, long writeTime, int ttl) { + this.entities = entities; + this.feature = feature; + this.value = value; + this.writeTime = writeTime; + this.ttl = ttl; + } + + public long getWriteTime() { + return writeTime; + } + + public int getTtl() { + return ttl; + } + + static String keyFromFeatureRow( + FeatureSetSpec featureSetSpec, FeatureRow featureRow, Boolean versionless) { + List entityNames = + featureSetSpec.getEntitiesList().stream() + .map(EntitySpec::getName) + .collect(Collectors.toList()); + Collections.sort(entityNames); + HashMap entities = new HashMap<>(); + for (Field field : featureRow.getFieldsList()) { + if (entityNames.contains(field.getName())) { + entities.put(entityNames.get(entityNames.indexOf(field.getName())), field); + } + } + String fsName; + if (versionless) { + fsName = featureRow.getFeatureSet().split(":")[0]; + } else { + fsName = featureRow.getFeatureSet(); + } + return fsName + + ":" + + entityNames.stream() + .map( + f -> + entities.get(f).getName() + + "=" + + ValueUtil.toString(entities.get(f).getValue())) + .collect(Collectors.joining("|")); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (o instanceof CassandraMutation) { + CassandraMutation that = (CassandraMutation) o; + return this.entities.equals(that.entities) + && this.feature.equals(that.feature) + && this.value.equals(that.value) + && this.writeTime == that.writeTime + && this.ttl == that.ttl; + } + return false; + } +} diff --git a/ingestion/src/main/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFn.java new file mode 100644 index 00000000000..d2d626cdd88 --- /dev/null +++ b/ingestion/src/main/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFn.java @@ -0,0 +1,97 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.store.serving.cassandra; + +import com.google.protobuf.Duration; +import com.google.protobuf.util.Timestamps; +import feast.core.FeatureSetProto.FeatureSet; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.beam.sdk.transforms.DoFn; +import org.slf4j.Logger; + +public class FeatureRowToCassandraMutationDoFn extends DoFn { + + private static final Logger log = + org.slf4j.LoggerFactory.getLogger(FeatureRowToCassandraMutationDoFn.class); + private Map featureSets; + private Map maxAges; + private Boolean versionless; + + public FeatureRowToCassandraMutationDoFn( + Map featureSets, Duration defaultTtl, Boolean versionless) { + this.featureSets = featureSets; + this.maxAges = new HashMap<>(); + this.versionless = versionless; + for (FeatureSet set : featureSets.values()) { + FeatureSetSpec spec = set.getSpec(); + String featureSetName; + featureSetName = spec.getProject() + "/" + spec.getName() + ":" + spec.getVersion(); + if (spec.getMaxAge() != null && spec.getMaxAge().getSeconds() > 0) { + maxAges.put(featureSetName, Math.toIntExact(spec.getMaxAge().getSeconds())); + } else { + maxAges.put(featureSetName, Math.toIntExact(defaultTtl.getSeconds())); + } + } + } + + /** Output a Cassandra mutation object for every feature in the feature row. */ + @ProcessElement + public void processElement(ProcessContext context) { + FeatureRow featureRow = context.element(); + try { + FeatureSetSpec featureSetSpec = featureSets.get(featureRow.getFeatureSet()).getSpec(); + Set featureNames = + featureSetSpec.getFeaturesList().stream() + .map(FeatureSpec::getName) + .collect(Collectors.toSet()); + String key = CassandraMutation.keyFromFeatureRow(featureSetSpec, featureRow, versionless); + + Collection mutations = new ArrayList<>(); + for (Field field : featureRow.getFieldsList()) { + if (featureNames.contains(field.getName())) { + ByteBuffer value = ByteBuffer.wrap(field.getValue().toByteArray()); + if (!value.hasRemaining()) { + continue; + } + mutations.add( + new CassandraMutation( + key, + field.getName(), + value, + Timestamps.toMicros(featureRow.getEventTimestamp()), + maxAges.get(featureRow.getFeatureSet()))); + } + } + + mutations.forEach(context::output); + } catch (Exception e) { + log.error(e.getMessage(), e); + log.error(maxAges.toString()); + log.error(featureRow.getFeatureSet()); + } + } +} diff --git a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java index 27cca2ffb2e..ca017c1f756 100644 --- a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java +++ b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java @@ -18,14 +18,18 @@ import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSet; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSpec; import feast.storage.RedisProto.RedisKey; import feast.storage.RedisProto.RedisKey.Builder; import feast.store.serving.redis.RedisCustomIO.Method; import feast.store.serving.redis.RedisCustomIO.RedisMutation; import feast.types.FeatureRowProto.FeatureRow; import feast.types.FieldProto.Field; +import feast.types.ValueProto; +import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import org.apache.beam.sdk.transforms.DoFn; import org.slf4j.Logger; @@ -42,28 +46,66 @@ public FeatureRowToRedisMutationDoFn(Map featureSets) { private RedisKey getKey(FeatureRow featureRow) { FeatureSet featureSet = featureSets.get(featureRow.getFeatureSet()); - Set entityNames = + List entityNames = featureSet.getSpec().getEntitiesList().stream() .map(EntitySpec::getName) - .collect(Collectors.toSet()); + .sorted() + .collect(Collectors.toList()); + Map entityFields = new HashMap<>(); Builder redisKeyBuilder = RedisKey.newBuilder().setFeatureSet(featureRow.getFeatureSet()); for (Field field : featureRow.getFieldsList()) { if (entityNames.contains(field.getName())) { - redisKeyBuilder.addEntities(field); + entityFields.putIfAbsent( + field.getName(), + Field.newBuilder().setName(field.getName()).setValue(field.getValue()).build()); } } + for (String entityName : entityNames) { + redisKeyBuilder.addEntities(entityFields.get(entityName)); + } return redisKeyBuilder.build(); } + private byte[] getValue(FeatureRow featureRow) { + FeatureSetSpec spec = featureSets.get(featureRow.getFeatureSet()).getSpec(); + + List featureNames = + spec.getFeaturesList().stream().map(FeatureSpec::getName).collect(Collectors.toList()); + Map fieldValueOnlyMap = + featureRow.getFieldsList().stream() + .filter(field -> featureNames.contains(field.getName())) + .distinct() + .collect( + Collectors.toMap( + Field::getName, + field -> Field.newBuilder().setValue(field.getValue()).build())); + + List values = + featureNames.stream() + .sorted() + .map( + featureName -> + fieldValueOnlyMap.getOrDefault( + featureName, + Field.newBuilder().setValue(ValueProto.Value.getDefaultInstance()).build())) + .collect(Collectors.toList()); + + return FeatureRow.newBuilder() + .setEventTimestamp(featureRow.getEventTimestamp()) + .addAllFields(values) + .build() + .toByteArray(); + } + /** Output a redis mutation object for every feature in the feature row. */ @ProcessElement public void processElement(ProcessContext context) { FeatureRow featureRow = context.element(); try { - RedisKey key = getKey(featureRow); - RedisMutation redisMutation = - new RedisMutation(Method.SET, key.toByteArray(), featureRow.toByteArray(), null, null); + byte[] key = getKey(featureRow).toByteArray(); + byte[] value = getValue(featureRow); + RedisMutation redisMutation = new RedisMutation(Method.SET, key, value, null, null); context.output(redisMutation); } catch (Exception e) { log.error(e.getMessage(), e); diff --git a/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java b/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java index 20afc43d76c..633c2eb551d 100644 --- a/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java @@ -18,8 +18,13 @@ import feast.core.StoreProto; import feast.ingestion.values.FailedElement; -import feast.retry.BackOffExecutor; import feast.retry.Retriable; +import io.lettuce.core.RedisConnectionException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; import org.apache.avro.reflect.Nullable; import org.apache.beam.sdk.coders.AvroCoder; import org.apache.beam.sdk.coders.DefaultCoder; @@ -29,18 +34,9 @@ import org.apache.beam.sdk.transforms.windowing.GlobalWindow; import org.apache.beam.sdk.values.PCollection; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; -import redis.clients.jedis.exceptions.JedisConnectionException; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; public class RedisCustomIO { @@ -51,8 +47,8 @@ public class RedisCustomIO { private RedisCustomIO() {} - public static Write write(StoreProto.Store.RedisConfig redisConfig) { - return new Write(redisConfig); + public static Write write(StoreProto.Store store) { + return new Write(store); } public enum Method { @@ -164,12 +160,13 @@ public void setScore(@Nullable Long score) { } /** ServingStoreWrite data to a Redis server. */ - public static class Write extends PTransform, PCollection> { + public static class Write + extends PTransform, PCollection> { private WriteDoFn dofn; - private Write(StoreProto.Store.RedisConfig redisConfig) { - this.dofn = new WriteDoFn(redisConfig); + private Write(StoreProto.Store store) { + this.dofn = new WriteDoFn(store); } public Write withBatchSize(int batchSize) { @@ -189,22 +186,14 @@ public PCollection expand(PCollection input) { public static class WriteDoFn extends DoFn { - private final String host; - private final int port; - private final BackOffExecutor backOffExecutor; private final List mutations = new ArrayList<>(); - - private Jedis jedis; - private Pipeline pipeline; private int batchSize = DEFAULT_BATCH_SIZE; private int timeout = DEFAULT_TIMEOUT; + private RedisIngestionClient redisIngestionClient; - WriteDoFn(StoreProto.Store.RedisConfig redisConfig) { - this.host = redisConfig.getHost(); - this.port = redisConfig.getPort(); - long backoffMs = redisConfig.getInitialBackoffMs() > 0 ? redisConfig.getInitialBackoffMs() : 1; - this.backOffExecutor = new BackOffExecutor(redisConfig.getMaxRetries(), - Duration.millis(backoffMs)); + WriteDoFn(StoreProto.Store store) { + if (store.getType() == StoreProto.Store.StoreType.REDIS) + this.redisIngestionClient = new RedisStandaloneIngestionClient(store.getRedisConfig()); } public WriteDoFn withBatchSize(int batchSize) { @@ -223,57 +212,61 @@ public WriteDoFn withTimeout(int timeout) { @Setup public void setup() { - jedis = new Jedis(host, port, timeout); + this.redisIngestionClient.setup(); } @StartBundle public void startBundle() { + try { + redisIngestionClient.connect(); + } catch (RedisConnectionException e) { + log.error("Connection to redis cannot be established ", e); + } mutations.clear(); - pipeline = jedis.pipelined(); } private void executeBatch() throws Exception { - backOffExecutor.execute(new Retriable() { - @Override - public void execute() { - pipeline.multi(); - mutations.forEach(mutation -> { - writeRecord(mutation); - if (mutation.getExpiryMillis() != null && mutation.getExpiryMillis() > 0) { - pipeline.pexpire(mutation.getKey(), mutation.getExpiryMillis()); - } - }); - pipeline.exec(); - pipeline.sync(); - mutations.clear(); - } - - @Override - public Boolean isExceptionRetriable(Exception e) { - return e instanceof JedisConnectionException; - } - - @Override - public void cleanUpAfterFailure() { - try { - pipeline.close(); - } catch (IOException e) { - log.error(String.format("Error while closing pipeline: %s", e.getMessage())); - } - jedis = new Jedis(host, port, timeout); - pipeline = jedis.pipelined(); - } - }); + this.redisIngestionClient + .getBackOffExecutor() + .execute( + new Retriable() { + @Override + public void execute() throws ExecutionException, InterruptedException { + if (!redisIngestionClient.isConnected()) { + redisIngestionClient.connect(); + } + mutations.forEach( + mutation -> { + writeRecord(mutation); + if (mutation.getExpiryMillis() != null + && mutation.getExpiryMillis() > 0) { + redisIngestionClient.pexpire( + mutation.getKey(), mutation.getExpiryMillis()); + } + }); + redisIngestionClient.sync(); + mutations.clear(); + } + + @Override + public Boolean isExceptionRetriable(Exception e) { + return e instanceof RedisConnectionException; + } + + @Override + public void cleanUpAfterFailure() {} + }); } - private FailedElement toFailedElement(RedisMutation mutation, Exception exception, String jobName) { + private FailedElement toFailedElement( + RedisMutation mutation, Exception exception, String jobName) { return FailedElement.newBuilder() - .setJobName(jobName) - .setTransformName("RedisCustomIO") - .setPayload(mutation.getValue().toString()) - .setErrorMessage(exception.getMessage()) - .setStackTrace(ExceptionUtils.getStackTrace(exception)) - .build(); + .setJobName(jobName) + .setTransformName("RedisCustomIO") + .setPayload(Arrays.toString(mutation.getValue())) + .setErrorMessage(exception.getMessage()) + .setStackTrace(ExceptionUtils.getStackTrace(exception)) + .build(); } @ProcessElement @@ -284,30 +277,37 @@ public void processElement(ProcessContext context) { try { executeBatch(); } catch (Exception e) { - mutations.forEach(failedMutation -> { - FailedElement failedElement = toFailedElement( - failedMutation, e, context.getPipelineOptions().getJobName()); - context.output(failedElement); - }); + mutations.forEach( + failedMutation -> { + FailedElement failedElement = + toFailedElement(failedMutation, e, context.getPipelineOptions().getJobName()); + context.output(failedElement); + }); mutations.clear(); } } } - private Response writeRecord(RedisMutation mutation) { + private void writeRecord(RedisMutation mutation) { switch (mutation.getMethod()) { case APPEND: - return pipeline.append(mutation.getKey(), mutation.getValue()); + redisIngestionClient.append(mutation.getKey(), mutation.getValue()); + return; case SET: - return pipeline.set(mutation.getKey(), mutation.getValue()); + redisIngestionClient.set(mutation.getKey(), mutation.getValue()); + return; case LPUSH: - return pipeline.lpush(mutation.getKey(), mutation.getValue()); + redisIngestionClient.lpush(mutation.getKey(), mutation.getValue()); + return; case RPUSH: - return pipeline.rpush(mutation.getKey(), mutation.getValue()); + redisIngestionClient.rpush(mutation.getKey(), mutation.getValue()); + return; case SADD: - return pipeline.sadd(mutation.getKey(), mutation.getValue()); + redisIngestionClient.sadd(mutation.getKey(), mutation.getValue()); + return; case ZADD: - return pipeline.zadd(mutation.getKey(), mutation.getScore(), mutation.getValue()); + redisIngestionClient.zadd(mutation.getKey(), mutation.getScore(), mutation.getValue()); + return; default: throw new UnsupportedOperationException( String.format("Not implemented writing records for %s", mutation.getMethod())); @@ -315,16 +315,18 @@ private Response writeRecord(RedisMutation mutation) { } @FinishBundle - public void finishBundle(FinishBundleContext context) throws IOException, InterruptedException { - if(mutations.size() > 0) { + public void finishBundle(FinishBundleContext context) + throws IOException, InterruptedException { + if (mutations.size() > 0) { try { executeBatch(); } catch (Exception e) { - mutations.forEach(failedMutation -> { - FailedElement failedElement = toFailedElement( - failedMutation, e, context.getPipelineOptions().getJobName()); - context.output(failedElement, Instant.now(), GlobalWindow.INSTANCE); - }); + mutations.forEach( + failedMutation -> { + FailedElement failedElement = + toFailedElement(failedMutation, e, context.getPipelineOptions().getJobName()); + context.output(failedElement, Instant.now(), GlobalWindow.INSTANCE); + }); mutations.clear(); } } @@ -332,7 +334,7 @@ public void finishBundle(FinishBundleContext context) throws IOException, Interr @Teardown public void teardown() { - jedis.close(); + redisIngestionClient.shutdown(); } } } diff --git a/ingestion/src/main/java/feast/store/serving/redis/RedisIngestionClient.java b/ingestion/src/main/java/feast/store/serving/redis/RedisIngestionClient.java new file mode 100644 index 00000000000..d51eead53fb --- /dev/null +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisIngestionClient.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.store.serving.redis; + +import feast.retry.BackOffExecutor; +import java.io.Serializable; + +public interface RedisIngestionClient extends Serializable { + + void setup(); + + BackOffExecutor getBackOffExecutor(); + + void shutdown(); + + void connect(); + + boolean isConnected(); + + void sync(); + + void pexpire(byte[] key, Long expiryMillis); + + void append(byte[] key, byte[] value); + + void set(byte[] key, byte[] value); + + void lpush(byte[] key, byte[] value); + + void rpush(byte[] key, byte[] value); + + void sadd(byte[] key, byte[] value); + + void zadd(byte[] key, Long score, byte[] value); +} diff --git a/ingestion/src/main/java/feast/store/serving/redis/RedisStandaloneIngestionClient.java b/ingestion/src/main/java/feast/store/serving/redis/RedisStandaloneIngestionClient.java new file mode 100644 index 00000000000..d95ebbbf64a --- /dev/null +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisStandaloneIngestionClient.java @@ -0,0 +1,122 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.store.serving.redis; + +import com.google.common.collect.Lists; +import feast.core.StoreProto; +import feast.retry.BackOffExecutor; +import io.lettuce.core.*; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.async.RedisAsyncCommands; +import io.lettuce.core.codec.ByteArrayCodec; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.joda.time.Duration; + +public class RedisStandaloneIngestionClient implements RedisIngestionClient { + private final String host; + private final int port; + private final BackOffExecutor backOffExecutor; + private RedisClient redisclient; + private static final int DEFAULT_TIMEOUT = 2000; + private StatefulRedisConnection connection; + private RedisAsyncCommands commands; + private List futures = Lists.newArrayList(); + + public RedisStandaloneIngestionClient(StoreProto.Store.RedisConfig redisConfig) { + this.host = redisConfig.getHost(); + this.port = redisConfig.getPort(); + long backoffMs = redisConfig.getInitialBackoffMs() > 0 ? redisConfig.getInitialBackoffMs() : 1; + this.backOffExecutor = + new BackOffExecutor(redisConfig.getMaxRetries(), Duration.millis(backoffMs)); + } + + @Override + public void setup() { + this.redisclient = + RedisClient.create(new RedisURI(host, port, java.time.Duration.ofMillis(DEFAULT_TIMEOUT))); + } + + @Override + public BackOffExecutor getBackOffExecutor() { + return this.backOffExecutor; + } + + @Override + public void shutdown() { + this.redisclient.shutdown(); + } + + @Override + public void connect() { + if (!isConnected()) { + this.connection = this.redisclient.connect(new ByteArrayCodec()); + this.commands = connection.async(); + } + } + + @Override + public boolean isConnected() { + return connection != null; + } + + @Override + public void sync() { + // Wait for some time for futures to complete + // TODO: should this be configurable? + try { + LettuceFutures.awaitAll(60, TimeUnit.SECONDS, futures.toArray(new RedisFuture[0])); + } finally { + futures.clear(); + } + } + + @Override + public void pexpire(byte[] key, Long expiryMillis) { + commands.pexpire(key, expiryMillis); + } + + @Override + public void append(byte[] key, byte[] value) { + futures.add(commands.append(key, value)); + } + + @Override + public void set(byte[] key, byte[] value) { + futures.add(commands.set(key, value)); + } + + @Override + public void lpush(byte[] key, byte[] value) { + futures.add(commands.lpush(key, value)); + } + + @Override + public void rpush(byte[] key, byte[] value) { + futures.add(commands.rpush(key, value)); + } + + @Override + public void sadd(byte[] key, byte[] value) { + futures.add(commands.sadd(key, value)); + } + + @Override + public void zadd(byte[] key, Long score, byte[] value) { + futures.add(commands.zadd(key, score, value)); + } +} diff --git a/ingestion/src/main/proto/feast b/ingestion/src/main/proto/feast deleted file mode 120000 index d520da9126b..00000000000 --- a/ingestion/src/main/proto/feast +++ /dev/null @@ -1 +0,0 @@ -../../../../protos/feast \ No newline at end of file diff --git a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto b/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto deleted file mode 100644 index cb64dd715f6..00000000000 --- a/ingestion/src/main/proto/feast_ingestion/types/CoalesceAccum.proto +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -import "google/protobuf/timestamp.proto"; -import "feast/types/Field.proto"; - -option java_package = "feast_ingestion.types"; -option java_outer_classname = "CoalesceAccumProto"; - -// Accumlator for merging feature rows. -message CoalesceAccum { - string entityKey = 1; - google.protobuf.Timestamp eventTimestamp = 3; - string entityName = 4; - - map features = 6; - // map of features to their counter values when they were last added to accumulator - map featureMarks = 7; - int64 counter = 8; -} \ No newline at end of file diff --git a/ingestion/src/main/proto/third_party b/ingestion/src/main/proto/third_party deleted file mode 120000 index 363d20598e6..00000000000 --- a/ingestion/src/main/proto/third_party +++ /dev/null @@ -1 +0,0 @@ -../../../../protos/third_party \ No newline at end of file diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java index 290b38dabee..0b000df0f59 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java @@ -30,13 +30,20 @@ import feast.core.StoreProto.Store.RedisConfig; import feast.core.StoreProto.Store.StoreType; import feast.core.StoreProto.Store.Subscription; +import feast.ingestion.options.BZip2Compressor; import feast.ingestion.options.ImportOptions; import feast.storage.RedisProto.RedisKey; import feast.test.TestUtil; import feast.test.TestUtil.LocalKafka; import feast.test.TestUtil.LocalRedis; import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto; import feast.types.ValueProto.ValueType.Enum; +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.sync.RedisCommands; +import io.lettuce.core.codec.ByteArrayCodec; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -44,6 +51,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.PipelineResult.State; @@ -56,7 +64,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; public class ImportJobTest { @@ -162,12 +169,15 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow() .build(); ImportOptions options = PipelineOptionsFactory.create().as(ImportOptions.class); - options.setFeatureSetJson( - Collections.singletonList( - JsonFormat.printer().omittingInsignificantWhitespace().print(featureSet.getSpec()))); - options.setStoreJson( - Collections.singletonList( - JsonFormat.printer().omittingInsignificantWhitespace().print(redis))); + BZip2Compressor compressor = + new BZip2Compressor<>( + option -> { + JsonFormat.Printer printer = + JsonFormat.printer().omittingInsignificantWhitespace().printingEnumsAsInts(); + return printer.print(option).getBytes(); + }); + options.setFeatureSetJson(compressor.compress(spec)); + options.setStoreJson(Collections.singletonList(JsonFormat.printer().print(redis))); options.setProject(""); options.setBlockOnRun(false); @@ -181,6 +191,23 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow() FeatureRow randomRow = TestUtil.createRandomFeatureRow(featureSet); RedisKey redisKey = TestUtil.createRedisKey(featureSet, randomRow); input.add(randomRow); + List fields = + randomRow.getFieldsList().stream() + .filter( + field -> + spec.getFeaturesList().stream() + .map(FeatureSpec::getName) + .collect(Collectors.toList()) + .contains(field.getName())) + .map(field -> field.toBuilder().clearName().build()) + .collect(Collectors.toList()); + randomRow = + randomRow + .toBuilder() + .clearFields() + .addAllFields(fields) + .clearFeatureSet() + .build(); expected.put(redisKey, randomRow); }); @@ -202,21 +229,24 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow() Duration.standardSeconds(IMPORT_JOB_CHECK_INTERVAL_DURATION_SEC)); LOGGER.info("Validating the actual values written to Redis ..."); - Jedis jedis = new Jedis(REDIS_HOST, REDIS_PORT); + RedisClient redisClient = + RedisClient.create(new RedisURI(REDIS_HOST, REDIS_PORT, java.time.Duration.ofMillis(2000))); + StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); + RedisCommands sync = connection.sync(); expected.forEach( (key, expectedValue) -> { // Ensure ingested key exists. - byte[] actualByteValue = jedis.get(key.toByteArray()); + byte[] actualByteValue = sync.get(key.toByteArray()); if (actualByteValue == null) { LOGGER.error("Key not found in Redis: " + key); LOGGER.info("Redis INFO:"); - LOGGER.info(jedis.info()); - String randomKey = jedis.randomKey(); + LOGGER.info(sync.info()); + byte[] randomKey = sync.randomkey(); if (randomKey != null) { LOGGER.info("Sample random key, value (for debugging purpose):"); LOGGER.info("Key: " + randomKey); - LOGGER.info("Value: " + jedis.get(randomKey)); + LOGGER.info("Value: " + sync.get(randomKey)); } Assert.fail("Missing key in Redis."); } @@ -235,5 +265,6 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow() // Ensure the retrieved FeatureRow is equal to the ingested FeatureRow. Assert.assertEquals(expectedValue, actualValue); }); + redisClient.shutdown(); } } diff --git a/ingestion/src/test/java/feast/ingestion/options/BZip2CompressorTest.java b/ingestion/src/test/java/feast/ingestion/options/BZip2CompressorTest.java new file mode 100644 index 00000000000..cd03b18c793 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/options/BZip2CompressorTest.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.options; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.junit.Assert; +import org.junit.Test; + +public class BZip2CompressorTest { + + @Test + public void shouldHavBZip2CompatibleOutput() throws IOException { + BZip2Compressor compressor = new BZip2Compressor<>(String::getBytes); + String origString = "somestring"; + try (ByteArrayInputStream inputStream = + new ByteArrayInputStream(compressor.compress(origString)); + BZip2CompressorInputStream bzip2Input = new BZip2CompressorInputStream(inputStream); + BufferedReader reader = new BufferedReader(new InputStreamReader(bzip2Input))) { + Assert.assertEquals(origString, reader.readLine()); + } + } +} diff --git a/ingestion/src/test/java/feast/ingestion/options/BZip2DecompressorTest.java b/ingestion/src/test/java/feast/ingestion/options/BZip2DecompressorTest.java new file mode 100644 index 00000000000..fe7cc789d86 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/options/BZip2DecompressorTest.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.options; + +import static org.junit.Assert.*; + +import java.io.*; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.junit.Test; + +public class BZip2DecompressorTest { + + @Test + public void shouldDecompressBZip2Stream() throws IOException { + BZip2Decompressor decompressor = + new BZip2Decompressor<>( + inputStream -> { + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); + String output = reader.readLine(); + reader.close(); + return output; + }); + + String originalString = "abc"; + ByteArrayOutputStream compressedStream = new ByteArrayOutputStream(); + try (BZip2CompressorOutputStream bzip2Output = + new BZip2CompressorOutputStream(compressedStream)) { + bzip2Output.write(originalString.getBytes()); + } + + String decompressedString = decompressor.decompress(compressedStream.toByteArray()); + assertEquals(originalString, decompressedString); + } +} diff --git a/ingestion/src/test/java/feast/ingestion/options/StringListStreamConverterTest.java b/ingestion/src/test/java/feast/ingestion/options/StringListStreamConverterTest.java new file mode 100644 index 00000000000..5ce9f054bc9 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/options/StringListStreamConverterTest.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.options; + +import static org.junit.Assert.*; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import org.junit.Test; + +public class StringListStreamConverterTest { + + @Test + public void shouldReadStreamAsNewlineSeparatedStrings() throws IOException { + StringListStreamConverter converter = new StringListStreamConverter(); + String originalString = "abc\ndef"; + InputStream stringStream = new ByteArrayInputStream(originalString.getBytes()); + assertEquals(Arrays.asList("abc", "def"), converter.readStream(stringStream)); + } +} diff --git a/ingestion/src/test/java/feast/ingestion/transform/CassandraWriteToStoreIT.java b/ingestion/src/test/java/feast/ingestion/transform/CassandraWriteToStoreIT.java new file mode 100644 index 00000000000..78d31cbd023 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/transform/CassandraWriteToStoreIT.java @@ -0,0 +1,253 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.transform; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.datastax.driver.core.ResultSet; +import com.datastax.driver.core.Row; +import com.google.protobuf.InvalidProtocolBufferException; +import feast.core.FeatureSetProto.FeatureSet; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.StoreProto.Store; +import feast.core.StoreProto.Store.CassandraConfig; +import feast.core.StoreProto.Store.StoreType; +import feast.test.TestUtil; +import feast.test.TestUtil.LocalCassandra; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import feast.types.ValueProto.Value; +import feast.types.ValueProto.ValueType.Enum; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.PCollection; +import org.apache.thrift.transport.TTransportException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +public class CassandraWriteToStoreIT implements Serializable { + private FeatureSetSpec featureSetSpec; + private FeatureRow row; + + class FakeCassandraWriteToStore extends WriteToStore { + + private FeatureSetSpec featureSetSpec; + + FakeCassandraWriteToStore(FeatureSetSpec featureSetSpec) { + this.featureSetSpec = featureSetSpec; + } + + @Override + public Store getStore() { + return Store.newBuilder() + .setType(StoreType.CASSANDRA) + .setName("SERVING") + .setCassandraConfig(getCassandraConfig()) + .build(); + } + + @Override + public Map getFeatureSets() { + return new HashMap() { + { + put( + featureSetSpec.getName() + ":" + featureSetSpec.getVersion(), + FeatureSet.newBuilder().setSpec(featureSetSpec).build()); + } + }; + } + } + + private static CassandraConfig getCassandraConfig() { + return CassandraConfig.newBuilder() + .setBootstrapHosts(LocalCassandra.getHost()) + .setPort(LocalCassandra.getPort()) + .setTableName("feature_store") + .setKeyspace("test") + .putAllReplicationOptions( + new HashMap() { + { + put("class", "SimpleStrategy"); + put("replication_factor", "1"); + } + }) + .build(); + } + + @BeforeClass + public static void startServer() throws InterruptedException, IOException, TTransportException { + LocalCassandra.start(); + LocalCassandra.createKeyspaceAndTable(getCassandraConfig()); + } + + @Before + public void setUp() { + featureSetSpec = + TestUtil.createFeatureSetSpec( + "fs", + "test_project", + 1, + 10, + new HashMap() { + { + put("entity1", Enum.INT64); + put("entity2", Enum.STRING); + } + }, + new HashMap() { + { + put("feature1", Enum.INT64); + put("feature2", Enum.INT64); + } + }); + row = + TestUtil.createFeatureRow( + featureSetSpec, + 100, + new HashMap() { + { + put("entity1", TestUtil.intValue(1)); + put("entity2", TestUtil.strValue("a")); + put("feature1", TestUtil.intValue(1)); + put("feature2", TestUtil.intValue(2)); + } + }); + } + + @Rule public transient TestPipeline testPipeline = TestPipeline.create(); + + @AfterClass + public static void cleanUp() { + LocalCassandra.stop(); + } + + @Test + public void testWriteCassandra_happyPath() throws InvalidProtocolBufferException { + PCollection input = testPipeline.apply(Create.of(row)); + + input.apply(new FakeCassandraWriteToStore(featureSetSpec)); + + testPipeline.run(); + + ResultSet resultSet = LocalCassandra.getSession().execute("SELECT * FROM test.feature_store"); + List actualResults = getResults(resultSet); + + List expectedFields = + Arrays.asList( + Field.newBuilder().setName("feature1").setValue(TestUtil.intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(TestUtil.intValue(2)).build()); + + assertTrue(actualResults.containsAll(expectedFields)); + assertEquals(expectedFields.size(), actualResults.size()); + } + + @Test(timeout = 30000) + public void testWriteCassandra_shouldNotRetrieveExpiredValues() + throws InvalidProtocolBufferException { + // Set max age to 1 second + FeatureSetSpec featureSetSpec = + TestUtil.createFeatureSetSpec( + "fs", + "test_project", + 1, + 1, + new HashMap() { + { + put("entity1", Enum.INT64); + put("entity2", Enum.STRING); + } + }, + new HashMap() { + { + put("feature1", Enum.INT64); + put("feature2", Enum.INT64); + } + }); + + PCollection input = testPipeline.apply(Create.of(row)); + + input.apply(new FakeCassandraWriteToStore(featureSetSpec)); + + testPipeline.run(); + + while (true) { + ResultSet resultSet = + LocalCassandra.getSession() + .execute("SELECT feature, value, ttl(value) as expiry FROM test.feature_store"); + List results = getResults(resultSet); + if (results.isEmpty()) break; + } + } + + @Test + public void testWriteCassandra_shouldNotOverrideNewerValues() + throws InvalidProtocolBufferException { + FeatureRow olderRow = + TestUtil.createFeatureRow( + featureSetSpec, + 10, + new HashMap() { + { + put("entity1", TestUtil.intValue(1)); + put("entity2", TestUtil.strValue("a")); + put("feature1", TestUtil.intValue(3)); + put("feature2", TestUtil.intValue(4)); + } + }); + + PCollection input = testPipeline.apply(Create.of(row, olderRow)); + + input.apply(new FakeCassandraWriteToStore(featureSetSpec)); + + testPipeline.run(); + + ResultSet resultSet = LocalCassandra.getSession().execute("SELECT * FROM test.feature_store"); + List actualResults = getResults(resultSet); + + List expectedFields = + Arrays.asList( + Field.newBuilder().setName("feature1").setValue(TestUtil.intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(TestUtil.intValue(2)).build()); + + assertTrue(actualResults.containsAll(expectedFields)); + assertEquals(expectedFields.size(), actualResults.size()); + } + + private List getResults(ResultSet resultSet) throws InvalidProtocolBufferException { + List results = new ArrayList<>(); + while (!resultSet.isExhausted()) { + Row row = resultSet.one(); + results.add( + Field.newBuilder() + .setName(row.getString("feature")) + .setValue(Value.parseFrom(row.getBytes("value"))) + .build()); + } + return results; + } +} diff --git a/ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java b/ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java index aca39563877..5c9860ed97f 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/ValidateFeatureRowsTest.java @@ -180,12 +180,14 @@ public void shouldExcludeUnregisteredFields() { FeatureRow randomRow = TestUtil.createRandomFeatureRow(fs1); expected.add(randomRow); - input.add(randomRow.toBuilder() - .addFields(Field.newBuilder() - .setName("extra") - .setValue(Value.newBuilder().setStringVal("hello"))) - .build() - ); + input.add( + randomRow + .toBuilder() + .addFields( + Field.newBuilder() + .setName("extra") + .setValue(Value.newBuilder().setStringVal("hello"))) + .build()); PCollectionTuple output = p.apply(Create.of(input)) diff --git a/ingestion/src/test/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFnTest.java b/ingestion/src/test/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFnTest.java new file mode 100644 index 00000000000..d2b0275c6fe --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFnTest.java @@ -0,0 +1,318 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.transform.metrics; + +import static org.junit.Assert.fail; + +import com.google.protobuf.ByteString; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FeatureRowProto.FeatureRow.Builder; +import feast.types.FieldProto.Field; +import feast.types.ValueProto.BoolList; +import feast.types.ValueProto.BytesList; +import feast.types.ValueProto.DoubleList; +import feast.types.ValueProto.FloatList; +import feast.types.ValueProto.Int32List; +import feast.types.ValueProto.Int64List; +import feast.types.ValueProto.StringList; +import feast.types.ValueProto.Value; +import java.io.BufferedReader; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.SocketException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.junit.Rule; +import org.junit.Test; + +public class WriteFeatureValueMetricsDoFnTest { + + @Rule public final transient TestPipeline pipeline = TestPipeline.create(); + private static final int STATSD_SERVER_PORT = 17254; + private final DummyStatsDServer statsDServer = new DummyStatsDServer(STATSD_SERVER_PORT); + + @Test + public void shouldSendCorrectStatsDMetrics() throws IOException, InterruptedException { + PipelineOptions pipelineOptions = PipelineOptionsFactory.create(); + pipelineOptions.setJobName("job"); + + Map> input = + readTestInput("feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input"); + List expectedLines = + readTestOutput("feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output"); + + pipeline + .apply(Create.of(input)) + .apply( + ParDo.of( + WriteFeatureValueMetricsDoFn.newBuilder() + .setStatsdHost("localhost") + .setStatsdPort(STATSD_SERVER_PORT) + .setStoreName("store") + .build())); + pipeline.run(pipelineOptions).waitUntilFinish(); + // Wait until StatsD has finished processed all messages, 3 sec is a reasonable duration + // based on empirical testing. + Thread.sleep(3000); + + List actualLines = statsDServer.messagesReceived(); + for (String expected : expectedLines) { + boolean matched = false; + for (String actual : actualLines) { + if (actual.equals(expected)) { + matched = true; + break; + } + } + if (!matched) { + System.out.println("Print actual metrics output for debugging:"); + for (String line : actualLines) { + System.out.println(line); + } + fail(String.format("Expected StatsD metric not found:\n%s", expected)); + } + } + } + + // Test utility method to read expected StatsD metrics output from a text file. + @SuppressWarnings("SameParameterValue") + private List readTestOutput(String path) throws IOException { + URL url = Thread.currentThread().getContextClassLoader().getResource(path); + if (url == null) { + throw new IllegalArgumentException( + "cannot read test data, path contains null url. Path: " + path); + } + List lines = new ArrayList<>(); + try (BufferedReader reader = Files.newBufferedReader(Paths.get(url.getPath()))) { + String line = reader.readLine(); + while (line != null) { + if (line.trim().length() > 1) { + lines.add(line); + } + line = reader.readLine(); + } + } + return lines; + } + + // Test utility method to create test feature row data from a text file. + @SuppressWarnings("SameParameterValue") + private Map> readTestInput(String path) throws IOException { + Map> data = new HashMap<>(); + URL url = Thread.currentThread().getContextClassLoader().getResource(path); + if (url == null) { + throw new IllegalArgumentException( + "cannot read test data, path contains null url. Path: " + path); + } + List lines = new ArrayList<>(); + try (BufferedReader reader = Files.newBufferedReader(Paths.get(url.getPath()))) { + String line = reader.readLine(); + while (line != null) { + lines.add(line); + line = reader.readLine(); + } + } + List colNames = new ArrayList<>(); + for (String line : lines) { + if (line.strip().length() < 1) { + continue; + } + String[] splits = line.split(","); + colNames.addAll(Arrays.asList(splits)); + + if (line.startsWith("featuresetref")) { + // Header line + colNames.addAll(Arrays.asList(splits).subList(1, splits.length)); + continue; + } + + Builder featureRowBuilder = FeatureRow.newBuilder(); + for (int i = 0; i < splits.length; i++) { + String colVal = splits[i].strip(); + if (i == 0) { + featureRowBuilder.setFeatureSet(colVal); + continue; + } + String colName = colNames.get(i); + Field.Builder fieldBuilder = Field.newBuilder().setName(colName); + if (!colVal.isEmpty()) { + switch (colName) { + case "int32": + fieldBuilder.setValue(Value.newBuilder().setInt32Val((Integer.parseInt(colVal)))); + break; + case "int64": + fieldBuilder.setValue(Value.newBuilder().setInt64Val((Long.parseLong(colVal)))); + break; + case "double": + fieldBuilder.setValue(Value.newBuilder().setDoubleVal((Double.parseDouble(colVal)))); + break; + case "float": + fieldBuilder.setValue(Value.newBuilder().setFloatVal((Float.parseFloat(colVal)))); + break; + case "bool": + fieldBuilder.setValue(Value.newBuilder().setBoolVal((Boolean.parseBoolean(colVal)))); + break; + case "int32list": + List int32List = new ArrayList<>(); + for (String val : colVal.split("\\|")) { + int32List.add(Integer.parseInt(val)); + } + fieldBuilder.setValue( + Value.newBuilder().setInt32ListVal(Int32List.newBuilder().addAllVal(int32List))); + break; + case "int64list": + List int64list = new ArrayList<>(); + for (String val : colVal.split("\\|")) { + int64list.add(Long.parseLong(val)); + } + fieldBuilder.setValue( + Value.newBuilder().setInt64ListVal(Int64List.newBuilder().addAllVal(int64list))); + break; + case "doublelist": + List doubleList = new ArrayList<>(); + for (String val : colVal.split("\\|")) { + doubleList.add(Double.parseDouble(val)); + } + fieldBuilder.setValue( + Value.newBuilder() + .setDoubleListVal(DoubleList.newBuilder().addAllVal(doubleList))); + break; + case "floatlist": + List floatList = new ArrayList<>(); + for (String val : colVal.split("\\|")) { + floatList.add(Float.parseFloat(val)); + } + fieldBuilder.setValue( + Value.newBuilder().setFloatListVal(FloatList.newBuilder().addAllVal(floatList))); + break; + case "boollist": + List boolList = new ArrayList<>(); + for (String val : colVal.split("\\|")) { + boolList.add(Boolean.parseBoolean(val)); + } + fieldBuilder.setValue( + Value.newBuilder().setBoolListVal(BoolList.newBuilder().addAllVal(boolList))); + break; + case "bytes": + fieldBuilder.setValue( + Value.newBuilder().setBytesVal(ByteString.copyFromUtf8("Dummy"))); + break; + case "byteslist": + fieldBuilder.setValue( + Value.newBuilder().setBytesListVal(BytesList.getDefaultInstance())); + break; + case "string": + fieldBuilder.setValue(Value.newBuilder().setStringVal("Dummy")); + break; + case "stringlist": + fieldBuilder.setValue( + Value.newBuilder().setStringListVal(StringList.getDefaultInstance())); + break; + } + } + featureRowBuilder.addFields(fieldBuilder); + } + + if (!data.containsKey(featureRowBuilder.getFeatureSet())) { + data.put(featureRowBuilder.getFeatureSet(), new ArrayList<>()); + } + List featureRowsByFeatureSetRef = data.get(featureRowBuilder.getFeatureSet()); + featureRowsByFeatureSetRef.add(featureRowBuilder.build()); + } + + // Convert List to Iterable to match the function signature in + // WriteFeatureValueMetricsDoFn + Map> dataWithIterable = new HashMap<>(); + for (Entry> entrySet : data.entrySet()) { + String key = entrySet.getKey(); + Iterable value = entrySet.getValue(); + dataWithIterable.put(key, value); + } + return dataWithIterable; + } + + // Modified version of + // https://github.com/tim-group/java-statsd-client/blob/master/src/test/java/com/timgroup/statsd/NonBlockingStatsDClientTest.java + @SuppressWarnings("CatchMayIgnoreException") + private static final class DummyStatsDServer { + + private final List messagesReceived = new ArrayList(); + private final DatagramSocket server; + + public DummyStatsDServer(int port) { + try { + server = new DatagramSocket(port); + } catch (SocketException e) { + throw new IllegalStateException(e); + } + new Thread( + () -> { + try { + while (true) { + final DatagramPacket packet = new DatagramPacket(new byte[65535], 65535); + server.receive(packet); + messagesReceived.add( + new String(packet.getData(), StandardCharsets.UTF_8).trim() + "\n"); + // The sleep duration here is shorter than that used in waitForMessage() at + // 50ms. + // Otherwise sometimes some messages seem to be lost, leading to flaky tests. + Thread.sleep(15L); + } + + } catch (Exception e) { + } + }) + .start(); + } + + public void stop() { + server.close(); + } + + public void waitForMessage() { + while (messagesReceived.isEmpty()) { + try { + Thread.sleep(50L); + } catch (InterruptedException e) { + } + } + } + + public List messagesReceived() { + List out = new ArrayList<>(); + for (String msg : messagesReceived) { + String[] lines = msg.split("\n"); + out.addAll(Arrays.asList(lines)); + } + return out; + } + } +} diff --git a/ingestion/src/test/java/feast/ingestion/utils/CassandraStoreUtilIT.java b/ingestion/src/test/java/feast/ingestion/utils/CassandraStoreUtilIT.java new file mode 100644 index 00000000000..8c6874ecac7 --- /dev/null +++ b/ingestion/src/test/java/feast/ingestion/utils/CassandraStoreUtilIT.java @@ -0,0 +1,167 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.util; + +import com.datastax.driver.core.DataType; +import com.datastax.driver.core.TableMetadata; +import com.datastax.driver.core.schemabuilder.Create; +import com.datastax.driver.core.schemabuilder.SchemaBuilder; +import feast.core.StoreProto.Store.CassandraConfig; +import feast.ingestion.utils.StoreUtil; +import feast.store.serving.cassandra.CassandraMutation; +import feast.test.TestUtil.LocalCassandra; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.thrift.transport.TTransportException; +import org.junit.After; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class CassandraStoreUtilIT { + + @BeforeClass + public static void startServer() throws InterruptedException, IOException, TTransportException { + LocalCassandra.start(); + } + + @After + public void teardown() { + LocalCassandra.stop(); + } + + @Test + public void setupCassandra_shouldCreateKeyspaceAndTable() { + CassandraConfig config = + CassandraConfig.newBuilder() + .setBootstrapHosts(LocalCassandra.getHost()) + .setPort(LocalCassandra.getPort()) + .setKeyspace("test") + .setTableName("feature_store") + .putAllReplicationOptions( + new HashMap() { + { + put("class", "NetworkTopologyStrategy"); + put("dc1", "2"); + put("dc2", "3"); + } + }) + .build(); + StoreUtil.setupCassandra(config); + + Map actualReplication = + LocalCassandra.getCluster().getMetadata().getKeyspace("test").getReplication(); + Map expectedReplication = + new HashMap() { + { + put("class", "org.apache.cassandra.locator.NetworkTopologyStrategy"); + put("dc1", "2"); + put("dc2", "3"); + } + }; + TableMetadata tableMetadata = + LocalCassandra.getCluster().getMetadata().getKeyspace("test").getTable("feature_store"); + + Assert.assertEquals(expectedReplication, actualReplication); + Assert.assertNotNull(tableMetadata); + } + + @Test + public void setupCassandra_shouldBeIdempotent_whenTableAlreadyExistsAndSchemaMatches() { + CassandraConfig config = + CassandraConfig.newBuilder() + .setBootstrapHosts(LocalCassandra.getHost()) + .setPort(LocalCassandra.getPort()) + .setKeyspace("test") + .setTableName("feature_store") + .putAllReplicationOptions( + new HashMap() { + { + put("class", "SimpleStrategy"); + put("replication_factor", "2"); + } + }) + .build(); + + LocalCassandra.createKeyspaceAndTable(config); + + // Check table is created + Assert.assertNotNull( + LocalCassandra.getCluster().getMetadata().getKeyspace("test").getTable("feature_store")); + + StoreUtil.setupCassandra(config); + + Assert.assertNotNull( + LocalCassandra.getCluster().getMetadata().getKeyspace("test").getTable("feature_store")); + } + + @Test(expected = RuntimeException.class) + public void setupCassandra_shouldThrowException_whenTableNameDoesNotMatchObjectMapper() { + CassandraConfig config = + CassandraConfig.newBuilder() + .setBootstrapHosts(LocalCassandra.getHost()) + .setPort(LocalCassandra.getPort()) + .setKeyspace("test") + .setTableName("test_data_store") + .putAllReplicationOptions( + new HashMap() { + { + put("class", "NetworkTopologyStrategy"); + put("dc1", "2"); + put("dc2", "3"); + } + }) + .build(); + StoreUtil.setupCassandra(config); + } + + @Test(expected = RuntimeException.class) + public void setupCassandra_shouldThrowException_whenTableSchemaDoesNotMatchObjectMapper() { + LocalCassandra.getSession() + .execute( + "CREATE KEYSPACE test " + + "WITH REPLICATION = {" + + "'class': 'SimpleStrategy', 'replication_factor': 2 }"); + + Create createTable = + SchemaBuilder.createTable("test", "feature_store") + .ifNotExists() + .addPartitionKey(CassandraMutation.ENTITIES, DataType.text()) + .addClusteringColumn( + "featureName", DataType.text()) // Column name does not match in CassandraMutation + .addColumn(CassandraMutation.VALUE, DataType.blob()); + LocalCassandra.getSession().execute(createTable); + + CassandraConfig config = + CassandraConfig.newBuilder() + .setBootstrapHosts(LocalCassandra.getHost()) + .setPort(LocalCassandra.getPort()) + .setKeyspace("test") + .setTableName("feature_store") + .putAllReplicationOptions( + new HashMap() { + { + put("class", "SimpleStrategy"); + put("replication_factor", "2"); + } + }) + .build(); + + StoreUtil.setupCassandra(config); + } +} diff --git a/ingestion/src/test/java/feast/ingestion/util/DateUtilTest.java b/ingestion/src/test/java/feast/ingestion/utils/DateUtilTest.java similarity index 92% rename from ingestion/src/test/java/feast/ingestion/util/DateUtilTest.java rename to ingestion/src/test/java/feast/ingestion/utils/DateUtilTest.java index 71d4e67beaa..151d501a596 100644 --- a/ingestion/src/test/java/feast/ingestion/util/DateUtilTest.java +++ b/ingestion/src/test/java/feast/ingestion/utils/DateUtilTest.java @@ -14,15 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.ingestion.util; +package feast.ingestion.utils; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.*; import com.google.protobuf.Timestamp; -import feast.ingestion.utils.DateUtil; import junit.framework.TestCase; import org.joda.time.DateTime; diff --git a/ingestion/src/test/java/feast/ingestion/util/JsonUtilTest.java b/ingestion/src/test/java/feast/ingestion/utils/JsonUtilTest.java similarity index 95% rename from ingestion/src/test/java/feast/ingestion/util/JsonUtilTest.java rename to ingestion/src/test/java/feast/ingestion/utils/JsonUtilTest.java index 02af4d819f9..62c74dfc345 100644 --- a/ingestion/src/test/java/feast/ingestion/util/JsonUtilTest.java +++ b/ingestion/src/test/java/feast/ingestion/utils/JsonUtilTest.java @@ -14,12 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.ingestion.util; +package feast.ingestion.utils; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertThat; -import feast.ingestion.utils.JsonUtil; import java.util.Collections; import java.util.HashMap; import java.util.Map; diff --git a/ingestion/src/test/java/feast/ingestion/util/StoreUtilTest.java b/ingestion/src/test/java/feast/ingestion/utils/StoreUtilTest.java similarity index 91% rename from ingestion/src/test/java/feast/ingestion/util/StoreUtilTest.java rename to ingestion/src/test/java/feast/ingestion/utils/StoreUtilTest.java index 4e2297e405d..82988121bc8 100644 --- a/ingestion/src/test/java/feast/ingestion/util/StoreUtilTest.java +++ b/ingestion/src/test/java/feast/ingestion/utils/StoreUtilTest.java @@ -14,22 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.ingestion.util; +package feast.ingestion.utils; -import static feast.types.ValueProto.ValueType.Enum.BOOL; -import static feast.types.ValueProto.ValueType.Enum.BOOL_LIST; -import static feast.types.ValueProto.ValueType.Enum.BYTES; -import static feast.types.ValueProto.ValueType.Enum.BYTES_LIST; -import static feast.types.ValueProto.ValueType.Enum.DOUBLE; -import static feast.types.ValueProto.ValueType.Enum.DOUBLE_LIST; -import static feast.types.ValueProto.ValueType.Enum.FLOAT; -import static feast.types.ValueProto.ValueType.Enum.FLOAT_LIST; -import static feast.types.ValueProto.ValueType.Enum.INT32; -import static feast.types.ValueProto.ValueType.Enum.INT32_LIST; -import static feast.types.ValueProto.ValueType.Enum.INT64; -import static feast.types.ValueProto.ValueType.Enum.INT64_LIST; -import static feast.types.ValueProto.ValueType.Enum.STRING; -import static feast.types.ValueProto.ValueType.Enum.STRING_LIST; +import static feast.types.ValueProto.ValueType.Enum.*; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.Field; @@ -40,7 +27,6 @@ import feast.core.FeatureSetProto.FeatureSet; import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.FeatureSetProto.FeatureSpec; -import feast.ingestion.utils.StoreUtil; import java.util.Arrays; import org.junit.Assert; import org.junit.Test; diff --git a/ingestion/src/test/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFnTest.java b/ingestion/src/test/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFnTest.java new file mode 100644 index 00000000000..75246260f1a --- /dev/null +++ b/ingestion/src/test/java/feast/store/serving/cassandra/FeatureRowToCassandraMutationDoFnTest.java @@ -0,0 +1,244 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.store.serving.cassandra; + +import com.google.protobuf.Duration; +import feast.core.FeatureSetProto.FeatureSet; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.test.TestUtil; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.ValueProto.Value; +import feast.types.ValueProto.ValueType.Enum; +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.HashMap; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; + +public class FeatureRowToCassandraMutationDoFnTest implements Serializable { + + @Rule public transient TestPipeline testPipeline = TestPipeline.create(); + + @Test + public void processElement_shouldCreateCassandraMutation_givenFeatureRow() { + FeatureSetSpec featureSetSpec = + TestUtil.createFeatureSetSpec( + "fs", + "test_project", + 1, + 10, + new HashMap() { + { + put("entity1", Enum.INT64); + } + }, + new HashMap() { + { + put("feature1", Enum.STRING); + } + }); + FeatureRow featureRow = + TestUtil.createFeatureRow( + featureSetSpec, + 10, + new HashMap() { + { + put("entity1", TestUtil.intValue(1)); + put("feature1", TestUtil.strValue("a")); + } + }); + + PCollection input = testPipeline.apply(Create.of(featureRow)); + + PCollection output = + input.apply( + ParDo.of( + new FeatureRowToCassandraMutationDoFn( + new HashMap() { + { + put( + featureSetSpec.getProject() + + "/" + + featureSetSpec.getName() + + ":" + + featureSetSpec.getVersion(), + FeatureSet.newBuilder().setSpec(featureSetSpec).build()); + } + }, + Duration.newBuilder().setSeconds(0).build(), + false))); + + CassandraMutation[] expected = + new CassandraMutation[] { + new CassandraMutation( + "test_project/fs:1:entity1=1", + "feature1", + ByteBuffer.wrap(TestUtil.strValue("a").toByteArray()), + 10000000, + 10) + }; + + PAssert.that(output).containsInAnyOrder(expected); + + testPipeline.run(); + } + + @Test + public void + processElement_shouldCreateCassandraMutations_givenFeatureRowWithMultipleEntitiesAndFeatures() { + FeatureSetSpec featureSetSpec = + TestUtil.createFeatureSetSpec( + "fs", + "test_project", + 1, + 10, + new HashMap() { + { + put("entity1", Enum.INT64); + put("entity2", Enum.STRING); + } + }, + new HashMap() { + { + put("feature1", Enum.STRING); + put("feature2", Enum.INT64); + } + }); + FeatureRow featureRow = + TestUtil.createFeatureRow( + featureSetSpec, + 10, + new HashMap() { + { + put("entity1", TestUtil.intValue(1)); + put("entity2", TestUtil.strValue("b")); + put("feature1", TestUtil.strValue("a")); + put("feature2", TestUtil.intValue(2)); + } + }); + + PCollection input = testPipeline.apply(Create.of(featureRow)); + + PCollection output = + input.apply( + ParDo.of( + new FeatureRowToCassandraMutationDoFn( + new HashMap() { + { + put( + featureSetSpec.getProject() + + "/" + + featureSetSpec.getName() + + ":" + + featureSetSpec.getVersion(), + FeatureSet.newBuilder().setSpec(featureSetSpec).build()); + } + }, + Duration.newBuilder().setSeconds(0).build(), + false))); + + CassandraMutation[] expected = + new CassandraMutation[] { + new CassandraMutation( + "test_project/fs:1:entity1=1|entity2=b", + "feature1", + ByteBuffer.wrap(TestUtil.strValue("a").toByteArray()), + 10000000, + 10), + new CassandraMutation( + "test_project/fs:1:entity1=1|entity2=b", + "feature2", + ByteBuffer.wrap(TestUtil.intValue(2).toByteArray()), + 10000000, + 10) + }; + + PAssert.that(output).containsInAnyOrder(expected); + + testPipeline.run(); + } + + @Test + public void processElement_shouldUseDefaultMaxAge_whenMissingMaxAge() { + Duration defaultTtl = Duration.newBuilder().setSeconds(500).build(); + FeatureSetSpec featureSetSpec = + TestUtil.createFeatureSetSpec( + "fs", + "test_project", + 1, + 0, + new HashMap() { + { + put("entity1", Enum.INT64); + } + }, + new HashMap() { + { + put("feature1", Enum.STRING); + } + }); + FeatureRow featureRow = + TestUtil.createFeatureRow( + featureSetSpec, + 10, + new HashMap() { + { + put("entity1", TestUtil.intValue(1)); + put("feature1", TestUtil.strValue("a")); + } + }); + + PCollection input = testPipeline.apply(Create.of(featureRow)); + + PCollection output = + input.apply( + ParDo.of( + new FeatureRowToCassandraMutationDoFn( + new HashMap() { + { + put( + featureSetSpec.getProject() + + "/" + + featureSetSpec.getName() + + ":" + + featureSetSpec.getVersion(), + FeatureSet.newBuilder().setSpec(featureSetSpec).build()); + } + }, + defaultTtl, + false))); + + CassandraMutation[] expected = + new CassandraMutation[] { + new CassandraMutation( + "test_project/fs:1:entity1=1", + "feature1", + ByteBuffer.wrap(TestUtil.strValue("a").toByteArray()), + 10000000, + 500) + }; + + PAssert.that(output).containsInAnyOrder(expected); + + testPipeline.run(); + } +} diff --git a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java new file mode 100644 index 00000000000..86b4feae05f --- /dev/null +++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java @@ -0,0 +1,345 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.store.serving.redis; + +import static org.junit.Assert.*; + +import com.google.protobuf.Timestamp; +import feast.core.FeatureSetProto; +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.storage.RedisProto.RedisKey; +import feast.store.serving.redis.RedisCustomIO.RedisMutation; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import feast.types.ValueProto.Value; +import feast.types.ValueProto.ValueType.Enum; +import java.util.*; +import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; +import org.apache.beam.sdk.testing.PAssert; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.values.PCollection; +import org.junit.Rule; +import org.junit.Test; + +public class FeatureRowToRedisMutationDoFnTest { + + @Rule public transient TestPipeline p = TestPipeline.create(); + + private FeatureSetProto.FeatureSet fs = + FeatureSetProto.FeatureSet.newBuilder() + .setSpec( + FeatureSetSpec.newBuilder() + .setName("feature_set") + .setVersion(1) + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_primary") + .setValueType(Enum.INT32) + .build()) + .addEntities( + EntitySpec.newBuilder() + .setName("entity_id_secondary") + .setValueType(Enum.STRING) + .build()) + .addFeatures( + FeatureSpec.newBuilder() + .setName("feature_1") + .setValueType(Enum.STRING) + .build()) + .addFeatures( + FeatureSpec.newBuilder() + .setName("feature_2") + .setValueType(Enum.INT64) + .build())) + .build(); + + @Test + public void shouldConvertRowWithDuplicateEntitiesToValidKey() { + Map featureSets = new HashMap<>(); + featureSets.put("feature_set", fs); + + FeatureRow offendingRow = + FeatureRow.newBuilder() + .setFeatureSet("feature_set") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) + .addFields( + Field.newBuilder() + .setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addFields( + Field.newBuilder() + .setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(2))) + .addFields( + Field.newBuilder() + .setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .addFields( + Field.newBuilder() + .setName("feature_1") + .setValue(Value.newBuilder().setStringVal("strValue1"))) + .addFields( + Field.newBuilder() + .setName("feature_2") + .setValue(Value.newBuilder().setInt64Val(1001))) + .build(); + + PCollection output = + p.apply(Create.of(Collections.singletonList(offendingRow))) + .setCoder(ProtoCoder.of(FeatureRow.class)) + .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSets))); + + RedisKey expectedKey = + RedisKey.newBuilder() + .setFeatureSet("feature_set") + .addEntities( + Field.newBuilder() + .setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addEntities( + Field.newBuilder() + .setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .build(); + + FeatureRow expectedValue = + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) + .addFields(Field.newBuilder().setValue(Value.newBuilder().setStringVal("strValue1"))) + .addFields(Field.newBuilder().setValue(Value.newBuilder().setInt64Val(1001))) + .build(); + + PAssert.that(output) + .satisfies( + (SerializableFunction, Void>) + input -> { + input.forEach( + rm -> { + assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray())); + assert (Arrays.equals(rm.getValue(), expectedValue.toByteArray())); + }); + return null; + }); + p.run(); + } + + @Test + public void shouldConvertRowWithOutOfOrderFieldsToValidKey() { + Map featureSets = new HashMap<>(); + featureSets.put("feature_set", fs); + + FeatureRow offendingRow = + FeatureRow.newBuilder() + .setFeatureSet("feature_set") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) + .addFields( + Field.newBuilder() + .setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .addFields( + Field.newBuilder() + .setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addFields( + Field.newBuilder() + .setName("feature_2") + .setValue(Value.newBuilder().setInt64Val(1001))) + .addFields( + Field.newBuilder() + .setName("feature_1") + .setValue(Value.newBuilder().setStringVal("strValue1"))) + .build(); + + PCollection output = + p.apply(Create.of(Collections.singletonList(offendingRow))) + .setCoder(ProtoCoder.of(FeatureRow.class)) + .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSets))); + + RedisKey expectedKey = + RedisKey.newBuilder() + .setFeatureSet("feature_set") + .addEntities( + Field.newBuilder() + .setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addEntities( + Field.newBuilder() + .setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .build(); + + List expectedFields = + Arrays.asList( + Field.newBuilder().setValue(Value.newBuilder().setStringVal("strValue1")).build(), + Field.newBuilder().setValue(Value.newBuilder().setInt64Val(1001)).build()); + FeatureRow expectedValue = + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) + .addAllFields(expectedFields) + .build(); + + PAssert.that(output) + .satisfies( + (SerializableFunction, Void>) + input -> { + input.forEach( + rm -> { + assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray())); + assert (Arrays.equals(rm.getValue(), expectedValue.toByteArray())); + }); + return null; + }); + p.run(); + } + + @Test + public void shouldMergeDuplicateFeatureFields() { + Map featureSets = new HashMap<>(); + featureSets.put("feature_set", fs); + + FeatureRow featureRowWithDuplicatedFeatureFields = + FeatureRow.newBuilder() + .setFeatureSet("feature_set") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) + .addFields( + Field.newBuilder() + .setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addFields( + Field.newBuilder() + .setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .addFields( + Field.newBuilder() + .setName("feature_1") + .setValue(Value.newBuilder().setStringVal("strValue1"))) + .addFields( + Field.newBuilder() + .setName("feature_1") + .setValue(Value.newBuilder().setStringVal("strValue1"))) + .addFields( + Field.newBuilder() + .setName("feature_2") + .setValue(Value.newBuilder().setInt64Val(1001))) + .build(); + + PCollection output = + p.apply(Create.of(Collections.singletonList(featureRowWithDuplicatedFeatureFields))) + .setCoder(ProtoCoder.of(FeatureRow.class)) + .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSets))); + + RedisKey expectedKey = + RedisKey.newBuilder() + .setFeatureSet("feature_set") + .addEntities( + Field.newBuilder() + .setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addEntities( + Field.newBuilder() + .setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .build(); + + FeatureRow expectedValue = + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) + .addFields(Field.newBuilder().setValue(Value.newBuilder().setStringVal("strValue1"))) + .addFields(Field.newBuilder().setValue(Value.newBuilder().setInt64Val(1001))) + .build(); + + PAssert.that(output) + .satisfies( + (SerializableFunction, Void>) + input -> { + input.forEach( + rm -> { + assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray())); + assert (Arrays.equals(rm.getValue(), expectedValue.toByteArray())); + }); + return null; + }); + p.run(); + } + + @Test + public void shouldPopulateMissingFeatureValuesWithDefaultInstance() { + Map featureSets = new HashMap<>(); + featureSets.put("feature_set", fs); + + FeatureRow featureRowWithDuplicatedFeatureFields = + FeatureRow.newBuilder() + .setFeatureSet("feature_set") + .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) + .addFields( + Field.newBuilder() + .setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addFields( + Field.newBuilder() + .setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .addFields( + Field.newBuilder() + .setName("feature_1") + .setValue(Value.newBuilder().setStringVal("strValue1"))) + .build(); + + PCollection output = + p.apply(Create.of(Collections.singletonList(featureRowWithDuplicatedFeatureFields))) + .setCoder(ProtoCoder.of(FeatureRow.class)) + .apply(ParDo.of(new FeatureRowToRedisMutationDoFn(featureSets))); + + RedisKey expectedKey = + RedisKey.newBuilder() + .setFeatureSet("feature_set") + .addEntities( + Field.newBuilder() + .setName("entity_id_primary") + .setValue(Value.newBuilder().setInt32Val(1))) + .addEntities( + Field.newBuilder() + .setName("entity_id_secondary") + .setValue(Value.newBuilder().setStringVal("a"))) + .build(); + + FeatureRow expectedValue = + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(10)) + .addFields(Field.newBuilder().setValue(Value.newBuilder().setStringVal("strValue1"))) + .addFields(Field.newBuilder().setValue(Value.getDefaultInstance())) + .build(); + + PAssert.that(output) + .satisfies( + (SerializableFunction, Void>) + input -> { + input.forEach( + rm -> { + assert (Arrays.equals(rm.getKey(), expectedKey.toByteArray())); + assert (Arrays.equals(rm.getValue(), expectedValue.toByteArray())); + }); + return null; + }); + p.run(); + } +} diff --git a/ingestion/src/test/java/feast/store/serving/redis/RedisCustomIOTest.java b/ingestion/src/test/java/feast/store/serving/redis/RedisCustomIOTest.java index 94167059b43..75663d24a6a 100644 --- a/ingestion/src/test/java/feast/store/serving/redis/RedisCustomIOTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/RedisCustomIOTest.java @@ -16,12 +16,29 @@ */ package feast.store.serving.redis; +import static feast.test.TestUtil.field; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + import feast.core.StoreProto; import feast.storage.RedisProto.RedisKey; import feast.store.serving.redis.RedisCustomIO.Method; import feast.store.serving.redis.RedisCustomIO.RedisMutation; import feast.types.FeatureRowProto.FeatureRow; import feast.types.ValueProto.ValueType.Enum; +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.sync.RedisStringCommands; +import io.lettuce.core.codec.ByteArrayCodec; +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.beam.sdk.testing.PAssert; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Count; @@ -31,51 +48,38 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import redis.clients.jedis.Jedis; import redis.embedded.Redis; import redis.embedded.RedisServer; -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static feast.test.TestUtil.field; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; - public class RedisCustomIOTest { - @Rule - public transient TestPipeline p = TestPipeline.create(); + @Rule public transient TestPipeline p = TestPipeline.create(); private static String REDIS_HOST = "localhost"; private static int REDIS_PORT = 51234; private Redis redis; - private Jedis jedis; - + private RedisClient redisClient; + private RedisStringCommands sync; @Before public void setUp() throws IOException { redis = new RedisServer(REDIS_PORT); redis.start(); - jedis = new Jedis(REDIS_HOST, REDIS_PORT); + redisClient = + RedisClient.create(new RedisURI(REDIS_HOST, REDIS_PORT, java.time.Duration.ofMillis(2000))); + StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); + sync = connection.sync(); } @After public void teardown() { + redisClient.shutdown(); redis.stop(); } @Test public void shouldWriteToRedis() { - StoreProto.Store.RedisConfig redisConfig = StoreProto.Store.RedisConfig.newBuilder() - .setHost(REDIS_HOST) - .setPort(REDIS_PORT) - .build(); + StoreProto.Store.RedisConfig redisConfig = + StoreProto.Store.RedisConfig.newBuilder().setHost(REDIS_HOST).setPort(REDIS_PORT).build(); HashMap kvs = new LinkedHashMap<>(); kvs.put( RedisKey.newBuilder() @@ -110,81 +114,122 @@ public void shouldWriteToRedis() { null)) .collect(Collectors.toList()); - p.apply(Create.of(featureRowWrites)) - .apply(RedisCustomIO.write(redisConfig)); + StoreProto.Store store = + StoreProto.Store.newBuilder() + .setRedisConfig(redisConfig) + .setType(StoreProto.Store.StoreType.REDIS) + .build(); + p.apply(Create.of(featureRowWrites)).apply(RedisCustomIO.write(store)); p.run(); kvs.forEach( (key, value) -> { - byte[] actual = jedis.get(key.toByteArray()); + byte[] actual = sync.get(key.toByteArray()); assertThat(actual, equalTo(value.toByteArray())); }); } @Test(timeout = 10000) public void shouldRetryFailConnection() throws InterruptedException { - StoreProto.Store.RedisConfig redisConfig = StoreProto.Store.RedisConfig.newBuilder() + StoreProto.Store.RedisConfig redisConfig = + StoreProto.Store.RedisConfig.newBuilder() .setHost(REDIS_HOST) .setPort(REDIS_PORT) .setMaxRetries(4) .setInitialBackoffMs(2000) .build(); HashMap kvs = new LinkedHashMap<>(); - kvs.put(RedisKey.newBuilder().setFeatureSet("fs:1") - .addEntities(field("entity", 1, Enum.INT64)).build(), - FeatureRow.newBuilder().setFeatureSet("fs:1") - .addFields(field("entity", 1, Enum.INT64)) - .addFields(field("feature", "one", Enum.STRING)).build()); - - List featureRowWrites = kvs.entrySet().stream() - .map(kv -> new RedisMutation(Method.SET, kv.getKey().toByteArray(), - kv.getValue().toByteArray(), - null, null) - ) + kvs.put( + RedisKey.newBuilder() + .setFeatureSet("fs:1") + .addEntities(field("entity", 1, Enum.INT64)) + .build(), + FeatureRow.newBuilder() + .setFeatureSet("fs:1") + .addFields(field("entity", 1, Enum.INT64)) + .addFields(field("feature", "one", Enum.STRING)) + .build()); + + List featureRowWrites = + kvs.entrySet().stream() + .map( + kv -> + new RedisMutation( + Method.SET, + kv.getKey().toByteArray(), + kv.getValue().toByteArray(), + null, + null)) .collect(Collectors.toList()); - PCollection failedElementCount = p.apply(Create.of(featureRowWrites)) - .apply(RedisCustomIO.write(redisConfig)) - .apply(Count.globally()); + StoreProto.Store store = + StoreProto.Store.newBuilder() + .setRedisConfig(redisConfig) + .setType(StoreProto.Store.StoreType.REDIS) + .build(); + PCollection failedElementCount = + p.apply(Create.of(featureRowWrites)) + .apply(RedisCustomIO.write(store)) + .apply(Count.globally()); redis.stop(); final ScheduledThreadPoolExecutor redisRestartExecutor = new ScheduledThreadPoolExecutor(1); - ScheduledFuture scheduledRedisRestart = redisRestartExecutor.schedule(() -> { - redis.start(); - }, 3, TimeUnit.SECONDS); + ScheduledFuture scheduledRedisRestart = + redisRestartExecutor.schedule( + () -> { + redis.start(); + }, + 3, + TimeUnit.SECONDS); PAssert.that(failedElementCount).containsInAnyOrder(0L); p.run(); scheduledRedisRestart.cancel(true); - kvs.forEach((key, value) -> { - byte[] actual = jedis.get(key.toByteArray()); - assertThat(actual, equalTo(value.toByteArray())); - }); + kvs.forEach( + (key, value) -> { + byte[] actual = sync.get(key.toByteArray()); + assertThat(actual, equalTo(value.toByteArray())); + }); } @Test public void shouldProduceFailedElementIfRetryExceeded() { - StoreProto.Store.RedisConfig redisConfig = StoreProto.Store.RedisConfig.newBuilder() - .setHost(REDIS_HOST) - .setPort(REDIS_PORT) - .build(); + StoreProto.Store.RedisConfig redisConfig = + StoreProto.Store.RedisConfig.newBuilder().setHost(REDIS_HOST).setPort(REDIS_PORT).build(); HashMap kvs = new LinkedHashMap<>(); - kvs.put(RedisKey.newBuilder().setFeatureSet("fs:1") - .addEntities(field("entity", 1, Enum.INT64)).build(), - FeatureRow.newBuilder().setFeatureSet("fs:1") + kvs.put( + RedisKey.newBuilder() + .setFeatureSet("fs:1") + .addEntities(field("entity", 1, Enum.INT64)) + .build(), + FeatureRow.newBuilder() + .setFeatureSet("fs:1") .addFields(field("entity", 1, Enum.INT64)) - .addFields(field("feature", "one", Enum.STRING)).build()); + .addFields(field("feature", "one", Enum.STRING)) + .build()); - List featureRowWrites = kvs.entrySet().stream() - .map(kv -> new RedisMutation(Method.SET, kv.getKey().toByteArray(), - kv.getValue().toByteArray(), - null, null) - ).collect(Collectors.toList()); + List featureRowWrites = + kvs.entrySet().stream() + .map( + kv -> + new RedisMutation( + Method.SET, + kv.getKey().toByteArray(), + kv.getValue().toByteArray(), + null, + null)) + .collect(Collectors.toList()); - PCollection failedElementCount = p.apply(Create.of(featureRowWrites)) - .apply(RedisCustomIO.write(redisConfig)) - .apply(Count.globally()); + StoreProto.Store store = + StoreProto.Store.newBuilder() + .setRedisConfig(redisConfig) + .setType(StoreProto.Store.StoreType.REDIS) + .build(); + PCollection failedElementCount = + p.apply(Create.of(featureRowWrites)) + .apply(RedisCustomIO.write(store)) + .apply(Count.globally()); redis.stop(); PAssert.that(failedElementCount).containsInAnyOrder(1L); diff --git a/ingestion/src/test/java/feast/test/TestUtil.java b/ingestion/src/test/java/feast/test/TestUtil.java index 5c16d7e9e31..fc5122ba7b6 100644 --- a/ingestion/src/test/java/feast/test/TestUtil.java +++ b/ingestion/src/test/java/feast/test/TestUtil.java @@ -18,10 +18,18 @@ import static feast.ingestion.utils.SpecUtil.getFeatureSetReference; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Session; import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; +import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSet; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.core.StoreProto.Store.CassandraConfig; import feast.ingestion.transform.WriteToStore; +import feast.ingestion.utils.StoreUtil; import feast.storage.RedisProto.RedisKey; import feast.types.FeatureRowProto.FeatureRow; import feast.types.FeatureRowProto.FeatureRow.Builder; @@ -35,13 +43,18 @@ import feast.types.ValueProto.StringList; import feast.types.ValueProto.Value; import feast.types.ValueProto.ValueType; +import feast.types.ValueProto.ValueType.Enum; import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import java.util.stream.Stream; import kafka.server.KafkaConfig; import kafka.server.KafkaServerStartable; import org.apache.beam.sdk.PipelineResult; @@ -53,8 +66,10 @@ import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.LongSerializer; +import org.apache.thrift.transport.TTransportException; import org.apache.zookeeper.server.ServerConfig; import org.apache.zookeeper.server.ZooKeeperServerMain; +import org.cassandraunit.utils.EmbeddedCassandraServerHelper; import org.joda.time.Duration; import redis.embedded.RedisServer; @@ -83,6 +98,37 @@ public static void stop() { } } + public static class LocalCassandra { + + public static void start() throws InterruptedException, IOException, TTransportException { + EmbeddedCassandraServerHelper.startEmbeddedCassandra(); + } + + public static void createKeyspaceAndTable(CassandraConfig config) { + StoreUtil.setupCassandra(config); + } + + public static String getHost() { + return EmbeddedCassandraServerHelper.getHost(); + } + + public static int getPort() { + return EmbeddedCassandraServerHelper.getNativeTransportPort(); + } + + public static Cluster getCluster() { + return EmbeddedCassandraServerHelper.getCluster(); + } + + public static Session getSession() { + return EmbeddedCassandraServerHelper.getSession(); + } + + public static void stop() { + EmbeddedCassandraServerHelper.cleanEmbeddedCassandra(); + } + } + public static class LocalKafka { private static KafkaServerStartable server; @@ -165,6 +211,92 @@ public static void publishFeatureRowsToKafka( }); } + /** + * Create a Feature Set Spec. + * + * @param name name of the feature set + * @param version version of the feature set + * @param maxAgeSeconds max age + * @param entities entities provided as map of string to {@link Enum} + * @param features features provided as map of string to {@link Enum} + * @return {@link FeatureSetSpec} + */ + public static FeatureSetSpec createFeatureSetSpec( + String name, + String project, + int version, + int maxAgeSeconds, + Map entities, + Map features) { + FeatureSetSpec.Builder featureSetSpec = + FeatureSetSpec.newBuilder() + .setName(name) + .setProject(project) + .setVersion(version) + .setMaxAge(com.google.protobuf.Duration.newBuilder().setSeconds(maxAgeSeconds).build()); + + for (Entry entity : entities.entrySet()) { + featureSetSpec.addEntities( + EntitySpec.newBuilder().setName(entity.getKey()).setValueType(entity.getValue()).build()); + } + + for (Entry feature : features.entrySet()) { + featureSetSpec.addFeatures( + FeatureSpec.newBuilder() + .setName(feature.getKey()) + .setValueType(feature.getValue()) + .build()); + } + + return featureSetSpec.build(); + } + + /** + * Create a Feature Row. + * + * @param featureSetSpec {@link FeatureSetSpec} + * @param timestampSeconds timestamp given in seconds + * @param fields fields provided as a map name to {@link Value} + * @return {@link FeatureRow} + */ + public static FeatureRow createFeatureRow( + FeatureSetSpec featureSetSpec, long timestampSeconds, Map fields) { + List featureNames = + featureSetSpec.getFeaturesList().stream() + .map(FeatureSpec::getName) + .collect(Collectors.toList()); + List entityNames = + featureSetSpec.getEntitiesList().stream() + .map(EntitySpec::getName) + .collect(Collectors.toList()); + List requiredFields = + Stream.concat(featureNames.stream(), entityNames.stream()).collect(Collectors.toList()); + + if (fields.keySet().containsAll(requiredFields)) { + FeatureRow.Builder featureRow = + FeatureRow.newBuilder() + .setFeatureSet( + featureSetSpec.getProject() + + "/" + + featureSetSpec.getName() + + ":" + + featureSetSpec.getVersion()) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(timestampSeconds).build()); + for (Entry field : fields.entrySet()) { + featureRow.addFields( + Field.newBuilder().setName(field.getKey()).setValue(field.getValue()).build()); + } + return featureRow.build(); + } else { + String missingFields = + requiredFields.stream() + .filter(f -> !fields.keySet().contains(f)) + .collect(Collectors.joining(",")); + throw new IllegalArgumentException( + "FeatureRow is missing some fields defined in FeatureSetSpec: " + missingFields); + } + } + /** * Create a Feature Row with random value according to the FeatureSetSpec * @@ -418,4 +550,12 @@ public static void waitUntilAllElementsAreWrittenToStore( } } } + + public static Value intValue(int val) { + return Value.newBuilder().setInt64Val(val).build(); + } + + public static Value strValue(String val) { + return Value.newBuilder().setStringVal(val).build(); + } } diff --git a/ingestion/src/test/proto/DriverArea.proto b/ingestion/src/test/proto/DriverArea.proto deleted file mode 100644 index fee838b9e17..00000000000 --- a/ingestion/src/test/proto/DriverArea.proto +++ /dev/null @@ -1,10 +0,0 @@ -syntax = "proto3"; - -package feast; - -option java_outer_classname = "DriverAreaProto"; - -message DriverArea { - int32 driverId = 1; - int32 areaId = 2; -} \ No newline at end of file diff --git a/ingestion/src/test/proto/Ping.proto b/ingestion/src/test/proto/Ping.proto deleted file mode 100644 index b1069afa5bd..00000000000 --- a/ingestion/src/test/proto/Ping.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; - -package feast; -import "google/protobuf/timestamp.proto"; - -option java_outer_classname = "PingProto"; - -message Ping { - double lat = 1; - double lng = 2; - google.protobuf.Timestamp timestamp = 3; -} diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.README b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.README new file mode 100644 index 00000000000..3c8759d1702 --- /dev/null +++ b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.README @@ -0,0 +1,9 @@ +WriteFeatureValueMetricsDoFnTest.input file contains data that can be read by test utility +into map of FeatureSetRef -> [FeatureRow]. In the first row, the cell value corresponds to the +field name in the FeatureRow. This should not be changed as the test utility derives the value +type from this name. Empty value in the cell is a value that is not set. For list type, the values +of different element is separated by the '|' character. + +WriteFeatureValueMetricsDoFnTest.output file contains lines of expected StatsD metrics that should +be sent when WriteFeatureValueMetricsDoFn runs. It can be checked against the actual outputted +StatsD metrics to test for correctness. diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input new file mode 100644 index 00000000000..d2985711cee --- /dev/null +++ b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input @@ -0,0 +1,4 @@ +featuresetref,int32,int64,double,float,bool,int32list,int64list,doublelist,floatlist,boollist,bytes,byteslist,string,stringlist +project/featureset:1,1,5,8,5,true,1|4|3,5|1|12,5|7|3,-2.0,true|false,,,, +project/featureset:1,5,-10,8,10.0,true,1|12|5,,,-1.0|-3.0,false|true,,,, +project/featureset:1,6,-4,8,0.0,true,2,2|5,,,true|false,,,, \ No newline at end of file diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output new file mode 100644 index 00000000000..63bc7bbfa4e --- /dev/null +++ b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output @@ -0,0 +1,66 @@ +feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:6|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:4|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:6|g|#ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store + +feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:-10|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:5|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:0|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:-3|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:-4|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:5|g|#ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store + +feast_ingestion.feature_value_min:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:8|g|#ingestion_job_name:job,feast_feature_name:double,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store + +feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:10|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:5|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:10|g|#ingestion_job_name:job,feast_feature_name:float,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store + +feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:1|g|#ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store + +feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:12|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:4|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:3|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:12|g|#ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store + +feast_ingestion.feature_value_min:1|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:12|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:5|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:12|g|#ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store + +feast_ingestion.feature_value_min:3|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:7|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:5|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:5|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:7|g|#ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store + +feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_min:-3|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:-1|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:-2|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:-2|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:0|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:-1|g|#ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store + +feast_ingestion.feature_value_min:0|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_max:1|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_mean:0.5|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_50:0.5|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store +feast_ingestion.feature_value_percentile_90:1|g|#ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_version:1,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store \ No newline at end of file diff --git a/out b/out new file mode 100644 index 00000000000..fb3cdf00d02 Binary files /dev/null and b/out differ diff --git a/pom.xml b/pom.xml index 05fb701ac44..cd04e59e26b 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,7 @@ pom + datatypes/java ingestion core serving @@ -54,6 +55,8 @@ 2.28.2 0.21.0 + + 2.12.1 @@ -140,6 +143,11 @@ + + io.grpc + grpc-core + ${grpcVersion} + io.grpc grpc-netty @@ -200,7 +208,7 @@ com.google.guava guava - 26.0-jre + 25.0-jre com.google.protobuf @@ -260,6 +268,31 @@ + + org.apache.logging.log4j + log4j-api + ${log4jVersion} + + + org.apache.logging.log4j + log4j-core + ${log4jVersion} + + + org.apache.logging.log4j + log4j-jul + ${log4jVersion} + + + org.apache.logging.log4j + log4j-web + ${log4jVersion} + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4jVersion} + + + spotless-check + process-test-classes + + check + + + org.apache.maven.plugins maven-compiler-plugin 3.8.1 - 1.8 - 1.8 + 11 -Xlint:all @@ -369,6 +423,13 @@ org.apache.maven.plugins maven-enforcer-plugin 3.0.0-M2 + + + org.codehaus.mojo + extra-enforcer-rules + 1.2 + + valid-build-environment @@ -378,10 +439,10 @@ - [3.5,4.0) + [3.6,4.0) - [1.8,1.9) + [1.8,11.1) @@ -493,6 +554,7 @@ It is assumed that the GPG command used is version 2.x. --> + true --pinentry-mode loopback @@ -513,6 +575,20 @@ docker-maven-plugin 0.20.1 + + org.apache.maven.plugins + maven-dependency-plugin + 3.1.1 + + + + org.apache.maven.shared + maven-dependency-analyzer + 1.11.1 + + + org.apache.maven.plugins maven-javadoc-plugin @@ -542,25 +618,6 @@ org.xolstice.maven.plugins protobuf-maven-plugin 0.6.1 - - true - - com.google.protobuf:protoc:${protocVersion}:exe:${os.detected.classifier} - - grpc-java - - io.grpc:protoc-gen-grpc-java:${grpcVersion}:exe:${os.detected.classifier} - - - - - - compile - compile-custom - test-compile - - - diff --git a/protos/Makefile b/protos/Makefile index 5418d85d20f..b4f737270c3 100644 --- a/protos/Makefile +++ b/protos/Makefile @@ -9,8 +9,8 @@ gen-go: gen-python: pip install grpcio-tools pip install mypy-protobuf - @$(foreach dir,$(dirs),python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/$(dir)/*.proto;) - @$(foreach dir,$(service_dirs),python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/ feast/$(dir)/*.proto;) + @$(foreach dir,$(dirs),python3 -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/$(dir)/*.proto;) + @$(foreach dir,$(service_dirs),python3 -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/ feast/$(dir)/*.proto;) install-dependencies-docs: mkdir -p $$HOME/bin @@ -30,4 +30,4 @@ install-dependencies-docs: gen-docs: protoc --docs_out=../dist/grpc feast/*/*.proto || \ - $(MAKE) install-dependencies-docs && PATH=$$HOME/bin:$$PATH protoc -I $$HOME/include/ -I . --docs_out=../dist/grpc feast/*/*.proto \ No newline at end of file + $(MAKE) install-dependencies-docs && PATH=$$HOME/bin:$$PATH protoc -I $$HOME/include/ -I . --docs_out=../dist/grpc feast/*/*.proto diff --git a/protos/feast/core/FeatureSet.proto b/protos/feast/core/FeatureSet.proto index 910cc375f7b..429d99c8547 100644 --- a/protos/feast/core/FeatureSet.proto +++ b/protos/feast/core/FeatureSet.proto @@ -24,6 +24,7 @@ import "feast/types/Value.proto"; import "feast/core/Source.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; +import "tensorflow_metadata/proto/v0/schema.proto"; message FeatureSet { // User-specified specifications of this feature set. @@ -67,6 +68,46 @@ message EntitySpec { // Value type of the feature. feast.types.ValueType.Enum value_type = 2; + + // presence_constraints, shape_type and domain_info are referenced from: + // https://github.com/tensorflow/metadata/blob/36f65d1268cbc92cdbcf812ee03dcf47fb53b91e/tensorflow_metadata/proto/v0/schema.proto#L107 + + oneof presence_constraints { + // Constraints on the presence of this feature in the examples. + tensorflow.metadata.v0.FeaturePresence presence = 3; + // Only used in the context of a "group" context, e.g., inside a sequence. + tensorflow.metadata.v0.FeaturePresenceWithinGroup group_presence = 4; + } + + // The shape of the feature which governs the number of values that appear in + // each example. + oneof shape_type { + // The feature has a fixed shape corresponding to a multi-dimensional + // tensor. + tensorflow.metadata.v0.FixedShape shape = 5; + // The feature doesn't have a well defined shape. All we know are limits on + // the minimum and maximum number of values. + tensorflow.metadata.v0.ValueCount value_count = 6; + } + + // Domain for the values of the feature. + oneof domain_info { + // Reference to a domain defined at the schema level. + string domain = 7; + // Inline definitions of domains. + tensorflow.metadata.v0.IntDomain int_domain = 8; + tensorflow.metadata.v0.FloatDomain float_domain = 9; + tensorflow.metadata.v0.StringDomain string_domain = 10; + tensorflow.metadata.v0.BoolDomain bool_domain = 11; + tensorflow.metadata.v0.StructDomain struct_domain = 12; + // Supported semantic domains. + tensorflow.metadata.v0.NaturalLanguageDomain natural_language_domain = 13; + tensorflow.metadata.v0.ImageDomain image_domain = 14; + tensorflow.metadata.v0.MIDDomain mid_domain = 15; + tensorflow.metadata.v0.URLDomain url_domain = 16; + tensorflow.metadata.v0.TimeDomain time_domain = 17; + tensorflow.metadata.v0.TimeOfDayDomain time_of_day_domain = 18; + } } message FeatureSpec { @@ -75,6 +116,46 @@ message FeatureSpec { // Value type of the feature. feast.types.ValueType.Enum value_type = 2; + + // presence_constraints, shape_type and domain_info are referenced from: + // https://github.com/tensorflow/metadata/blob/36f65d1268cbc92cdbcf812ee03dcf47fb53b91e/tensorflow_metadata/proto/v0/schema.proto#L107 + + oneof presence_constraints { + // Constraints on the presence of this feature in the examples. + tensorflow.metadata.v0.FeaturePresence presence = 3; + // Only used in the context of a "group" context, e.g., inside a sequence. + tensorflow.metadata.v0.FeaturePresenceWithinGroup group_presence = 4; + } + + // The shape of the feature which governs the number of values that appear in + // each example. + oneof shape_type { + // The feature has a fixed shape corresponding to a multi-dimensional + // tensor. + tensorflow.metadata.v0.FixedShape shape = 5; + // The feature doesn't have a well defined shape. All we know are limits on + // the minimum and maximum number of values. + tensorflow.metadata.v0.ValueCount value_count = 6; + } + + // Domain for the values of the feature. + oneof domain_info { + // Reference to a domain defined at the schema level. + string domain = 7; + // Inline definitions of domains. + tensorflow.metadata.v0.IntDomain int_domain = 8; + tensorflow.metadata.v0.FloatDomain float_domain = 9; + tensorflow.metadata.v0.StringDomain string_domain = 10; + tensorflow.metadata.v0.BoolDomain bool_domain = 11; + tensorflow.metadata.v0.StructDomain struct_domain = 12; + // Supported semantic domains. + tensorflow.metadata.v0.NaturalLanguageDomain natural_language_domain = 13; + tensorflow.metadata.v0.ImageDomain image_domain = 14; + tensorflow.metadata.v0.MIDDomain mid_domain = 15; + tensorflow.metadata.v0.URLDomain url_domain = 16; + tensorflow.metadata.v0.TimeDomain time_domain = 17; + tensorflow.metadata.v0.TimeOfDayDomain time_of_day_domain = 18; + } } message FeatureSetMeta { diff --git a/protos/feast/core/Store.proto b/protos/feast/core/Store.proto index 931a9d46b69..a010d162547 100644 --- a/protos/feast/core/Store.proto +++ b/protos/feast/core/Store.proto @@ -17,6 +17,8 @@ syntax = "proto3"; package feast.core; +import "google/protobuf/duration.proto"; + option java_package = "feast.core"; option java_outer_classname = "StoreProto"; option go_package = "github.com/gojek/feast/sdk/go/protos/feast/core"; @@ -26,7 +28,7 @@ option go_package = "github.com/gojek/feast/sdk/go/protos/feast/core"; // The way FeatureRow is encoded and decoded when it is written to and read from // the Store depends on the type of the Store. // -// For example, a FeatureRow will materialize as a row in a table in +// For example, a FeatureRow will materialize as a row in a table in // BigQuery but it will materialize as a key, value pair element in Redis. // message Store { @@ -39,30 +41,30 @@ message Store { // The Redis data types used (https://redis.io/topics/data-types): // - key: STRING // - value: STRING - // + // // Encodings: // - key: byte array of RedisKey (refer to feast.storage.RedisKey) // - value: byte array of FeatureRow (refer to feast.types.FeatureRow) - // + // REDIS = 1; // BigQuery stores a FeatureRow element as a row in a BigQuery table. - // + // // Table name is derived from the feature set name and version as: - // [feature_set_name]_v[feature_set_version] - // + // [feature_set_name]_v[feature_set_version] + // // For example: // A feature row for feature set "driver" and version "1" will be written // to table "driver_v1". - // - // The entities and features in a FeatureSetSpec corresponds to the - // fields in the BigQuery table (these make up the BigQuery schema). - // The name of the entity spec and feature spec corresponds to the column - // names, and the value_type of entity spec and feature spec corresponds - // to BigQuery standard SQL data type of the column. - // - // The following BigQuery fields are reserved for Feast internal use. - // Ingestion of entity or feature spec with names identical + // + // The entities and features in a FeatureSetSpec corresponds to the + // fields in the BigQuery table (these make up the BigQuery schema). + // The name of the entity spec and feature spec corresponds to the column + // names, and the value_type of entity spec and feature spec corresponds + // to BigQuery standard SQL data type of the column. + // + // The following BigQuery fields are reserved for Feast internal use. + // Ingestion of entity or feature spec with names identical // to the following field names will raise an exception during ingestion. // // column_name | column_data_type | description @@ -75,21 +77,21 @@ message Store { // of the FeatureRow (https://cloud.google.com/bigquery/docs/partitioned-tables). // // Since newer version of feature set can introduce breaking, non backward- - // compatible BigQuery schema updates, incrementing the version of a + // compatible BigQuery schema updates, incrementing the version of a // feature set will result in the creation of a new empty BigQuery table // with the new schema. - // - // The following table shows how ValueType in Feast is mapped to - // BigQuery Standard SQL data types + // + // The following table shows how ValueType in Feast is mapped to + // BigQuery Standard SQL data types // (https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types): // - // BYTES : BYTES - // STRING : STRING + // BYTES : BYTES + // STRING : STRING // INT32 : INT64 - // INT64 : IN64 + // INT64 : IN64 // DOUBLE : FLOAT64 - // FLOAT : FLOAT64 - // BOOL : BOOL + // FLOAT : FLOAT64 + // BOOL : BOOL // BYTES_LIST : ARRAY // STRING_LIST : ARRAY // INT32_LIST : ARRAY @@ -103,7 +105,20 @@ message Store { // BIGQUERY = 2; - // Unsupported in Feast 0.3 + // Cassandra stores entities as a string partition key, feature as clustering column. + // NOTE: This store currently uses max_age defined in FeatureSet for ttl + // + // Columns: + // - entities: concatenated string of feature set name and all entities' keys and values + // entities concatenated format - [feature_set]:[entity_name1=entity_value1]|[entity_name2=entity_value2] + // TODO: string representation of float or double types may have different value in different runtime or platform + // - feature: clustering column where each feature is a column + // - value: byte array of Value (refer to feast.types.Value) + // + // Internal columns: + // - writeTime: timestamp of the written record. This is used to ensure that new records are not replaced + // by older ones + // - ttl: expiration time the record. Currently using max_age from feature set spec as ttl CASSANDRA = 3; } @@ -123,8 +138,25 @@ message Store { } message CassandraConfig { - string host = 1; + // - bootstrapHosts: [comma delimited value of hosts] + string bootstrap_hosts = 1; int32 port = 2; + string keyspace = 3; + + // Please note that table name must be "feature_store" as is specified in the @Table annotation of the + // datastax object mapper + string table_name = 4; + + // This specifies the replication strategy to use. Please refer to docs for more details: + // https://docs.datastax.com/en/dse/6.7/cql/cql/cql_reference/cql_commands/cqlCreateKeyspace.html#cqlCreateKeyspace__cqlCreateKeyspacereplicationmap-Pr3yUQ7t + map replication_options = 5; + + // Default expiration in seconds to use when FeatureSetSpec does not have max_age defined. + // Specify 0 for no default expiration + google.protobuf.Duration default_ttl = 6; + bool versionless = 7; + string consistency = 8; + bool tracing = 9; } message Subscription { diff --git a/protos/feast/storage/Redis.proto b/protos/feast/storage/Redis.proto index ae287f4e6bf..f58b137e9c1 100644 --- a/protos/feast/storage/Redis.proto +++ b/protos/feast/storage/Redis.proto @@ -32,6 +32,7 @@ message RedisKey { string feature_set = 2; // List of fields containing entity names and their respective values - // contained within this feature row. + // contained within this feature row. The entities should be sorted + // by the entity name alphabetically in ascending order. repeated feast.types.Field entities = 3; } diff --git a/protos/feast/types/FeatureRow.proto b/protos/feast/types/FeatureRow.proto index 24293c6faa6..c170cd5d502 100644 --- a/protos/feast/types/FeatureRow.proto +++ b/protos/feast/types/FeatureRow.proto @@ -36,7 +36,7 @@ message FeatureRow { google.protobuf.Timestamp event_timestamp = 3; // Complete reference to the featureSet this featureRow belongs to, in the form of - // featureSetName:version. This value will be used by the feast ingestion job to filter + // /:. This value will be used by the feast ingestion job to filter // rows, and write the values to the correct tables. string feature_set = 6; -} \ No newline at end of file +} diff --git a/protos/tensorflow_metadata/proto/v0/path.proto b/protos/tensorflow_metadata/proto/v0/path.proto new file mode 100644 index 00000000000..cac09b7a086 --- /dev/null +++ b/protos/tensorflow_metadata/proto/v0/path.proto @@ -0,0 +1,43 @@ +// Copyright 2018 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +syntax = "proto2"; +option cc_enable_arenas = true; + +package tensorflow.metadata.v0; + +option java_package = "org.tensorflow.metadata.v0"; +option java_multiple_files = true; + +// A path is a more general substitute for the name of a field or feature that +// can be used for flat examples as well as structured data. For example, if +// we had data in a protocol buffer: +// message Person { +// int age = 1; +// optional string gender = 2; +// repeated Person parent = 3; +// } +// Thus, here the path {step:["parent", "age"]} in statistics would refer to the +// age of a parent, and {step:["parent", "parent", "age"]} would refer to the +// age of a grandparent. This allows us to distinguish between the statistics +// of parents' ages and grandparents' ages. In general, repeated messages are +// to be preferred to linked lists of arbitrary length. +// For SequenceExample, if we have a feature list "foo", this is represented +// by {step:["##SEQUENCE##", "foo"]}. +message Path { + // Any string is a valid step. + // However, whenever possible have a step be [A-Za-z0-9_]+. + repeated string step = 1; +} diff --git a/protos/tensorflow_metadata/proto/v0/schema.proto b/protos/tensorflow_metadata/proto/v0/schema.proto new file mode 100644 index 00000000000..ce30515c69d --- /dev/null +++ b/protos/tensorflow_metadata/proto/v0/schema.proto @@ -0,0 +1,672 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +syntax = "proto2"; + +package tensorflow.metadata.v0; + +import "google/protobuf/any.proto"; +import "tensorflow_metadata/proto/v0/path.proto"; + +option cc_enable_arenas = true; +option java_package = "org.tensorflow.metadata.v0"; +option java_multiple_files = true; + +// LifecycleStage. Only UNKNOWN_STAGE, BETA, and PRODUCTION features are +// actually validated. +// PLANNED, ALPHA, and DEBUG are treated as DEPRECATED. +enum LifecycleStage { + UNKNOWN_STAGE = 0; // Unknown stage. + PLANNED = 1; // Planned feature, may not be created yet. + ALPHA = 2; // Prototype feature, not used in experiments yet. + BETA = 3; // Used in user-facing experiments. + PRODUCTION = 4; // Used in a significant fraction of user traffic. + DEPRECATED = 5; // No longer supported: do not use in new models. + DEBUG_ONLY = 6; // Only exists for debugging purposes. +} + +// +// Message to represent schema information. +// NextID: 14 +message Schema { + // Features described in this schema. + repeated Feature feature = 1; + + // Sparse features described in this schema. + repeated SparseFeature sparse_feature = 6; + + // Weighted features described in this schema. + repeated WeightedFeature weighted_feature = 12; + + // Use StructDomain instead. + // Sequences described in this schema. A sequence may be described in terms of + // several features. Any features appearing within a sequence must *not* be + // declared as top-level features in . +// GOOGLE-LEGACY repeated Sequence sequence = 2; + + // declared as top-level features in . + // String domains referenced in the features. + repeated StringDomain string_domain = 4; + + // top level float domains that can be reused by features + repeated FloatDomain float_domain = 9; + + // top level int domains that can be reused by features + repeated IntDomain int_domain = 10; + + // Default environments for each feature. + // An environment represents both a type of location (e.g. a server or phone) + // and a time (e.g. right before model X is run). In the standard scenario, + // 99% of the features should be in the default environments TRAINING, + // SERVING, and the LABEL (or labels) AND WEIGHT is only available at TRAINING + // (not at serving). + // Other possible variations: + // 1. There may be TRAINING_MOBILE, SERVING_MOBILE, TRAINING_SERVICE, + // and SERVING_SERVICE. + // 2. If one is ensembling three models, where the predictions of the first + // three models are available for the ensemble model, there may be + // TRAINING, SERVING_INITIAL, SERVING_ENSEMBLE. + // See FeatureProto::not_in_environment and FeatureProto::in_environment. + repeated string default_environment = 5; + + /* BEGIN GOOGLE-LEGACY + // TODO(b/73109633): Change default to false, before removing this field. + optional bool generate_legacy_feature_spec = 7 [default = true]; + END GOOGLE-LEGACY */ + + // Additional information about the schema as a whole. Features may also + // be annotated individually. + optional Annotation annotation = 8; + + // Dataset-level constraints. This is currently used for specifying + // information about changes in num_examples. + optional DatasetConstraints dataset_constraints = 11; + + // TensorRepresentation groups. The keys are the names of the groups. + // Key "" (empty string) denotes the "default" group, which is what should + // be used when a group name is not provided. + // See the documentation at TensorRepresentationGroup for more info. + // Under development. DO NOT USE. + map tensor_representation_group = 13; +} + +// Describes schema-level information about a specific feature. +// NextID: 31 +message Feature { + // The name of the feature. + optional string name = 1; // required + + // This field is no longer supported. Instead, use: + // lifecycle_stage: DEPRECATED + // TODO(b/111450258): remove this. + optional bool deprecated = 2 [deprecated = true]; + + // Comment field for a human readable description of the field. + // TODO(b/123518108): remove this. +// GOOGLE-LEGACY optional string comment = 3 [deprecated = true]; + + oneof presence_constraints { + // Constraints on the presence of this feature in the examples. + FeaturePresence presence = 14; + // Only used in the context of a "group" context, e.g., inside a sequence. + FeaturePresenceWithinGroup group_presence = 17; + } + + // The shape of the feature which governs the number of values that appear in + // each example. + oneof shape_type { + // The feature has a fixed shape corresponding to a multi-dimensional + // tensor. + FixedShape shape = 23; + // The feature doesn't have a well defined shape. All we know are limits on + // the minimum and maximum number of values. + ValueCount value_count = 5; + } + + // Physical type of the feature's values. + // Note that you can have: + // type: BYTES + // int_domain: { + // min: 0 + // max: 3 + // } + // This would be a field that is syntactically BYTES (i.e. strings), but + // semantically an int, i.e. it would be "0", "1", "2", or "3". + optional FeatureType type = 6; + + // Domain for the values of the feature. + oneof domain_info { + // Reference to a domain defined at the schema level. + string domain = 7; + // Inline definitions of domains. + IntDomain int_domain = 9; + FloatDomain float_domain = 10; + StringDomain string_domain = 11; + BoolDomain bool_domain = 13; + StructDomain struct_domain = 29; + // Supported semantic domains. + NaturalLanguageDomain natural_language_domain = 24; + ImageDomain image_domain = 25; + MIDDomain mid_domain = 26; + URLDomain url_domain = 27; + TimeDomain time_domain = 28; + TimeOfDayDomain time_of_day_domain = 30; + } + + // Constraints on the distribution of the feature values. + // Currently only supported for StringDomains. + // TODO(b/69473628): Extend functionality to other domain types. + optional DistributionConstraints distribution_constraints = 15; + + // Additional information about the feature for documentation purpose. + optional Annotation annotation = 16; + + // Tests comparing the distribution to the associated serving data. + optional FeatureComparator skew_comparator = 18; + + // Tests comparing the distribution between two consecutive spans (e.g. days). + optional FeatureComparator drift_comparator = 21; + + // List of environments this feature is present in. + // Should be disjoint from not_in_environment. + // This feature is in environment "foo" if: + // ("foo" is in in_environment or default_environments) AND + // "foo" is not in not_in_environment. + // See Schema::default_environments. + repeated string in_environment = 20; + + // List of environments this feature is not present in. + // Should be disjoint from of in_environment. + // See Schema::default_environments and in_environment. + repeated string not_in_environment = 19; + + // The lifecycle stage of a feature. It can also apply to its descendants. + // i.e., if a struct is DEPRECATED, its children are implicitly deprecated. + optional LifecycleStage lifecycle_stage = 22; +} + +// Additional information about the schema or about a feature. +message Annotation { + // Tags can be used to mark features. For example, tag on user_age feature can + // be `user_feature`, tag on user_country feature can be `location_feature`, + // `user_feature`. + repeated string tag = 1; + // Free-text comments. This can be used as a description of the feature, + // developer notes etc. + repeated string comment = 2; + // Application-specific metadata may be attached here. + repeated .google.protobuf.Any extra_metadata = 3; +} + +// Checks that the ratio of the current value to the previous value is not below +// the min_fraction_threshold or above the max_fraction_threshold. That is, +// previous value * min_fraction_threshold <= current value <= +// previous value * max_fraction_threshold. +// To specify that the value cannot change, set both min_fraction_threshold and +// max_fraction_threshold to 1.0. +message NumericValueComparator { + optional double min_fraction_threshold = 1; + optional double max_fraction_threshold = 2; +} + +// Constraints on the entire dataset. +message DatasetConstraints { + // Tests differences in number of examples between the current data and the + // previous span. + optional NumericValueComparator num_examples_drift_comparator = 1; + // Tests comparisions in number of examples between the current data and the + // previous version of that data. + optional NumericValueComparator num_examples_version_comparator = 2; + // Minimum number of examples in the dataset. + optional int64 min_examples_count = 3; +} + +// Specifies a fixed shape for the feature's values. The immediate implication +// is that each feature has a fixed number of values. Moreover, these values +// can be parsed in a multi-dimensional tensor using the specified axis sizes. +// The FixedShape defines a lexicographical ordering of the data. For instance, +// if there is a FixedShape { +// dim {size:3} dim {size:2} +// } +// then tensor[0][0]=field[0] +// then tensor[0][1]=field[1] +// then tensor[1][0]=field[2] +// then tensor[1][1]=field[3] +// then tensor[2][0]=field[4] +// then tensor[2][1]=field[5] +// +// The FixedShape message is identical with the TensorFlow TensorShape proto +// message. +message FixedShape { + // The dimensions that define the shape. The total number of values in each + // example is the product of sizes of each dimension. + repeated Dim dim = 2; + + // An axis in a multi-dimensional feature representation. + message Dim { + optional int64 size = 1; + + // Optional name of the tensor dimension. + optional string name = 2; + } +} + +// Limits on maximum and minimum number of values in a +// single example (when the feature is present). Use this when the minimum +// value count can be different than the maximum value count. Otherwise prefer +// FixedShape. +message ValueCount { + optional int64 min = 1; + optional int64 max = 2; +} + +/* BEGIN GOOGLE-LEGACY +// Constraint on the number of elements in a sequence. +message LengthConstraint { + optional int64 min = 1; + optional int64 max = 2; +} + +// A sequence is a logical feature that comprises several "raw" features that +// encode values at different "steps" within the sequence. +// TODO(b/110490010): Delete this. This is a special case of StructDomain. +message Sequence { + // An optional name for this sequence. Used mostly for debugging and + // presentation. + optional string name = 1; + + // Features that comprise the sequence. These features are "zipped" together + // to form the values for the sequence at different steps. + // - Use group_presence within each feature to encode presence constraints + // within the sequence. + // - If all features have the same value-count constraints then + // declare this once using the shape_constraint below. + repeated Feature feature = 2; + + // Constraints on the presence of the sequence across all examples in the + // dataset. The sequence is assumed to be present if at least one of its + // features is present. + optional FeaturePresence presence = 3; + + // Shape constraints that apply on all the features that comprise the + // sequence. If this is set then the value_count in 'feature' is + // ignored. + // TODO(martinz): delete: there is no reason to believe the shape of the + // fields in a sequence will be the same. Use the fields in Feature instead. + oneof shape_constraint { + ValueCount value_count = 4; + FixedShape fixed_shape = 5; + } + + // Constraint on the number of elements in a sequence. + optional LengthConstraint length_constraint = 6; +} +END GOOGLE-LEGACY */ + +// Represents a weighted feature that is encoded as a combination of raw base +// features. The `weight_feature` should be a float feature with identical +// shape as the `feature`. This is useful for representing weights associated +// with categorical tokens (e.g. a TFIDF weight associated with each token). +// TODO(b/142122960): Handle WeightedCategorical end to end in TFX (validation, +// TFX Unit Testing, etc) +message WeightedFeature { + // Name for the weighted feature. This should not clash with other features in + // the same schema. + optional string name = 1; // required + // Path of a base feature to be weighted. Required. + optional Path feature = 2; + // Path of weight feature to associate with the base feature. Must be same + // shape as feature. Required. + optional Path weight_feature = 3; + // The lifecycle_stage determines where a feature is expected to be used, + // and therefore how important issues with it are. + optional LifecycleStage lifecycle_stage = 4; +} + +// A sparse feature represents a sparse tensor that is encoded with a +// combination of raw features, namely index features and a value feature. Each +// index feature defines a list of indices in a different dimension. +message SparseFeature { + reserved 11; + // Name for the sparse feature. This should not clash with other features in + // the same schema. + optional string name = 1; // required + + // This field is no longer supported. Instead, use: + // lifecycle_stage: DEPRECATED + // TODO(b/111450258): remove this. + optional bool deprecated = 2 [deprecated = true]; + + // The lifecycle_stage determines where a feature is expected to be used, + // and therefore how important issues with it are. + optional LifecycleStage lifecycle_stage = 7; + + // Comment field for a human readable description of the field. + // TODO(martinz): delete, convert to annotation. +// GOOGLE-LEGACY optional string comment = 3 [deprecated = true]; + + // Constraints on the presence of this feature in examples. + // Deprecated, this is inferred by the referred features. + optional FeaturePresence presence = 4 [deprecated = true]; + + // Shape of the sparse tensor that this SparseFeature represents. + // Currently not supported. + // TODO(b/109669962): Consider deriving this from the referred features. + optional FixedShape dense_shape = 5; + + // Features that represent indexes. Should be integers >= 0. + repeated IndexFeature index_feature = 6; // at least one + message IndexFeature { + // Name of the index-feature. This should be a reference to an existing + // feature in the schema. + optional string name = 1; + } + + // If true then the index values are already sorted lexicographically. + optional bool is_sorted = 8; + + optional ValueFeature value_feature = 9; // required + message ValueFeature { + // Name of the value-feature. This should be a reference to an existing + // feature in the schema. + optional string name = 1; + } + + // Type of value feature. + // Deprecated, this is inferred by the referred features. + optional FeatureType type = 10 [deprecated = true]; +} + +// Models constraints on the distribution of a feature's values. +// TODO(martinz): replace min_domain_mass with max_off_domain (but slowly). +message DistributionConstraints { + // The minimum fraction (in [0,1]) of values across all examples that + // should come from the feature's domain, e.g.: + // 1.0 => All values must come from the domain. + // .9 => At least 90% of the values must come from the domain. + optional double min_domain_mass = 1 [default = 1.0]; +} + +// Encodes information for domains of integer values. +// Note that FeatureType could be either INT or BYTES. +message IntDomain { + // Id of the domain. Required if the domain is defined at the schema level. If + // so, then the name must be unique within the schema. + optional string name = 1; + + // Min and max values for the domain. + optional int64 min = 3; + optional int64 max = 4; + + // If true then the domain encodes categorical values (i.e., ids) rather than + // ordinal values. + optional bool is_categorical = 5; +} + +// Encodes information for domains of float values. +// Note that FeatureType could be either INT or BYTES. +message FloatDomain { + // Id of the domain. Required if the domain is defined at the schema level. If + // so, then the name must be unique within the schema. + optional string name = 1; + + // Min and max values of the domain. + optional float min = 3; + optional float max = 4; +} + +// Domain for a recursive struct. +// NOTE: If a feature with a StructDomain is deprecated, then all the +// child features (features and sparse_features of the StructDomain) are also +// considered to be deprecated. Similarly child features can only be in +// environments of the parent feature. +message StructDomain { + repeated Feature feature = 1; + + repeated SparseFeature sparse_feature = 2; +} + +// Encodes information for domains of string values. +message StringDomain { + // Id of the domain. Required if the domain is defined at the schema level. If + // so, then the name must be unique within the schema. + optional string name = 1; + + // The values appearing in the domain. + repeated string value = 2; +} + +// Encodes information about the domain of a boolean attribute that encodes its +// TRUE/FALSE values as strings, or 0=false, 1=true. +// Note that FeatureType could be either INT or BYTES. +message BoolDomain { + // Id of the domain. Required if the domain is defined at the schema level. If + // so, then the name must be unique within the schema. + optional string name = 1; + + // Strings values for TRUE/FALSE. + optional string true_value = 2; + optional string false_value = 3; +} + +// BEGIN SEMANTIC-TYPES-PROTOS +// Semantic domains are specialized feature domains. For example a string +// Feature might represent a Time of a specific format. +// Semantic domains are defined as protocol buffers to allow further sub-types / +// specialization, e.g: NaturalLanguageDomain can provide information on the +// language of the text. + +// Natural language text. +message NaturalLanguageDomain {} + +// Image data. +message ImageDomain {} + +// Knowledge graph ID, see: https://www.wikidata.org/wiki/Property:P646 +message MIDDomain {} + +// A URL, see: https://en.wikipedia.org/wiki/URL +message URLDomain {} + +// Time or date representation. +message TimeDomain { + enum IntegerTimeFormat { + FORMAT_UNKNOWN = 0; + UNIX_DAYS = 5; // Number of days since 1970-01-01. + UNIX_SECONDS = 1; + UNIX_MILLISECONDS = 2; + UNIX_MICROSECONDS = 3; + UNIX_NANOSECONDS = 4; + } + + oneof format { + // Expected format that contains a combination of regular characters and + // special format specifiers. Format specifiers are a subset of the + // strptime standard. + string string_format = 1; + + // Expected format of integer times. + IntegerTimeFormat integer_format = 2; + } +} + +// Time of day, without a particular date. +message TimeOfDayDomain { + enum IntegerTimeOfDayFormat { + FORMAT_UNKNOWN = 0; + // Time values, containing hour/minute/second/nanos, encoded into 8-byte + // bit fields following the ZetaSQL convention: + // 6 5 4 3 2 1 + // MSB 3210987654321098765432109876543210987654321098765432109876543210 LSB + // | H || M || S ||---------- nanos -----------| + PACKED_64_NANOS = 1; + } + + oneof format { + // Expected format that contains a combination of regular characters and + // special format specifiers. Format specifiers are a subset of the + // strptime standard. + string string_format = 1; + + // Expected format of integer times. + IntegerTimeOfDayFormat integer_format = 2; + } +} +// END SEMANTIC-TYPES-PROTOS + +// Describes the physical representation of a feature. +// It may be different than the logical representation, which +// is represented as a Domain. +enum FeatureType { + TYPE_UNKNOWN = 0; + BYTES = 1; + INT = 2; + FLOAT = 3; + STRUCT = 4; +} + +// Describes constraints on the presence of the feature in the data. +message FeaturePresence { + // Minimum fraction of examples that have this feature. + optional double min_fraction = 1; + // Minimum number of examples that have this feature. + optional int64 min_count = 2; +} + +// Records constraints on the presence of a feature inside a "group" context +// (e.g., .presence inside a group of features that define a sequence). +message FeaturePresenceWithinGroup { + optional bool required = 1; +} + +// Checks that the L-infinity norm is below a certain threshold between the +// two discrete distributions. Since this is applied to a FeatureNameStatistics, +// it only considers the top k. +// L_infty(p,q) = max_i |p_i-q_i| +message InfinityNorm { + // The InfinityNorm is in the interval [0.0, 1.0] so sensible bounds should + // be in the interval [0.0, 1.0). + optional double threshold = 1; +} + +message FeatureComparator { + optional InfinityNorm infinity_norm = 1; +} + +// A TensorRepresentation captures the intent for converting columns in a +// dataset to TensorFlow Tensors (or more generally, tf.CompositeTensors). +// Note that one tf.CompositeTensor may consist of data from multiple columns, +// for example, a N-dimensional tf.SparseTensor may need N + 1 columns to +// provide the sparse indices and values. +// Note that the "column name" that a TensorRepresentation needs is a +// string, not a Path -- it means that the column name identifies a top-level +// Feature in the schema (i.e. you cannot specify a Feature nested in a STRUCT +// Feature). +message TensorRepresentation { + message DefaultValue { + oneof kind { + double float_value = 1; + // Note that the data column might be of a shorter integral type. It's the + // user's responsitiblity to make sure the default value fits that type. + int64 int_value = 2; + bytes bytes_value = 3; + // uint_value should only be used if the default value can't fit in a + // int64 (`int_value`). + uint64 uint_value = 4; + } + } + + // A tf.Tensor + message DenseTensor { + // Identifies the column in the dataset that provides the values of this + // Tensor. + optional string column_name = 1; + // The shape of each row of the data (i.e. does not include the batch + // dimension) + optional FixedShape shape = 2; + // If this column is missing values in a row, the default_value will be + // used to fill that row. + optional DefaultValue default_value = 3; + } + + // A ragged tf.SparseTensor that models nested lists. + message VarLenSparseTensor { + // Identifies the column in the dataset that should be converted to the + // VarLenSparseTensor. + optional string column_name = 1; + } + + // A tf.SparseTensor whose indices and values come from separate data columns. + // This will replace Schema.sparse_feature eventually. + // The index columns must be of INT type, and all the columns must co-occur + // and have the same valency at the same row. + message SparseTensor { + // The dense shape of the resulting SparseTensor (does not include the batch + // dimension). + optional FixedShape dense_shape = 1; + // The columns constitute the coordinates of the values. + // indices_column[i][j] contains the coordinate of the i-th dimension of the + // j-th value. + repeated string index_column_names = 2; + // The column that contains the values. + optional string value_column_name = 3; + } + + oneof kind { + DenseTensor dense_tensor = 1; + VarLenSparseTensor varlen_sparse_tensor = 2; + SparseTensor sparse_tensor = 3; + } +} + +// A TensorRepresentationGroup is a collection of TensorRepresentations with +// names. These names may serve as identifiers when converting the dataset +// to a collection of Tensors or tf.CompositeTensors. +// For example, given the following group: +// { +// key: "dense_tensor" +// tensor_representation { +// dense_tensor { +// column_name: "univalent_feature" +// shape { +// dim { +// size: 1 +// } +// } +// default_value { +// float_value: 0 +// } +// } +// } +// } +// { +// key: "varlen_sparse_tensor" +// tensor_representation { +// varlen_sparse_tensor { +// column_name: "multivalent_feature" +// } +// } +// } +// +// Then the schema is expected to have feature "univalent_feature" and +// "multivalent_feature", and when a batch of data is converted to Tensors using +// this TensorRepresentationGroup, the result may be the following dict: +// { +// "dense_tensor": tf.Tensor(...), +// "varlen_sparse_tensor": tf.SparseTensor(...), +// } +message TensorRepresentationGroup { + map tensor_representation = 1; +} diff --git a/sdk/__init__.py b/sdk/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/go/protos/feast/core/CoreService.pb.go b/sdk/go/protos/feast/core/CoreService.pb.go index 45ad9ed79a4..18882b47d63 100644 --- a/sdk/go/protos/feast/core/CoreService.pb.go +++ b/sdk/go/protos/feast/core/CoreService.pb.go @@ -940,7 +940,9 @@ func init() { proto.RegisterType((*ListProjectsResponse)(nil), "feast.core.ListProjectsResponse") } -func init() { proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_d9be266444105411) } +func init() { + proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_d9be266444105411) +} var fileDescriptor_d9be266444105411 = []byte{ // 762 bytes of a gzipped FileDescriptorProto @@ -996,11 +998,11 @@ var fileDescriptor_d9be266444105411 = []byte{ // Reference imports to suppress errors if they are not otherwise used. var _ context.Context -var _ grpc.ClientConn +var _ grpc.ClientConnInterface // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 +const _ = grpc.SupportPackageIsVersion6 // CoreServiceClient is the client API for CoreService service. // @@ -1048,10 +1050,10 @@ type CoreServiceClient interface { } type coreServiceClient struct { - cc *grpc.ClientConn + cc grpc.ClientConnInterface } -func NewCoreServiceClient(cc *grpc.ClientConn) CoreServiceClient { +func NewCoreServiceClient(cc grpc.ClientConnInterface) CoreServiceClient { return &coreServiceClient{cc} } diff --git a/sdk/go/protos/feast/core/FeatureSet.pb.go b/sdk/go/protos/feast/core/FeatureSet.pb.go index 26d9d9c4f7e..1eec11cbe0f 100644 --- a/sdk/go/protos/feast/core/FeatureSet.pb.go +++ b/sdk/go/protos/feast/core/FeatureSet.pb.go @@ -10,6 +10,7 @@ import ( duration "github.com/golang/protobuf/ptypes/duration" timestamp "github.com/golang/protobuf/ptypes/timestamp" math "math" + v0 "tensorflow_metadata/proto/v0" ) // Reference imports to suppress errors if they are not otherwise used. @@ -204,10 +205,37 @@ type EntitySpec struct { // Name of the entity. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // Value type of the feature. - ValueType types.ValueType_Enum `protobuf:"varint,2,opt,name=value_type,json=valueType,proto3,enum=feast.types.ValueType_Enum" json:"value_type,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ValueType types.ValueType_Enum `protobuf:"varint,2,opt,name=value_type,json=valueType,proto3,enum=feast.types.ValueType_Enum" json:"value_type,omitempty"` + // Types that are valid to be assigned to PresenceConstraints: + // *EntitySpec_Presence + // *EntitySpec_GroupPresence + PresenceConstraints isEntitySpec_PresenceConstraints `protobuf_oneof:"presence_constraints"` + // The shape of the feature which governs the number of values that appear in + // each example. + // + // Types that are valid to be assigned to ShapeType: + // *EntitySpec_Shape + // *EntitySpec_ValueCount + ShapeType isEntitySpec_ShapeType `protobuf_oneof:"shape_type"` + // Domain for the values of the feature. + // + // Types that are valid to be assigned to DomainInfo: + // *EntitySpec_Domain + // *EntitySpec_IntDomain + // *EntitySpec_FloatDomain + // *EntitySpec_StringDomain + // *EntitySpec_BoolDomain + // *EntitySpec_StructDomain + // *EntitySpec_NaturalLanguageDomain + // *EntitySpec_ImageDomain + // *EntitySpec_MidDomain + // *EntitySpec_UrlDomain + // *EntitySpec_TimeDomain + // *EntitySpec_TimeOfDayDomain + DomainInfo isEntitySpec_DomainInfo `protobuf_oneof:"domain_info"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *EntitySpec) Reset() { *m = EntitySpec{} } @@ -249,14 +277,304 @@ func (m *EntitySpec) GetValueType() types.ValueType_Enum { return types.ValueType_INVALID } +type isEntitySpec_PresenceConstraints interface { + isEntitySpec_PresenceConstraints() +} + +type EntitySpec_Presence struct { + Presence *v0.FeaturePresence `protobuf:"bytes,3,opt,name=presence,proto3,oneof"` +} + +type EntitySpec_GroupPresence struct { + GroupPresence *v0.FeaturePresenceWithinGroup `protobuf:"bytes,4,opt,name=group_presence,json=groupPresence,proto3,oneof"` +} + +func (*EntitySpec_Presence) isEntitySpec_PresenceConstraints() {} + +func (*EntitySpec_GroupPresence) isEntitySpec_PresenceConstraints() {} + +func (m *EntitySpec) GetPresenceConstraints() isEntitySpec_PresenceConstraints { + if m != nil { + return m.PresenceConstraints + } + return nil +} + +func (m *EntitySpec) GetPresence() *v0.FeaturePresence { + if x, ok := m.GetPresenceConstraints().(*EntitySpec_Presence); ok { + return x.Presence + } + return nil +} + +func (m *EntitySpec) GetGroupPresence() *v0.FeaturePresenceWithinGroup { + if x, ok := m.GetPresenceConstraints().(*EntitySpec_GroupPresence); ok { + return x.GroupPresence + } + return nil +} + +type isEntitySpec_ShapeType interface { + isEntitySpec_ShapeType() +} + +type EntitySpec_Shape struct { + Shape *v0.FixedShape `protobuf:"bytes,5,opt,name=shape,proto3,oneof"` +} + +type EntitySpec_ValueCount struct { + ValueCount *v0.ValueCount `protobuf:"bytes,6,opt,name=value_count,json=valueCount,proto3,oneof"` +} + +func (*EntitySpec_Shape) isEntitySpec_ShapeType() {} + +func (*EntitySpec_ValueCount) isEntitySpec_ShapeType() {} + +func (m *EntitySpec) GetShapeType() isEntitySpec_ShapeType { + if m != nil { + return m.ShapeType + } + return nil +} + +func (m *EntitySpec) GetShape() *v0.FixedShape { + if x, ok := m.GetShapeType().(*EntitySpec_Shape); ok { + return x.Shape + } + return nil +} + +func (m *EntitySpec) GetValueCount() *v0.ValueCount { + if x, ok := m.GetShapeType().(*EntitySpec_ValueCount); ok { + return x.ValueCount + } + return nil +} + +type isEntitySpec_DomainInfo interface { + isEntitySpec_DomainInfo() +} + +type EntitySpec_Domain struct { + Domain string `protobuf:"bytes,7,opt,name=domain,proto3,oneof"` +} + +type EntitySpec_IntDomain struct { + IntDomain *v0.IntDomain `protobuf:"bytes,8,opt,name=int_domain,json=intDomain,proto3,oneof"` +} + +type EntitySpec_FloatDomain struct { + FloatDomain *v0.FloatDomain `protobuf:"bytes,9,opt,name=float_domain,json=floatDomain,proto3,oneof"` +} + +type EntitySpec_StringDomain struct { + StringDomain *v0.StringDomain `protobuf:"bytes,10,opt,name=string_domain,json=stringDomain,proto3,oneof"` +} + +type EntitySpec_BoolDomain struct { + BoolDomain *v0.BoolDomain `protobuf:"bytes,11,opt,name=bool_domain,json=boolDomain,proto3,oneof"` +} + +type EntitySpec_StructDomain struct { + StructDomain *v0.StructDomain `protobuf:"bytes,12,opt,name=struct_domain,json=structDomain,proto3,oneof"` +} + +type EntitySpec_NaturalLanguageDomain struct { + NaturalLanguageDomain *v0.NaturalLanguageDomain `protobuf:"bytes,13,opt,name=natural_language_domain,json=naturalLanguageDomain,proto3,oneof"` +} + +type EntitySpec_ImageDomain struct { + ImageDomain *v0.ImageDomain `protobuf:"bytes,14,opt,name=image_domain,json=imageDomain,proto3,oneof"` +} + +type EntitySpec_MidDomain struct { + MidDomain *v0.MIDDomain `protobuf:"bytes,15,opt,name=mid_domain,json=midDomain,proto3,oneof"` +} + +type EntitySpec_UrlDomain struct { + UrlDomain *v0.URLDomain `protobuf:"bytes,16,opt,name=url_domain,json=urlDomain,proto3,oneof"` +} + +type EntitySpec_TimeDomain struct { + TimeDomain *v0.TimeDomain `protobuf:"bytes,17,opt,name=time_domain,json=timeDomain,proto3,oneof"` +} + +type EntitySpec_TimeOfDayDomain struct { + TimeOfDayDomain *v0.TimeOfDayDomain `protobuf:"bytes,18,opt,name=time_of_day_domain,json=timeOfDayDomain,proto3,oneof"` +} + +func (*EntitySpec_Domain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_IntDomain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_FloatDomain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_StringDomain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_BoolDomain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_StructDomain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_NaturalLanguageDomain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_ImageDomain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_MidDomain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_UrlDomain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_TimeDomain) isEntitySpec_DomainInfo() {} + +func (*EntitySpec_TimeOfDayDomain) isEntitySpec_DomainInfo() {} + +func (m *EntitySpec) GetDomainInfo() isEntitySpec_DomainInfo { + if m != nil { + return m.DomainInfo + } + return nil +} + +func (m *EntitySpec) GetDomain() string { + if x, ok := m.GetDomainInfo().(*EntitySpec_Domain); ok { + return x.Domain + } + return "" +} + +func (m *EntitySpec) GetIntDomain() *v0.IntDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_IntDomain); ok { + return x.IntDomain + } + return nil +} + +func (m *EntitySpec) GetFloatDomain() *v0.FloatDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_FloatDomain); ok { + return x.FloatDomain + } + return nil +} + +func (m *EntitySpec) GetStringDomain() *v0.StringDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_StringDomain); ok { + return x.StringDomain + } + return nil +} + +func (m *EntitySpec) GetBoolDomain() *v0.BoolDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_BoolDomain); ok { + return x.BoolDomain + } + return nil +} + +func (m *EntitySpec) GetStructDomain() *v0.StructDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_StructDomain); ok { + return x.StructDomain + } + return nil +} + +func (m *EntitySpec) GetNaturalLanguageDomain() *v0.NaturalLanguageDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_NaturalLanguageDomain); ok { + return x.NaturalLanguageDomain + } + return nil +} + +func (m *EntitySpec) GetImageDomain() *v0.ImageDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_ImageDomain); ok { + return x.ImageDomain + } + return nil +} + +func (m *EntitySpec) GetMidDomain() *v0.MIDDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_MidDomain); ok { + return x.MidDomain + } + return nil +} + +func (m *EntitySpec) GetUrlDomain() *v0.URLDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_UrlDomain); ok { + return x.UrlDomain + } + return nil +} + +func (m *EntitySpec) GetTimeDomain() *v0.TimeDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_TimeDomain); ok { + return x.TimeDomain + } + return nil +} + +func (m *EntitySpec) GetTimeOfDayDomain() *v0.TimeOfDayDomain { + if x, ok := m.GetDomainInfo().(*EntitySpec_TimeOfDayDomain); ok { + return x.TimeOfDayDomain + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*EntitySpec) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*EntitySpec_Presence)(nil), + (*EntitySpec_GroupPresence)(nil), + (*EntitySpec_Shape)(nil), + (*EntitySpec_ValueCount)(nil), + (*EntitySpec_Domain)(nil), + (*EntitySpec_IntDomain)(nil), + (*EntitySpec_FloatDomain)(nil), + (*EntitySpec_StringDomain)(nil), + (*EntitySpec_BoolDomain)(nil), + (*EntitySpec_StructDomain)(nil), + (*EntitySpec_NaturalLanguageDomain)(nil), + (*EntitySpec_ImageDomain)(nil), + (*EntitySpec_MidDomain)(nil), + (*EntitySpec_UrlDomain)(nil), + (*EntitySpec_TimeDomain)(nil), + (*EntitySpec_TimeOfDayDomain)(nil), + } +} + type FeatureSpec struct { // Name of the feature. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // Value type of the feature. - ValueType types.ValueType_Enum `protobuf:"varint,2,opt,name=value_type,json=valueType,proto3,enum=feast.types.ValueType_Enum" json:"value_type,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ValueType types.ValueType_Enum `protobuf:"varint,2,opt,name=value_type,json=valueType,proto3,enum=feast.types.ValueType_Enum" json:"value_type,omitempty"` + // Types that are valid to be assigned to PresenceConstraints: + // *FeatureSpec_Presence + // *FeatureSpec_GroupPresence + PresenceConstraints isFeatureSpec_PresenceConstraints `protobuf_oneof:"presence_constraints"` + // The shape of the feature which governs the number of values that appear in + // each example. + // + // Types that are valid to be assigned to ShapeType: + // *FeatureSpec_Shape + // *FeatureSpec_ValueCount + ShapeType isFeatureSpec_ShapeType `protobuf_oneof:"shape_type"` + // Domain for the values of the feature. + // + // Types that are valid to be assigned to DomainInfo: + // *FeatureSpec_Domain + // *FeatureSpec_IntDomain + // *FeatureSpec_FloatDomain + // *FeatureSpec_StringDomain + // *FeatureSpec_BoolDomain + // *FeatureSpec_StructDomain + // *FeatureSpec_NaturalLanguageDomain + // *FeatureSpec_ImageDomain + // *FeatureSpec_MidDomain + // *FeatureSpec_UrlDomain + // *FeatureSpec_TimeDomain + // *FeatureSpec_TimeOfDayDomain + DomainInfo isFeatureSpec_DomainInfo `protobuf_oneof:"domain_info"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *FeatureSpec) Reset() { *m = FeatureSpec{} } @@ -298,6 +616,269 @@ func (m *FeatureSpec) GetValueType() types.ValueType_Enum { return types.ValueType_INVALID } +type isFeatureSpec_PresenceConstraints interface { + isFeatureSpec_PresenceConstraints() +} + +type FeatureSpec_Presence struct { + Presence *v0.FeaturePresence `protobuf:"bytes,3,opt,name=presence,proto3,oneof"` +} + +type FeatureSpec_GroupPresence struct { + GroupPresence *v0.FeaturePresenceWithinGroup `protobuf:"bytes,4,opt,name=group_presence,json=groupPresence,proto3,oneof"` +} + +func (*FeatureSpec_Presence) isFeatureSpec_PresenceConstraints() {} + +func (*FeatureSpec_GroupPresence) isFeatureSpec_PresenceConstraints() {} + +func (m *FeatureSpec) GetPresenceConstraints() isFeatureSpec_PresenceConstraints { + if m != nil { + return m.PresenceConstraints + } + return nil +} + +func (m *FeatureSpec) GetPresence() *v0.FeaturePresence { + if x, ok := m.GetPresenceConstraints().(*FeatureSpec_Presence); ok { + return x.Presence + } + return nil +} + +func (m *FeatureSpec) GetGroupPresence() *v0.FeaturePresenceWithinGroup { + if x, ok := m.GetPresenceConstraints().(*FeatureSpec_GroupPresence); ok { + return x.GroupPresence + } + return nil +} + +type isFeatureSpec_ShapeType interface { + isFeatureSpec_ShapeType() +} + +type FeatureSpec_Shape struct { + Shape *v0.FixedShape `protobuf:"bytes,5,opt,name=shape,proto3,oneof"` +} + +type FeatureSpec_ValueCount struct { + ValueCount *v0.ValueCount `protobuf:"bytes,6,opt,name=value_count,json=valueCount,proto3,oneof"` +} + +func (*FeatureSpec_Shape) isFeatureSpec_ShapeType() {} + +func (*FeatureSpec_ValueCount) isFeatureSpec_ShapeType() {} + +func (m *FeatureSpec) GetShapeType() isFeatureSpec_ShapeType { + if m != nil { + return m.ShapeType + } + return nil +} + +func (m *FeatureSpec) GetShape() *v0.FixedShape { + if x, ok := m.GetShapeType().(*FeatureSpec_Shape); ok { + return x.Shape + } + return nil +} + +func (m *FeatureSpec) GetValueCount() *v0.ValueCount { + if x, ok := m.GetShapeType().(*FeatureSpec_ValueCount); ok { + return x.ValueCount + } + return nil +} + +type isFeatureSpec_DomainInfo interface { + isFeatureSpec_DomainInfo() +} + +type FeatureSpec_Domain struct { + Domain string `protobuf:"bytes,7,opt,name=domain,proto3,oneof"` +} + +type FeatureSpec_IntDomain struct { + IntDomain *v0.IntDomain `protobuf:"bytes,8,opt,name=int_domain,json=intDomain,proto3,oneof"` +} + +type FeatureSpec_FloatDomain struct { + FloatDomain *v0.FloatDomain `protobuf:"bytes,9,opt,name=float_domain,json=floatDomain,proto3,oneof"` +} + +type FeatureSpec_StringDomain struct { + StringDomain *v0.StringDomain `protobuf:"bytes,10,opt,name=string_domain,json=stringDomain,proto3,oneof"` +} + +type FeatureSpec_BoolDomain struct { + BoolDomain *v0.BoolDomain `protobuf:"bytes,11,opt,name=bool_domain,json=boolDomain,proto3,oneof"` +} + +type FeatureSpec_StructDomain struct { + StructDomain *v0.StructDomain `protobuf:"bytes,12,opt,name=struct_domain,json=structDomain,proto3,oneof"` +} + +type FeatureSpec_NaturalLanguageDomain struct { + NaturalLanguageDomain *v0.NaturalLanguageDomain `protobuf:"bytes,13,opt,name=natural_language_domain,json=naturalLanguageDomain,proto3,oneof"` +} + +type FeatureSpec_ImageDomain struct { + ImageDomain *v0.ImageDomain `protobuf:"bytes,14,opt,name=image_domain,json=imageDomain,proto3,oneof"` +} + +type FeatureSpec_MidDomain struct { + MidDomain *v0.MIDDomain `protobuf:"bytes,15,opt,name=mid_domain,json=midDomain,proto3,oneof"` +} + +type FeatureSpec_UrlDomain struct { + UrlDomain *v0.URLDomain `protobuf:"bytes,16,opt,name=url_domain,json=urlDomain,proto3,oneof"` +} + +type FeatureSpec_TimeDomain struct { + TimeDomain *v0.TimeDomain `protobuf:"bytes,17,opt,name=time_domain,json=timeDomain,proto3,oneof"` +} + +type FeatureSpec_TimeOfDayDomain struct { + TimeOfDayDomain *v0.TimeOfDayDomain `protobuf:"bytes,18,opt,name=time_of_day_domain,json=timeOfDayDomain,proto3,oneof"` +} + +func (*FeatureSpec_Domain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_IntDomain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_FloatDomain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_StringDomain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_BoolDomain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_StructDomain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_NaturalLanguageDomain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_ImageDomain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_MidDomain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_UrlDomain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_TimeDomain) isFeatureSpec_DomainInfo() {} + +func (*FeatureSpec_TimeOfDayDomain) isFeatureSpec_DomainInfo() {} + +func (m *FeatureSpec) GetDomainInfo() isFeatureSpec_DomainInfo { + if m != nil { + return m.DomainInfo + } + return nil +} + +func (m *FeatureSpec) GetDomain() string { + if x, ok := m.GetDomainInfo().(*FeatureSpec_Domain); ok { + return x.Domain + } + return "" +} + +func (m *FeatureSpec) GetIntDomain() *v0.IntDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_IntDomain); ok { + return x.IntDomain + } + return nil +} + +func (m *FeatureSpec) GetFloatDomain() *v0.FloatDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_FloatDomain); ok { + return x.FloatDomain + } + return nil +} + +func (m *FeatureSpec) GetStringDomain() *v0.StringDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_StringDomain); ok { + return x.StringDomain + } + return nil +} + +func (m *FeatureSpec) GetBoolDomain() *v0.BoolDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_BoolDomain); ok { + return x.BoolDomain + } + return nil +} + +func (m *FeatureSpec) GetStructDomain() *v0.StructDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_StructDomain); ok { + return x.StructDomain + } + return nil +} + +func (m *FeatureSpec) GetNaturalLanguageDomain() *v0.NaturalLanguageDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_NaturalLanguageDomain); ok { + return x.NaturalLanguageDomain + } + return nil +} + +func (m *FeatureSpec) GetImageDomain() *v0.ImageDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_ImageDomain); ok { + return x.ImageDomain + } + return nil +} + +func (m *FeatureSpec) GetMidDomain() *v0.MIDDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_MidDomain); ok { + return x.MidDomain + } + return nil +} + +func (m *FeatureSpec) GetUrlDomain() *v0.URLDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_UrlDomain); ok { + return x.UrlDomain + } + return nil +} + +func (m *FeatureSpec) GetTimeDomain() *v0.TimeDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_TimeDomain); ok { + return x.TimeDomain + } + return nil +} + +func (m *FeatureSpec) GetTimeOfDayDomain() *v0.TimeOfDayDomain { + if x, ok := m.GetDomainInfo().(*FeatureSpec_TimeOfDayDomain); ok { + return x.TimeOfDayDomain + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*FeatureSpec) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*FeatureSpec_Presence)(nil), + (*FeatureSpec_GroupPresence)(nil), + (*FeatureSpec_Shape)(nil), + (*FeatureSpec_ValueCount)(nil), + (*FeatureSpec_Domain)(nil), + (*FeatureSpec_IntDomain)(nil), + (*FeatureSpec_FloatDomain)(nil), + (*FeatureSpec_StringDomain)(nil), + (*FeatureSpec_BoolDomain)(nil), + (*FeatureSpec_StructDomain)(nil), + (*FeatureSpec_NaturalLanguageDomain)(nil), + (*FeatureSpec_ImageDomain)(nil), + (*FeatureSpec_MidDomain)(nil), + (*FeatureSpec_UrlDomain)(nil), + (*FeatureSpec_TimeDomain)(nil), + (*FeatureSpec_TimeOfDayDomain)(nil), + } +} + type FeatureSetMeta struct { // Created timestamp of this specific feature set. CreatedTimestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=created_timestamp,json=createdTimestamp,proto3" json:"created_timestamp,omitempty"` @@ -361,40 +942,69 @@ func init() { proto.RegisterType((*FeatureSetMeta)(nil), "feast.core.FeatureSetMeta") } -func init() { proto.RegisterFile("feast/core/FeatureSet.proto", fileDescriptor_972fbd278ac19c0c) } +func init() { + proto.RegisterFile("feast/core/FeatureSet.proto", fileDescriptor_972fbd278ac19c0c) +} var fileDescriptor_972fbd278ac19c0c = []byte{ - // 510 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x93, 0x4f, 0x6f, 0xda, 0x30, - 0x18, 0xc6, 0x07, 0xa5, 0x50, 0x5e, 0x26, 0x96, 0xf9, 0xb0, 0x66, 0xed, 0xb4, 0x21, 0x4e, 0xa8, - 0x07, 0x5b, 0x4a, 0x77, 0xda, 0x8d, 0x0a, 0x56, 0x21, 0x75, 0xa8, 0x72, 0x58, 0xa5, 0x4d, 0x9b, - 0x90, 0x09, 0x2f, 0x59, 0x5a, 0x82, 0xa3, 0xd8, 0x41, 0xe5, 0x53, 0xec, 0x33, 0xec, 0x9b, 0x4e, - 0x71, 0x12, 0x92, 0xa1, 0x6e, 0xa7, 0xdd, 0x62, 0x3f, 0x3f, 0xbf, 0x79, 0xde, 0x7f, 0x70, 0xbe, - 0x42, 0xa1, 0x34, 0xf3, 0x64, 0x8c, 0xec, 0x23, 0x0a, 0x9d, 0xc4, 0xe8, 0xa2, 0xa6, 0x51, 0x2c, - 0xb5, 0x24, 0x60, 0x44, 0x9a, 0x8a, 0x67, 0xa7, 0x19, 0xa8, 0x77, 0x11, 0x2a, 0x76, 0x27, 0xd6, - 0x09, 0x66, 0x50, 0x21, 0x98, 0x08, 0xae, 0x4c, 0x62, 0xaf, 0x10, 0xde, 0xfa, 0x52, 0xfa, 0x6b, - 0x64, 0xe6, 0xb4, 0x48, 0x56, 0x6c, 0x99, 0xc4, 0x42, 0x07, 0x72, 0x93, 0xeb, 0xef, 0x0e, 0x75, - 0x1d, 0x84, 0xa8, 0xb4, 0x08, 0xa3, 0x0c, 0xe8, 0xaf, 0x01, 0x4a, 0x4b, 0x84, 0x42, 0x43, 0x45, - 0xe8, 0xd9, 0xb5, 0x5e, 0x6d, 0xd0, 0x71, 0xce, 0x68, 0xe9, 0x8d, 0x96, 0x94, 0x1b, 0xa1, 0xc7, - 0x0d, 0x97, 0xf2, 0x21, 0x6a, 0x61, 0xd7, 0xff, 0xc5, 0x7f, 0x42, 0x2d, 0xb8, 0xe1, 0xfa, 0xbf, - 0xea, 0xd0, 0xfd, 0x33, 0x10, 0xb1, 0xa1, 0x15, 0xc5, 0xf2, 0x1e, 0x3d, 0x6d, 0xb7, 0x7a, 0xb5, - 0x41, 0x9b, 0x17, 0x47, 0x42, 0xa0, 0xb1, 0x11, 0x21, 0x1a, 0x33, 0x6d, 0x6e, 0xbe, 0x53, 0x7a, - 0x8b, 0xb1, 0x0a, 0xe4, 0xc6, 0xfc, 0xf3, 0x98, 0x17, 0x47, 0xe2, 0xc0, 0x09, 0x6e, 0x74, 0xa0, - 0x03, 0x54, 0xf6, 0x51, 0xef, 0x68, 0xd0, 0x71, 0x5e, 0x55, 0xed, 0x8c, 0x53, 0x6d, 0x67, 0xac, - 0xef, 0x39, 0x72, 0x09, 0x27, 0xab, 0xcc, 0x8d, 0xb2, 0x1b, 0xe6, 0xcd, 0xe9, 0x53, 0x29, 0x98, - 0x47, 0x05, 0x48, 0x1c, 0x68, 0x85, 0xe2, 0x71, 0x2e, 0x7c, 0xb4, 0x8f, 0x4d, 0xda, 0xaf, 0x69, - 0x56, 0x64, 0x5a, 0x14, 0x99, 0x8e, 0xf2, 0x26, 0xf0, 0x66, 0x28, 0x1e, 0x87, 0x3e, 0x92, 0x0b, - 0x68, 0x2a, 0xd3, 0x36, 0xbb, 0x69, 0x9e, 0x90, 0xea, 0x6f, 0xb2, 0x86, 0xf2, 0x9c, 0xe8, 0x7f, - 0x03, 0x28, 0xcd, 0x3e, 0x59, 0x84, 0x0f, 0x00, 0xdb, 0x74, 0x38, 0xe6, 0xe9, 0xa0, 0x98, 0x3a, - 0x74, 0x9d, 0xf3, 0x3c, 0xa2, 0x99, 0x1d, 0x6a, 0x66, 0x67, 0xb6, 0x8b, 0xd2, 0xbc, 0x93, 0x90, - 0xb7, 0xb7, 0xc5, 0xb9, 0xff, 0x1d, 0x3a, 0x95, 0xb4, 0xfe, 0x7b, 0xf8, 0x9f, 0xb5, 0x6a, 0x83, - 0xd3, 0xce, 0x93, 0x6b, 0x78, 0xe9, 0xc5, 0x28, 0x34, 0x2e, 0xe7, 0xfb, 0xe1, 0xdb, 0x0f, 0xd8, - 0x61, 0xe5, 0x66, 0x05, 0xc1, 0xad, 0xfc, 0xd1, 0xfe, 0x86, 0xbc, 0x87, 0xa6, 0xd2, 0x42, 0x27, - 0x2a, 0xf7, 0xf4, 0xe6, 0x2f, 0xe3, 0x69, 0x18, 0x9e, 0xb3, 0x17, 0x37, 0x60, 0x1d, 0x6a, 0x84, - 0x40, 0xd7, 0x9d, 0x0d, 0x67, 0x9f, 0xdd, 0xf9, 0x64, 0x7a, 0x37, 0xbc, 0x99, 0x8c, 0xac, 0x67, - 0x95, 0xbb, 0xdb, 0xf1, 0x74, 0x34, 0x99, 0x5e, 0x5b, 0x35, 0x62, 0xc1, 0xf3, 0xfc, 0x8e, 0x8f, - 0x87, 0xa3, 0x2f, 0x56, 0xfd, 0x6a, 0x0a, 0x95, 0x7d, 0xbd, 0x7a, 0x51, 0x46, 0xbe, 0x4d, 0x33, - 0xf8, 0xca, 0xfc, 0x40, 0xff, 0x48, 0x16, 0xd4, 0x93, 0x21, 0xf3, 0xe5, 0x3d, 0x3e, 0xb0, 0x6c, - 0x71, 0xd5, 0xf2, 0x81, 0xf9, 0x32, 0xdb, 0x42, 0xc5, 0xca, 0x65, 0x5e, 0x34, 0xcd, 0xd5, 0xe5, - 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb8, 0xd5, 0xf0, 0x13, 0x23, 0x04, 0x00, 0x00, + // 938 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x97, 0xdf, 0x6e, 0xe2, 0xc6, + 0x17, 0xc7, 0x03, 0x49, 0x48, 0x38, 0x10, 0xc2, 0x8e, 0x7e, 0xbf, 0x8d, 0xbb, 0x5b, 0xb5, 0x29, + 0xad, 0xd4, 0x74, 0xa5, 0xda, 0x2b, 0xb6, 0x57, 0x7b, 0x17, 0x0a, 0x0d, 0xa8, 0x59, 0x1a, 0x19, + 0x36, 0x55, 0xdb, 0x0b, 0x6b, 0x30, 0x83, 0x33, 0xbb, 0xf6, 0x8c, 0xe5, 0x19, 0xd3, 0xf0, 0x14, + 0x7d, 0x80, 0xf6, 0xa6, 0x6f, 0x5a, 0xcd, 0xd8, 0x63, 0x93, 0x28, 0xd0, 0x3e, 0x00, 0x77, 0x39, + 0x3e, 0xdf, 0xf9, 0xf8, 0xfc, 0xcb, 0xc1, 0x03, 0x2f, 0x17, 0x04, 0x0b, 0xe9, 0xf8, 0x3c, 0x21, + 0xce, 0x0f, 0x04, 0xcb, 0x34, 0x21, 0x13, 0x22, 0xed, 0x38, 0xe1, 0x92, 0x23, 0xd0, 0x4e, 0x5b, + 0x39, 0x5f, 0x9c, 0x65, 0x42, 0xb9, 0x8a, 0x89, 0x70, 0x6e, 0x71, 0x98, 0x92, 0x4c, 0x64, 0x1c, + 0x9a, 0x30, 0xe1, 0x69, 0xe2, 0x1b, 0xc7, 0x67, 0x01, 0xe7, 0x41, 0x48, 0x1c, 0x6d, 0xcd, 0xd2, + 0x85, 0x33, 0x4f, 0x13, 0x2c, 0x29, 0x67, 0xb9, 0xff, 0xf3, 0xc7, 0x7e, 0x49, 0x23, 0x22, 0x24, + 0x8e, 0xe2, 0x5c, 0xf0, 0x8d, 0x24, 0x4c, 0xf0, 0x64, 0x11, 0xf2, 0xdf, 0xbd, 0x88, 0x48, 0x3c, + 0xc7, 0x12, 0x67, 0x6a, 0x67, 0xf9, 0xda, 0x11, 0xfe, 0x1d, 0x89, 0x70, 0x26, 0xed, 0x84, 0x00, + 0x65, 0xf4, 0xc8, 0x86, 0x03, 0x11, 0x13, 0xdf, 0xaa, 0x9c, 0x57, 0x2e, 0x1a, 0xdd, 0x17, 0x76, + 0x99, 0x86, 0x5d, 0xaa, 0x26, 0x31, 0xf1, 0x5d, 0xad, 0x53, 0x7a, 0xc5, 0xb7, 0xaa, 0xdb, 0xf4, + 0xef, 0x88, 0xc4, 0xae, 0xd6, 0x75, 0xfe, 0xae, 0x42, 0xeb, 0x21, 0x08, 0x59, 0x70, 0x14, 0x27, + 0xfc, 0x03, 0xf1, 0xa5, 0x75, 0x74, 0x5e, 0xb9, 0xa8, 0xbb, 0xc6, 0x44, 0x08, 0x0e, 0x18, 0x8e, + 0x88, 0x0e, 0xa6, 0xee, 0xea, 0xbf, 0x95, 0x7a, 0x49, 0x12, 0x41, 0x39, 0xd3, 0xef, 0x3c, 0x74, + 0x8d, 0x89, 0xba, 0x70, 0x4c, 0x98, 0xa4, 0x92, 0x12, 0x61, 0xed, 0x9f, 0xef, 0x5f, 0x34, 0xba, + 0xcf, 0xd7, 0xc3, 0x19, 0x28, 0xdf, 0x4a, 0x87, 0x5e, 0xe8, 0xd0, 0x1b, 0x38, 0x5e, 0x64, 0xd1, + 0x08, 0xeb, 0x40, 0x9f, 0x39, 0x7b, 0x2a, 0x05, 0x7d, 0xc8, 0x08, 0x51, 0x17, 0x8e, 0x22, 0x7c, + 0xef, 0xe1, 0x80, 0x58, 0x87, 0x3a, 0xed, 0x4f, 0xec, 0xac, 0x1f, 0xb6, 0xe9, 0x87, 0xdd, 0xcf, + 0xfb, 0xe5, 0xd6, 0x22, 0x7c, 0x7f, 0x19, 0x10, 0xf4, 0x0a, 0x6a, 0x42, 0x77, 0xd8, 0xaa, 0xe9, + 0x23, 0x68, 0xfd, 0x35, 0x59, 0xef, 0xdd, 0x5c, 0xd1, 0xf9, 0x13, 0x00, 0xca, 0x68, 0x9f, 0xac, + 0xc2, 0x5b, 0x80, 0xa5, 0x1a, 0x24, 0x4f, 0x0d, 0x95, 0x2e, 0x44, 0xab, 0xfb, 0x32, 0x47, 0xea, + 0x39, 0xb3, 0xf5, 0x9c, 0x4d, 0x57, 0xb1, 0x4a, 0x3c, 0x8d, 0xdc, 0xfa, 0xd2, 0xd8, 0x68, 0x00, + 0xc7, 0x71, 0x42, 0x04, 0x61, 0x3e, 0xb1, 0xf6, 0x75, 0x30, 0x5f, 0xdb, 0xe5, 0xb8, 0xd8, 0x66, + 0x5c, 0xec, 0xe5, 0x6b, 0x93, 0xff, 0x4d, 0x2e, 0x1f, 0xee, 0xb9, 0xc5, 0x51, 0xf4, 0x1b, 0xb4, + 0x82, 0x84, 0xa7, 0xb1, 0x57, 0xc0, 0x0e, 0x34, 0xac, 0xfb, 0x1f, 0x61, 0x3f, 0x53, 0x79, 0x47, + 0xd9, 0x95, 0x42, 0x0c, 0xf7, 0xdc, 0x13, 0xcd, 0x32, 0x3e, 0xf4, 0x16, 0x0e, 0xc5, 0x1d, 0x8e, + 0x4d, 0x81, 0x3b, 0x1b, 0x99, 0xf4, 0x9e, 0xcc, 0x27, 0x4a, 0x39, 0xac, 0xb8, 0xd9, 0x11, 0x34, + 0x80, 0x46, 0x56, 0x1b, 0x9f, 0xa7, 0x4c, 0xe6, 0xf5, 0xde, 0x48, 0xd0, 0x75, 0xfa, 0x5e, 0x29, + 0x87, 0x15, 0x37, 0x2b, 0xaa, 0xb6, 0x90, 0x05, 0xb5, 0x39, 0x8f, 0x30, 0x65, 0xd9, 0x54, 0x0e, + 0xab, 0x6e, 0x6e, 0xa3, 0x1e, 0x00, 0x65, 0xd2, 0xcb, 0xbd, 0xc7, 0x9a, 0xff, 0xc5, 0x26, 0xfe, + 0x88, 0xc9, 0xbe, 0x16, 0x0e, 0xab, 0x6e, 0x9d, 0x1a, 0x03, 0x0d, 0xa1, 0xb9, 0x08, 0x39, 0x2e, + 0x28, 0x75, 0x4d, 0xf9, 0x72, 0x63, 0x9e, 0x4a, 0x5b, 0x70, 0x1a, 0x8b, 0xd2, 0x44, 0x3f, 0xc2, + 0x89, 0x90, 0x09, 0x65, 0x81, 0x41, 0x81, 0x46, 0x7d, 0xb5, 0x09, 0x35, 0xd1, 0xe2, 0x82, 0xd5, + 0x14, 0x6b, 0xb6, 0xaa, 0xdd, 0x8c, 0xf3, 0xd0, 0xa0, 0x1a, 0xdb, 0x6b, 0xd7, 0xe3, 0x3c, 0x2c, + 0x40, 0x30, 0x2b, 0xac, 0x3c, 0xa6, 0xd4, 0x2f, 0xd2, 0x6b, 0xfe, 0x6b, 0x4c, 0xa9, 0x2f, 0x1f, + 0xc4, 0x54, 0xd8, 0x28, 0x80, 0x33, 0xa6, 0x26, 0x07, 0x87, 0x5e, 0x88, 0x59, 0x90, 0xe2, 0x80, + 0x18, 0xec, 0x89, 0xc6, 0x7e, 0xbb, 0x09, 0x3b, 0xce, 0x8e, 0x5d, 0xe7, 0xa7, 0x0a, 0xfe, 0xff, + 0xd9, 0x53, 0x0e, 0xd5, 0x13, 0x1a, 0xad, 0xd1, 0x5b, 0xdb, 0x7b, 0x32, 0x8a, 0xd6, 0x99, 0x0d, + 0x5a, 0x9a, 0x6a, 0x42, 0x22, 0x3a, 0x37, 0x9c, 0xd3, 0xed, 0x13, 0xf2, 0x6e, 0xd4, 0x2f, 0x27, + 0x24, 0xa2, 0xf3, 0x92, 0x91, 0x26, 0x45, 0x27, 0xda, 0xdb, 0x19, 0xef, 0xdd, 0xeb, 0x92, 0x91, + 0x26, 0x61, 0xd9, 0x4e, 0xf5, 0xcb, 0x60, 0x20, 0xcf, 0xb6, 0xb7, 0x73, 0x4a, 0xa3, 0x32, 0x1f, + 0x90, 0x85, 0x85, 0x6e, 0x01, 0x69, 0x0c, 0x5f, 0x78, 0x73, 0xbc, 0x32, 0x34, 0xb4, 0x7d, 0x77, + 0x28, 0xda, 0x4f, 0x8b, 0x3e, 0x5e, 0x15, 0xc8, 0x53, 0xf9, 0xf0, 0x51, 0xef, 0x39, 0xfc, 0xcf, + 0x2c, 0x0f, 0xcf, 0xe7, 0x4c, 0xc8, 0x04, 0x53, 0x26, 0x45, 0xaf, 0x09, 0xa0, 0xff, 0x95, 0xf5, + 0x76, 0xeb, 0x9d, 0x40, 0x23, 0x7b, 0xa3, 0x47, 0xd9, 0x82, 0x77, 0xfe, 0x02, 0x68, 0xac, 0xed, + 0xe5, 0xdd, 0x7a, 0xdc, 0xad, 0xc7, 0xdd, 0x7a, 0xdc, 0xad, 0xc7, 0xdd, 0x7a, 0xcc, 0xd6, 0xe3, + 0x1f, 0x95, 0xf5, 0x0f, 0x6c, 0xf5, 0xe5, 0x8d, 0xae, 0xe0, 0x99, 0x9f, 0x10, 0x2c, 0xc9, 0xdc, + 0x2b, 0xee, 0x09, 0xc5, 0x07, 0xfe, 0xe3, 0x2f, 0xd7, 0xa9, 0x51, 0xb8, 0xed, 0xfc, 0x50, 0xf1, + 0x04, 0x7d, 0x07, 0x35, 0x21, 0xb1, 0x4c, 0x45, 0xbe, 0x52, 0x3f, 0xdd, 0x70, 0x3d, 0xd0, 0x1a, + 0x37, 0xd7, 0xbe, 0xba, 0x86, 0xf6, 0x63, 0x1f, 0x42, 0xd0, 0x9a, 0x4c, 0x2f, 0xa7, 0xef, 0x27, + 0xde, 0x68, 0x7c, 0x7b, 0x79, 0x3d, 0xea, 0xb7, 0xf7, 0xd6, 0x9e, 0xdd, 0x0c, 0xc6, 0xfd, 0xd1, + 0xf8, 0xaa, 0x5d, 0x41, 0x6d, 0x68, 0xe6, 0xcf, 0xdc, 0xc1, 0x65, 0xff, 0x97, 0x76, 0xb5, 0x37, + 0x86, 0xb5, 0xab, 0x55, 0xef, 0xb4, 0x24, 0xdf, 0xa8, 0x0c, 0x7e, 0x75, 0x02, 0x2a, 0xef, 0xd2, + 0x99, 0xed, 0xf3, 0xc8, 0x09, 0xf8, 0x07, 0xf2, 0xd1, 0xc9, 0xee, 0x58, 0x62, 0xfe, 0xd1, 0x09, + 0x78, 0x76, 0x05, 0x12, 0x4e, 0x79, 0xef, 0x9a, 0xd5, 0xf4, 0xa3, 0x37, 0xff, 0x04, 0x00, 0x00, + 0xff, 0xff, 0x4e, 0x2a, 0x9e, 0xd9, 0xce, 0x0d, 0x00, 0x00, } diff --git a/sdk/go/protos/feast/core/Source.pb.go b/sdk/go/protos/feast/core/Source.pb.go index 090210a4961..d8cfeace4c0 100644 --- a/sdk/go/protos/feast/core/Source.pb.go +++ b/sdk/go/protos/feast/core/Source.pb.go @@ -176,7 +176,9 @@ func init() { proto.RegisterType((*KafkaSourceConfig)(nil), "feast.core.KafkaSourceConfig") } -func init() { proto.RegisterFile("feast/core/Source.proto", fileDescriptor_4d161c4e53091468) } +func init() { + proto.RegisterFile("feast/core/Source.proto", fileDescriptor_4d161c4e53091468) +} var fileDescriptor_4d161c4e53091468 = []byte{ // 273 bytes of a gzipped FileDescriptorProto diff --git a/sdk/go/protos/feast/core/Store.pb.go b/sdk/go/protos/feast/core/Store.pb.go index 9120edcb42c..e8c7bbfe834 100644 --- a/sdk/go/protos/feast/core/Store.pb.go +++ b/sdk/go/protos/feast/core/Store.pb.go @@ -6,6 +6,7 @@ package core import ( fmt "fmt" proto "github.com/golang/protobuf/proto" + duration "github.com/golang/protobuf/ptypes/duration" math "math" ) @@ -91,7 +92,20 @@ const ( // in a FeatureRow corresponds to NULL value in BigQuery. // Store_BIGQUERY Store_StoreType = 2 - // Unsupported in Feast 0.3 + // Cassandra stores entities as a string partition key, feature as clustering column. + // NOTE: This store currently uses max_age defined in FeatureSet for ttl + // + // Columns: + // - entities: concatenated string of feature set name and all entities' keys and values + // entities concatenated format - [feature_set]:[entity_name1=entity_value1]|[entity_name2=entity_value2] + // TODO: string representation of float or double types may have different value in different runtime or platform + // - feature: clustering column where each feature is a column + // - value: byte array of Value (refer to feast.types.Value) + // + // Internal columns: + // - writeTime: timestamp of the written record. This is used to ensure that new records are not replaced + // by older ones + // - ttl: expiration time the record. Currently using max_age from feature set spec as ttl Store_CASSANDRA Store_StoreType = 3 ) @@ -250,8 +264,13 @@ func (*Store) XXX_OneofWrappers() []interface{} { } type Store_RedisConfig struct { - Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` - Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + // Optional. The number of milliseconds to wait before retrying failed Redis connection. + // By default, Feast uses exponential backoff policy and "initial_backoff_ms" sets the initial wait duration. + InitialBackoffMs int32 `protobuf:"varint,3,opt,name=initial_backoff_ms,json=initialBackoffMs,proto3" json:"initial_backoff_ms,omitempty"` + // Optional. Maximum total number of retries for connecting to Redis. Default to zero retries. + MaxRetries int32 `protobuf:"varint,4,opt,name=max_retries,json=maxRetries,proto3" json:"max_retries,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -296,6 +315,20 @@ func (m *Store_RedisConfig) GetPort() int32 { return 0 } +func (m *Store_RedisConfig) GetInitialBackoffMs() int32 { + if m != nil { + return m.InitialBackoffMs + } + return 0 +} + +func (m *Store_RedisConfig) GetMaxRetries() int32 { + if m != nil { + return m.MaxRetries + } + return 0 +} + type Store_BigQueryConfig struct { ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` DatasetId string `protobuf:"bytes,2,opt,name=dataset_id,json=datasetId,proto3" json:"dataset_id,omitempty"` @@ -344,11 +377,24 @@ func (m *Store_BigQueryConfig) GetDatasetId() string { } type Store_CassandraConfig struct { - Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` - Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // - bootstrapHosts: [comma delimited value of hosts] + BootstrapHosts string `protobuf:"bytes,1,opt,name=bootstrap_hosts,json=bootstrapHosts,proto3" json:"bootstrap_hosts,omitempty"` + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + Keyspace string `protobuf:"bytes,3,opt,name=keyspace,proto3" json:"keyspace,omitempty"` + // Please note that table name must be "feature_store" as is specified in the @Table annotation of the + // datastax object mapper + TableName string `protobuf:"bytes,4,opt,name=table_name,json=tableName,proto3" json:"table_name,omitempty"` + // This specifies the replication strategy to use. Please refer to docs for more details: + // https://docs.datastax.com/en/dse/6.7/cql/cql/cql_reference/cql_commands/cqlCreateKeyspace.html#cqlCreateKeyspace__cqlCreateKeyspacereplicationmap-Pr3yUQ7t + ReplicationOptions map[string]string `protobuf:"bytes,5,rep,name=replication_options,json=replicationOptions,proto3" json:"replication_options,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Default expiration in seconds to use when FeatureSetSpec does not have max_age defined. + // Specify 0 for no default expiration + DefaultTtl *duration.Duration `protobuf:"bytes,6,opt,name=default_ttl,json=defaultTtl,proto3" json:"default_ttl,omitempty"` + Versionless bool `protobuf:"varint,7,opt,name=versionless,proto3" json:"versionless,omitempty"` + Consistency string `protobuf:"bytes,8,opt,name=consistency,proto3" json:"consistency,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Store_CassandraConfig) Reset() { *m = Store_CassandraConfig{} } @@ -376,9 +422,9 @@ func (m *Store_CassandraConfig) XXX_DiscardUnknown() { var xxx_messageInfo_Store_CassandraConfig proto.InternalMessageInfo -func (m *Store_CassandraConfig) GetHost() string { +func (m *Store_CassandraConfig) GetBootstrapHosts() string { if m != nil { - return m.Host + return m.BootstrapHosts } return "" } @@ -390,6 +436,48 @@ func (m *Store_CassandraConfig) GetPort() int32 { return 0 } +func (m *Store_CassandraConfig) GetKeyspace() string { + if m != nil { + return m.Keyspace + } + return "" +} + +func (m *Store_CassandraConfig) GetTableName() string { + if m != nil { + return m.TableName + } + return "" +} + +func (m *Store_CassandraConfig) GetReplicationOptions() map[string]string { + if m != nil { + return m.ReplicationOptions + } + return nil +} + +func (m *Store_CassandraConfig) GetDefaultTtl() *duration.Duration { + if m != nil { + return m.DefaultTtl + } + return nil +} + +func (m *Store_CassandraConfig) GetVersionless() bool { + if m != nil { + return m.Versionless + } + return false +} + +func (m *Store_CassandraConfig) GetConsistency() string { + if m != nil { + return m.Consistency + } + return "" +} + type Store_Subscription struct { // Name of project that the feature sets belongs to. This can be one of // - [project_name] @@ -470,40 +558,58 @@ func init() { proto.RegisterType((*Store_RedisConfig)(nil), "feast.core.Store.RedisConfig") proto.RegisterType((*Store_BigQueryConfig)(nil), "feast.core.Store.BigQueryConfig") proto.RegisterType((*Store_CassandraConfig)(nil), "feast.core.Store.CassandraConfig") + proto.RegisterMapType((map[string]string)(nil), "feast.core.Store.CassandraConfig.ReplicationOptionsEntry") proto.RegisterType((*Store_Subscription)(nil), "feast.core.Store.Subscription") } -func init() { proto.RegisterFile("feast/core/Store.proto", fileDescriptor_4b177bc9ccf64875) } +func init() { + proto.RegisterFile("feast/core/Store.proto", fileDescriptor_4b177bc9ccf64875) +} var fileDescriptor_4b177bc9ccf64875 = []byte{ - // 450 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x93, 0x4f, 0x6f, 0xd3, 0x30, - 0x18, 0xc6, 0x97, 0xfe, 0x59, 0x97, 0x37, 0xfd, 0x13, 0xf9, 0x80, 0xa2, 0xa2, 0xa1, 0xb0, 0x53, - 0x4f, 0xb1, 0x54, 0xc4, 0x81, 0x1b, 0x4d, 0x3b, 0x41, 0x04, 0xaa, 0x98, 0x0b, 0x93, 0xe0, 0x32, - 0xa5, 0x89, 0x97, 0x79, 0xd3, 0xe2, 0x60, 0xbb, 0x48, 0xfd, 0xa8, 0x7c, 0x1b, 0x64, 0x27, 0x69, - 0x53, 0xda, 0xc3, 0x2e, 0x91, 0xfd, 0xbc, 0xcf, 0xf3, 0xcb, 0x2b, 0xdb, 0x2f, 0xbc, 0xba, 0xa7, - 0xb1, 0x54, 0x38, 0xe1, 0x82, 0xe2, 0x95, 0xe2, 0x82, 0x06, 0x85, 0xe0, 0x8a, 0x23, 0x30, 0x7a, - 0xa0, 0xf5, 0xab, 0xbf, 0x5d, 0xe8, 0x9a, 0x1a, 0x42, 0xd0, 0xc9, 0xe3, 0x67, 0xea, 0x59, 0xbe, - 0x35, 0xb1, 0x89, 0x59, 0x23, 0x0c, 0x1d, 0xb5, 0x2d, 0xa8, 0xd7, 0xf2, 0xad, 0xc9, 0x70, 0xfa, - 0x3a, 0xd8, 0x07, 0x83, 0x12, 0x68, 0xbe, 0xdf, 0xb7, 0x05, 0x25, 0xc6, 0x88, 0x16, 0x30, 0x90, - 0x9b, 0xb5, 0x4c, 0x04, 0x2b, 0x14, 0xe3, 0xb9, 0xf4, 0x3a, 0x7e, 0x7b, 0xe2, 0x4c, 0xdf, 0x9c, - 0x48, 0x36, 0x6c, 0xe4, 0x30, 0x84, 0x42, 0xe8, 0x0b, 0x9a, 0x32, 0x79, 0x97, 0xf0, 0xfc, 0x9e, - 0x65, 0x9e, 0xe3, 0x5b, 0x13, 0x67, 0x7a, 0x79, 0x0c, 0x21, 0xda, 0x35, 0x37, 0xa6, 0xcf, 0x67, - 0xc4, 0x11, 0xfb, 0x2d, 0xfa, 0x02, 0xa3, 0x35, 0xcb, 0x7e, 0x6f, 0xa8, 0xd8, 0xd6, 0x98, 0xbe, - 0xc1, 0xf8, 0xc7, 0x98, 0x90, 0x65, 0x37, 0xda, 0xb8, 0x23, 0x0d, 0xeb, 0x68, 0x05, 0x5b, 0x82, - 0x9b, 0xc4, 0x52, 0xc6, 0x79, 0x2a, 0xe2, 0x9a, 0x36, 0x30, 0xb4, 0xb7, 0xc7, 0xb4, 0x79, 0xed, - 0xdc, 0xe1, 0x46, 0xc9, 0xa1, 0x34, 0x7e, 0x0f, 0x4e, 0xa3, 0x75, 0x7d, 0xf4, 0x0f, 0x5c, 0xaa, - 0xfa, 0xe8, 0xf5, 0x5a, 0x6b, 0x05, 0x17, 0xca, 0x1c, 0x7d, 0x97, 0x98, 0xf5, 0x78, 0x09, 0xc3, - 0xc3, 0x56, 0xd1, 0x25, 0x40, 0x21, 0xf8, 0x23, 0x4d, 0xd4, 0x1d, 0x4b, 0xab, 0xbc, 0x5d, 0x29, - 0x51, 0xaa, 0xcb, 0x69, 0xac, 0x62, 0x49, 0x4d, 0xb9, 0x55, 0x96, 0x2b, 0x25, 0x4a, 0xc7, 0x1f, - 0x60, 0xf4, 0x5f, 0xb3, 0x2f, 0x6e, 0xe5, 0x16, 0xfa, 0xcd, 0x1b, 0x44, 0x1e, 0xf4, 0xaa, 0xdf, - 0x7a, 0x6d, 0x13, 0xad, 0xb7, 0x27, 0xdf, 0x95, 0x07, 0xbd, 0x3f, 0x54, 0x48, 0xc6, 0xf3, 0xaa, - 0xa9, 0x7a, 0x7b, 0xf5, 0x11, 0xec, 0xdd, 0x9b, 0x42, 0x0e, 0xf4, 0xa2, 0xe5, 0xed, 0xec, 0x6b, - 0xb4, 0x70, 0xcf, 0x90, 0x0d, 0x5d, 0x72, 0xbd, 0x88, 0x56, 0xae, 0x85, 0xfa, 0x70, 0x11, 0x46, - 0x9f, 0x6e, 0x7e, 0x5c, 0x93, 0x9f, 0x6e, 0x0b, 0x0d, 0xc0, 0x9e, 0xcf, 0x56, 0xab, 0xd9, 0x72, - 0x41, 0x66, 0x6e, 0x3b, 0xbc, 0x80, 0xf3, 0xf2, 0x86, 0xc2, 0x08, 0x1a, 0x2f, 0x3d, 0x04, 0xc3, - 0xfd, 0xa6, 0x27, 0xe0, 0x17, 0xce, 0x98, 0x7a, 0xd8, 0xac, 0x83, 0x84, 0x3f, 0xe3, 0x8c, 0x3f, - 0xd2, 0x27, 0x5c, 0x8e, 0x8a, 0x4c, 0x9f, 0x70, 0xc6, 0xb1, 0x19, 0x13, 0x89, 0xf7, 0xe3, 0xb3, - 0x3e, 0x37, 0xd2, 0xbb, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x56, 0xfe, 0x58, 0x14, 0x53, 0x03, - 0x00, 0x00, + // 699 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x54, 0x5f, 0x6f, 0xda, 0x48, + 0x10, 0x8f, 0xf9, 0x13, 0x60, 0x4c, 0x00, 0xed, 0x9d, 0xee, 0x7c, 0x9c, 0x92, 0xe3, 0xf2, 0x72, + 0x3c, 0x9c, 0x6c, 0x29, 0x7d, 0x69, 0xf3, 0x54, 0x08, 0xa8, 0x41, 0x6d, 0x69, 0x63, 0xd2, 0x48, + 0xed, 0x8b, 0xb5, 0xb6, 0x17, 0xc7, 0xc1, 0x78, 0xdd, 0xdd, 0x25, 0x0a, 0xef, 0xfd, 0x3a, 0xfd, + 0x06, 0xfd, 0x70, 0xd5, 0xae, 0xd7, 0x40, 0x4a, 0xaa, 0xbe, 0x58, 0xbb, 0xbf, 0xf9, 0xcd, 0xcf, + 0x33, 0xb3, 0x33, 0x03, 0x7f, 0xcc, 0x09, 0xe6, 0xc2, 0x09, 0x28, 0x23, 0xce, 0x4c, 0x50, 0x46, + 0xec, 0x8c, 0x51, 0x41, 0x11, 0x28, 0xdc, 0x96, 0x78, 0xf7, 0x24, 0xa2, 0x34, 0x4a, 0x88, 0xa3, + 0x2c, 0xfe, 0x6a, 0xee, 0x84, 0x2b, 0x86, 0x45, 0x4c, 0xd3, 0x9c, 0x7b, 0xfa, 0xb5, 0x01, 0x55, + 0xe5, 0x8b, 0x10, 0x54, 0x52, 0xbc, 0x24, 0x96, 0xd1, 0x33, 0xfa, 0x0d, 0x57, 0x9d, 0x91, 0x03, + 0x15, 0xb1, 0xce, 0x88, 0x55, 0xea, 0x19, 0xfd, 0xd6, 0xd9, 0xdf, 0xf6, 0x56, 0xd8, 0xce, 0x7f, + 0xa8, 0xbe, 0xd7, 0xeb, 0x8c, 0xb8, 0x8a, 0x88, 0x46, 0x70, 0xc4, 0x57, 0x3e, 0x0f, 0x58, 0x9c, + 0xc9, 0x9f, 0x70, 0xab, 0xd2, 0x2b, 0xf7, 0xcd, 0xb3, 0x93, 0x27, 0x3c, 0x77, 0x68, 0xee, 0x63, + 0x27, 0x34, 0x84, 0x26, 0x23, 0x61, 0xcc, 0xbd, 0x80, 0xa6, 0xf3, 0x38, 0xb2, 0xcc, 0x9e, 0xd1, + 0x37, 0xcf, 0x8e, 0xf7, 0x45, 0x5c, 0xc9, 0xba, 0x50, 0xa4, 0xcb, 0x03, 0xd7, 0x64, 0xdb, 0x2b, + 0x7a, 0x0d, 0x6d, 0x3f, 0x8e, 0x3e, 0xaf, 0x08, 0x5b, 0x17, 0x32, 0x4d, 0x25, 0xd3, 0xdb, 0x97, + 0x19, 0xc6, 0xd1, 0x95, 0x24, 0x6e, 0x94, 0x5a, 0x85, 0xab, 0x16, 0x9b, 0x42, 0x27, 0xc0, 0x9c, + 0xe3, 0x34, 0x64, 0xb8, 0x50, 0x3b, 0x52, 0x6a, 0xff, 0xee, 0xab, 0x5d, 0x14, 0xcc, 0x8d, 0x5c, + 0x3b, 0x78, 0x0c, 0x75, 0xbf, 0x18, 0x60, 0xee, 0xc4, 0x2e, 0x6b, 0x7f, 0x4b, 0xb9, 0x28, 0x6a, + 0x2f, 0xcf, 0x12, 0xcb, 0x28, 0x13, 0xaa, 0xf6, 0x55, 0x57, 0x9d, 0xd1, 0xff, 0x80, 0xe2, 0x34, + 0x16, 0x31, 0x4e, 0x3c, 0x1f, 0x07, 0x0b, 0x3a, 0x9f, 0x7b, 0x4b, 0x6e, 0x95, 0x15, 0xa3, 0xa3, + 0x2d, 0xc3, 0xdc, 0xf0, 0x96, 0xa3, 0x7f, 0xc0, 0x5c, 0xe2, 0x07, 0x8f, 0x11, 0xc1, 0x62, 0x22, + 0x9f, 0x42, 0xd2, 0x60, 0x89, 0x1f, 0xdc, 0x1c, 0xe9, 0x4e, 0xa1, 0xf5, 0x38, 0x75, 0x74, 0x0c, + 0x90, 0x31, 0x7a, 0x47, 0x02, 0xe1, 0xc5, 0xa1, 0x0e, 0xa7, 0xa1, 0x91, 0x49, 0x28, 0xcd, 0x21, + 0x16, 0x98, 0x13, 0x65, 0x2e, 0xe5, 0x66, 0x8d, 0x4c, 0xc2, 0xee, 0xb7, 0x32, 0xb4, 0x7f, 0xc8, + 0x1e, 0xfd, 0x07, 0x6d, 0x9f, 0x52, 0xc1, 0x05, 0xc3, 0x99, 0x27, 0x13, 0xe3, 0x5a, 0xb6, 0xb5, + 0x81, 0x2f, 0x25, 0xfa, 0x64, 0xbe, 0x5d, 0xa8, 0x2f, 0xc8, 0x9a, 0x67, 0x38, 0x20, 0x2a, 0xcb, + 0x86, 0xbb, 0xb9, 0xcb, 0x58, 0x04, 0xf6, 0x13, 0xe2, 0xa9, 0xae, 0xad, 0xe4, 0xb1, 0x28, 0x64, + 0x2a, 0x5b, 0xf7, 0x0e, 0x7e, 0x63, 0x24, 0x4b, 0xe2, 0x40, 0x75, 0xbb, 0x47, 0x75, 0x3f, 0x56, + 0x55, 0x3f, 0xbe, 0xf8, 0xe5, 0xab, 0xd9, 0xee, 0xd6, 0xf9, 0x5d, 0xee, 0x3b, 0x4e, 0x05, 0x5b, + 0xbb, 0x88, 0xed, 0x19, 0xd0, 0x39, 0x98, 0x21, 0x99, 0xe3, 0x55, 0x22, 0x3c, 0x21, 0x12, 0xeb, + 0x50, 0x75, 0xc6, 0x5f, 0x76, 0x3e, 0x7a, 0x76, 0x31, 0x7a, 0xf6, 0x48, 0x8f, 0x9e, 0x0b, 0x9a, + 0x7d, 0x2d, 0x12, 0xd4, 0x03, 0xf3, 0x9e, 0x30, 0x1e, 0xd3, 0x34, 0x21, 0x9c, 0x5b, 0xb5, 0x9e, + 0xd1, 0xaf, 0xbb, 0xbb, 0x90, 0x64, 0x04, 0x34, 0xe5, 0x31, 0x17, 0x24, 0x0d, 0xd6, 0x56, 0x5d, + 0x65, 0xba, 0x0b, 0x75, 0xc7, 0xf0, 0xe7, 0x4f, 0xc2, 0x45, 0x1d, 0x28, 0x2f, 0xc8, 0x5a, 0x97, + 0x5c, 0x1e, 0xd1, 0xef, 0x50, 0xbd, 0xc7, 0xc9, 0x8a, 0xe8, 0xe7, 0xcb, 0x2f, 0xe7, 0xa5, 0xe7, + 0x46, 0xf7, 0x06, 0x9a, 0xbb, 0x53, 0x89, 0x2c, 0xa8, 0xe9, 0xa7, 0xd7, 0xc5, 0x2f, 0xae, 0x4f, + 0xee, 0x0a, 0x0b, 0x6a, 0x3a, 0x6a, 0xad, 0x5c, 0x5c, 0x4f, 0x5f, 0x42, 0x63, 0xb3, 0x27, 0x90, + 0x09, 0xb5, 0xc9, 0xf4, 0x66, 0xf0, 0x66, 0x32, 0xea, 0x1c, 0xa0, 0x06, 0x54, 0xdd, 0xf1, 0x68, + 0x32, 0xeb, 0x18, 0xa8, 0x09, 0xf5, 0xe1, 0xe4, 0xd5, 0xd5, 0x87, 0xb1, 0xfb, 0xb1, 0x53, 0x42, + 0x47, 0xd0, 0xb8, 0x18, 0xcc, 0x66, 0x83, 0xe9, 0xc8, 0x1d, 0x74, 0xca, 0xc3, 0x3a, 0x1c, 0xe6, + 0x53, 0x37, 0x9c, 0xc0, 0xce, 0x76, 0x1b, 0x82, 0xd2, 0x7d, 0x2f, 0x0b, 0xfc, 0xc9, 0x89, 0x62, + 0x71, 0xbb, 0xf2, 0xed, 0x80, 0x2e, 0x9d, 0x88, 0xde, 0x91, 0x85, 0x93, 0xaf, 0x47, 0x1e, 0x2e, + 0x9c, 0x88, 0xe6, 0x0b, 0x90, 0x3b, 0xdb, 0x95, 0xe9, 0x1f, 0x2a, 0xe8, 0xd9, 0xf7, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x65, 0x5a, 0xb3, 0xd3, 0x47, 0x05, 0x00, 0x00, } diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index 212e8606ce7..1cde2f358dd 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -886,7 +886,9 @@ func init() { proto.RegisterType((*DatasetSource_FileSource)(nil), "feast.serving.DatasetSource.FileSource") } -func init() { proto.RegisterFile("feast/serving/ServingService.proto", fileDescriptor_0c1ba93cf29a8d9d) } +func init() { + proto.RegisterFile("feast/serving/ServingService.proto", fileDescriptor_0c1ba93cf29a8d9d) +} var fileDescriptor_0c1ba93cf29a8d9d = []byte{ // 1101 bytes of a gzipped FileDescriptorProto @@ -963,11 +965,11 @@ var fileDescriptor_0c1ba93cf29a8d9d = []byte{ // Reference imports to suppress errors if they are not otherwise used. var _ context.Context -var _ grpc.ClientConn +var _ grpc.ClientConnInterface // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 +const _ = grpc.SupportPackageIsVersion6 // ServingServiceClient is the client API for ServingService service. // @@ -991,10 +993,10 @@ type ServingServiceClient interface { } type servingServiceClient struct { - cc *grpc.ClientConn + cc grpc.ClientConnInterface } -func NewServingServiceClient(cc *grpc.ClientConn) ServingServiceClient { +func NewServingServiceClient(cc grpc.ClientConnInterface) ServingServiceClient { return &servingServiceClient{cc} } diff --git a/sdk/go/protos/feast/storage/Redis.pb.go b/sdk/go/protos/feast/storage/Redis.pb.go index 55cf42becd8..2c10b1de206 100644 --- a/sdk/go/protos/feast/storage/Redis.pb.go +++ b/sdk/go/protos/feast/storage/Redis.pb.go @@ -25,7 +25,8 @@ type RedisKey struct { // FeatureSet this row belongs to, this is defined as featureSetName:version. FeatureSet string `protobuf:"bytes,2,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` // List of fields containing entity names and their respective values - // contained within this feature row. + // contained within this feature row. The entities should be sorted + // by the entity name alphabetically in ascending order. Entities []*types.Field `protobuf:"bytes,3,rep,name=entities,proto3" json:"entities,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -75,7 +76,9 @@ func init() { proto.RegisterType((*RedisKey)(nil), "feast.storage.RedisKey") } -func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_64e898a359fc9e5d) } +func init() { + proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_64e898a359fc9e5d) +} var fileDescriptor_64e898a359fc9e5d = []byte{ // 193 bytes of a gzipped FileDescriptorProto diff --git a/sdk/go/protos/feast/types/FeatureRow.pb.go b/sdk/go/protos/feast/types/FeatureRow.pb.go index dd9aa6b8c96..26868ebdd0b 100644 --- a/sdk/go/protos/feast/types/FeatureRow.pb.go +++ b/sdk/go/protos/feast/types/FeatureRow.pb.go @@ -29,7 +29,7 @@ type FeatureRow struct { // will use to perform joins, determine latest values, and coalesce rows. EventTimestamp *timestamp.Timestamp `protobuf:"bytes,3,opt,name=event_timestamp,json=eventTimestamp,proto3" json:"event_timestamp,omitempty"` // Complete reference to the featureSet this featureRow belongs to, in the form of - // featureSetName:version. This value will be used by the feast ingestion job to filter + // /:. This value will be used by the feast ingestion job to filter // rows, and write the values to the correct tables. FeatureSet string `protobuf:"bytes,6,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -87,7 +87,9 @@ func init() { proto.RegisterType((*FeatureRow)(nil), "feast.types.FeatureRow") } -func init() { proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_fbbea9c89787d1c7) } +func init() { + proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_fbbea9c89787d1c7) +} var fileDescriptor_fbbea9c89787d1c7 = []byte{ // 238 bytes of a gzipped FileDescriptorProto diff --git a/sdk/go/protos/feast/types/Field.pb.go b/sdk/go/protos/feast/types/Field.pb.go index 345b5259997..0666d9bf1fb 100644 --- a/sdk/go/protos/feast/types/Field.pb.go +++ b/sdk/go/protos/feast/types/Field.pb.go @@ -71,7 +71,9 @@ func init() { proto.RegisterType((*Field)(nil), "feast.types.Field") } -func init() { proto.RegisterFile("feast/types/Field.proto", fileDescriptor_8c568a78dfaa9ca9) } +func init() { + proto.RegisterFile("feast/types/Field.proto", fileDescriptor_8c568a78dfaa9ca9) +} var fileDescriptor_8c568a78dfaa9ca9 = []byte{ // 165 bytes of a gzipped FileDescriptorProto diff --git a/sdk/go/protos/feast/types/Value.pb.go b/sdk/go/protos/feast/types/Value.pb.go index 3f9808b994f..f6ae73c2de2 100644 --- a/sdk/go/protos/feast/types/Value.pb.go +++ b/sdk/go/protos/feast/types/Value.pb.go @@ -664,7 +664,9 @@ func init() { proto.RegisterType((*BoolList)(nil), "feast.types.BoolList") } -func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_47c504407d284ecc) } +func init() { + proto.RegisterFile("feast/types/Value.proto", fileDescriptor_47c504407d284ecc) +} var fileDescriptor_47c504407d284ecc = []byte{ // 600 bytes of a gzipped FileDescriptorProto diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml index 2970dae3ee2..e8a82a485fc 100644 --- a/sdk/java/pom.xml +++ b/sdk/java/pom.xml @@ -21,6 +21,12 @@ + + dev.feast + datatypes-java + ${project.version} + + io.grpc @@ -79,10 +85,6 @@ - - org.xolstice.maven.plugins - protobuf-maven-plugin - org.apache.maven.plugins diff --git a/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java b/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java index 075c570c4e9..874196e92bd 100644 --- a/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java +++ b/sdk/java/src/main/java/com/gojek/feast/RequestUtil.java @@ -61,7 +61,7 @@ public static List createFeatureRefs( "Feature id '%s' contains invalid version. Expected format: /:.", featureRefString)); } - } else if (projectSplit.length == 1) { + } else if (featureSplit.length == 1) { name = featureSplit[0]; } else { throw new IllegalArgumentException( diff --git a/sdk/java/src/main/java/com/gojek/feast/Row.java b/sdk/java/src/main/java/com/gojek/feast/Row.java index 9366fe1bb03..ceef139aa13 100644 --- a/sdk/java/src/main/java/com/gojek/feast/Row.java +++ b/sdk/java/src/main/java/com/gojek/feast/Row.java @@ -31,7 +31,7 @@ @SuppressWarnings("UnusedReturnValue") public class Row { private Timestamp entity_timestamp; - private Map fields; + private HashMap fields; public static Row create() { Row row = new Row(); diff --git a/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java b/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java index 1c58e9435c6..3b9429ad8f6 100644 --- a/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java +++ b/sdk/java/src/test/java/com/gojek/feast/RequestUtilTest.java @@ -60,7 +60,7 @@ private static Stream provideValidFeatureIds() { Arrays.asList( "driver_project/driver_id:1", "driver_project/driver_name:1", - "booking_project/driver_name:1"), + "booking_project/driver_name"), Arrays.asList( FeatureReference.newBuilder() .setProject("driver_project") @@ -74,7 +74,6 @@ private static Stream provideValidFeatureIds() { .build(), FeatureReference.newBuilder() .setProject("booking_project") - .setVersion(1) .setName("driver_name") .build()))); } diff --git a/sdk/python/docs/requirements.txt b/sdk/python/docs/requirements.txt index a7e825b5d49..3a9bbfee453 100644 --- a/sdk/python/docs/requirements.txt +++ b/sdk/python/docs/requirements.txt @@ -18,6 +18,14 @@ fastavro==0.21.* grpcio-testing==1.* pytest-ordering==0.6.* pyarrow +Click==7.* +toml==0.10.* +tqdm==4.* +confluent_kafka +google +pandavro==1.5.* +kafka-python==1.* +tabulate==0.8.* Sphinx==2.* sphinx-autodoc-napoleon-typehints sphinx-autodoc-typehints @@ -28,4 +36,4 @@ sphinxcontrib-htmlhelp sphinxcontrib-jsmath sphinxcontrib-napoleon sphinxcontrib-qthelp -sphinxcontrib-serializinghtml +sphinxcontrib-serializinghtml \ No newline at end of file diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 601f41d4b11..8e8f185d038 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -17,7 +17,6 @@ import click from feast import config as feast_config from feast.client import Client -from feast.resource import ResourceFactory from feast.feature_set import FeatureSet import toml import pkg_resources @@ -147,17 +146,25 @@ def feature_set_list(): print(tabulate(table, headers=["NAME", "VERSION"], tablefmt="plain")) -@feature_set.command("create") -@click.argument("name") -def feature_set_create(name): +@feature_set.command("apply") +@click.option( + "--filename", + "-f", + help="Path to a feature set configuration file that will be applied", + type=click.Path(exists=True), +) +def feature_set_create(filename): """ - Create a feature set + Create or update a feature set """ + + feature_sets = [FeatureSet.from_dict(fs_dict) for fs_dict in yaml_loader(filename)] + feast_client = Client( core_url=feast_config.get_config_property_or_fail("core_url") ) # type: Client - feast_client.apply(FeatureSet(name=name)) + feast_client.apply(feature_sets) @feature_set.command("describe") @@ -264,29 +271,5 @@ def ingest(name, version, filename, file_type): feature_set.ingest_file(file_path=filename) -@cli.command() -@click.option( - "--filename", - "-f", - help="Path to the configuration file that will be applied", - type=click.Path(exists=True), -) -def apply(filename): - """ - Apply a configuration to a resource by filename or stdin - """ - - resources = [ - ResourceFactory.get_resource(res_dict["kind"]).from_dict(res_dict) - for res_dict in yaml_loader(filename) - ] - - feast_client = Client( - core_url=feast_config.get_config_property_or_fail("core_url") - ) # type: Client - - feast_client.apply(resources) - - if __name__ == "__main__": cli() diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index a68f0fe2bc5..8600d8693c5 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -21,7 +21,6 @@ from collections import OrderedDict from math import ceil from typing import Dict, List, Tuple, Union, Optional -from typing import List from urllib.parse import urlparse import fastavro @@ -29,6 +28,7 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq + from feast.core.CoreService_pb2 import ( GetFeastCoreVersionRequest, ListFeatureSetsResponse, @@ -48,11 +48,11 @@ from feast.core.FeatureSet_pb2 import FeatureSetStatus from feast.feature_set import FeatureSet, Entity from feast.job import Job -from feast.serving.ServingService_pb2 import FeatureReference from feast.loaders.abstract_producer import get_producer from feast.loaders.file import export_source_to_staging_location from feast.loaders.ingest import KAFKA_CHUNK_PRODUCTION_TIMEOUT from feast.loaders.ingest import get_feature_row_chunks +from feast.serving.ServingService_pb2 import FeatureReference from feast.serving.ServingService_pb2 import GetFeastServingInfoResponse from feast.serving.ServingService_pb2 import ( GetOnlineFeaturesRequest, @@ -69,9 +69,11 @@ GRPC_CONNECTION_TIMEOUT_DEFAULT = 3 # type: int GRPC_CONNECTION_TIMEOUT_APPLY = 600 # type: int -FEAST_SERVING_URL_ENV_KEY = "FEAST_SERVING_URL" # type: str -FEAST_CORE_URL_ENV_KEY = "FEAST_CORE_URL" # type: str -FEAST_PROJECT_ENV_KEY = "FEAST_PROJECT" # type: str +FEAST_CORE_URL_ENV_KEY = "FEAST_CORE_URL" +FEAST_SERVING_URL_ENV_KEY = "FEAST_SERVING_URL" +FEAST_PROJECT_ENV_KEY = "FEAST_PROJECT" +FEAST_CORE_SECURE_ENV_KEY = "FEAST_CORE_SECURE" +FEAST_SERVING_SECURE_ENV_KEY = "FEAST_SERVING_SECURE" BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS = 300 CPU_COUNT = os.cpu_count() # type: int @@ -82,7 +84,8 @@ class Client: """ def __init__( - self, core_url: str = None, serving_url: str = None, project: str = None + self, core_url: str = None, serving_url: str = None, project: str = None, + core_secure: bool = None, serving_secure: bool = None ): """ The Feast Client should be initialized with at least one service url @@ -91,10 +94,14 @@ def __init__( core_url: Feast Core URL. Used to manage features serving_url: Feast Serving URL. Used to retrieve features project: Sets the active project. This field is optional. - """ - self._core_url = core_url - self._serving_url = serving_url - self._project = project + core_secure: Use client-side SSL/TLS for Core gRPC API + serving_secure: Use client-side SSL/TLS for Serving gRPC API + """ + self._core_url: str = core_url + self._serving_url: str = serving_url + self._project: str = project + self._core_secure: bool = core_secure + self._serving_secure: bool = serving_secure self.__core_channel: grpc.Channel = None self.__serving_channel: grpc.Channel = None self._core_service_stub: CoreServiceStub = None @@ -149,6 +156,52 @@ def serving_url(self, value: str): """ self._serving_url = value + @property + def core_secure(self) -> bool: + """ + Retrieve Feast Core client-side SSL/TLS setting + + Returns: + Whether client-side SSL/TLS is enabled + """ + + if self._core_secure is not None: + return self._core_secure + return os.getenv(FEAST_CORE_SECURE_ENV_KEY, "").lower() is "true" + + @core_secure.setter + def core_secure(self, value: bool): + """ + Set the Feast Core client-side SSL/TLS setting + + Args: + value: True to enable client-side SSL/TLS + """ + self._core_secure = value + + @property + def serving_secure(self) -> bool: + """ + Retrieve Feast Serving client-side SSL/TLS setting + + Returns: + Whether client-side SSL/TLS is enabled + """ + + if self._serving_secure is not None: + return self._serving_secure + return os.getenv(FEAST_SERVING_SECURE_ENV_KEY, "").lower() is "true" + + @serving_secure.setter + def serving_secure(self, value: bool): + """ + Set the Feast Serving client-side SSL/TLS setting + + Args: + value: True to enable client-side SSL/TLS + """ + self._serving_secure = value + def version(self): """ Returns version information from Feast Core and Feast Serving @@ -185,7 +238,10 @@ def _connect_core(self, skip_if_connected: bool = True): raise ValueError("Please set Feast Core URL.") if self.__core_channel is None: - self.__core_channel = grpc.insecure_channel(self.core_url) + if self.core_secure or self.core_url.endswith(":443"): + self.__core_channel = grpc.secure_channel(self.core_url, grpc.ssl_channel_credentials()) + else: + self.__core_channel = grpc.insecure_channel(self.core_url) try: grpc.channel_ready_future(self.__core_channel).result( @@ -214,7 +270,10 @@ def _connect_serving(self, skip_if_connected=True): raise ValueError("Please set Feast Serving URL.") if self.__serving_channel is None: - self.__serving_channel = grpc.insecure_channel(self.serving_url) + if self.serving_secure or self.serving_url.endswith(":443"): + self.__serving_channel = grpc.secure_channel(self.serving_url, grpc.ssl_channel_credentials()) + else: + self.__serving_channel = grpc.insecure_channel(self.serving_url) try: grpc.channel_ready_future(self.__serving_channel).result( @@ -615,7 +674,7 @@ def ingest( Loads feature data into Feast for a specific feature set. Args: - feature_set (typing.Union[str, FeatureSet]): + feature_set (typing.Union[str, feast.feature_set.FeatureSet]): Feature set object or the string name of the feature set (without a version). @@ -681,6 +740,7 @@ def ingest( if timeout is not None and time.time() - current_time >= timeout: raise TimeoutError("Timed out waiting for feature set to be ready") feature_set = self.get_feature_set(name, version) + print(feature_set) if ( feature_set is not None and feature_set.status == FeatureSetStatus.STATUS_READY diff --git a/sdk/python/feast/core/CoreService_pb2.py b/sdk/python/feast/core/CoreService_pb2.py index 858703d7f3e..cdec88b8230 100644 --- a/sdk/python/feast/core/CoreService_pb2.py +++ b/sdk/python/feast/core/CoreService_pb2.py @@ -2,8 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/core/CoreService.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -21,8 +19,8 @@ name='feast/core/CoreService.proto', package='feast.core', syntax='proto3', - serialized_options=_b('\n\nfeast.coreB\020CoreServiceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core'), - serialized_pb=_b('\n\x1c\x66\x65\x61st/core/CoreService.proto\x12\nfeast.core\x1a\x1b\x66\x65\x61st/core/FeatureSet.proto\x1a\x16\x66\x65\x61st/core/Store.proto\"F\n\x14GetFeatureSetRequest\x12\x0f\n\x07project\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\x05\"D\n\x15GetFeatureSetResponse\x12+\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x16.feast.core.FeatureSet\"\xa5\x01\n\x16ListFeatureSetsRequest\x12\x39\n\x06\x66ilter\x18\x01 \x01(\x0b\x32).feast.core.ListFeatureSetsRequest.Filter\x1aP\n\x06\x46ilter\x12\x0f\n\x07project\x18\x03 \x01(\t\x12\x18\n\x10\x66\x65\x61ture_set_name\x18\x01 \x01(\t\x12\x1b\n\x13\x66\x65\x61ture_set_version\x18\x02 \x01(\t\"G\n\x17ListFeatureSetsResponse\x12,\n\x0c\x66\x65\x61ture_sets\x18\x01 \x03(\x0b\x32\x16.feast.core.FeatureSet\"a\n\x11ListStoresRequest\x12\x34\n\x06\x66ilter\x18\x01 \x01(\x0b\x32$.feast.core.ListStoresRequest.Filter\x1a\x16\n\x06\x46ilter\x12\x0c\n\x04name\x18\x01 \x01(\t\"6\n\x12ListStoresResponse\x12 \n\x05store\x18\x01 \x03(\x0b\x32\x11.feast.core.Store\"E\n\x16\x41pplyFeatureSetRequest\x12+\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x16.feast.core.FeatureSet\"\xb3\x01\n\x17\x41pplyFeatureSetResponse\x12+\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x16.feast.core.FeatureSet\x12:\n\x06status\x18\x02 \x01(\x0e\x32*.feast.core.ApplyFeatureSetResponse.Status\"/\n\x06Status\x12\r\n\tNO_CHANGE\x10\x00\x12\x0b\n\x07\x43REATED\x10\x01\x12\t\n\x05\x45RROR\x10\x02\"\x1c\n\x1aGetFeastCoreVersionRequest\".\n\x1bGetFeastCoreVersionResponse\x12\x0f\n\x07version\x18\x01 \x01(\t\"6\n\x12UpdateStoreRequest\x12 \n\x05store\x18\x01 \x01(\x0b\x32\x11.feast.core.Store\"\x95\x01\n\x13UpdateStoreResponse\x12 \n\x05store\x18\x01 \x01(\x0b\x32\x11.feast.core.Store\x12\x36\n\x06status\x18\x02 \x01(\x0e\x32&.feast.core.UpdateStoreResponse.Status\"$\n\x06Status\x12\r\n\tNO_CHANGE\x10\x00\x12\x0b\n\x07UPDATED\x10\x01\"$\n\x14\x43reateProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x17\n\x15\x43reateProjectResponse\"%\n\x15\x41rchiveProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x18\n\x16\x41rchiveProjectResponse\"\x15\n\x13ListProjectsRequest\"(\n\x14ListProjectsResponse\x12\x10\n\x08projects\x18\x01 \x03(\t2\xa2\x06\n\x0b\x43oreService\x12\x66\n\x13GetFeastCoreVersion\x12&.feast.core.GetFeastCoreVersionRequest\x1a\'.feast.core.GetFeastCoreVersionResponse\x12T\n\rGetFeatureSet\x12 .feast.core.GetFeatureSetRequest\x1a!.feast.core.GetFeatureSetResponse\x12Z\n\x0fListFeatureSets\x12\".feast.core.ListFeatureSetsRequest\x1a#.feast.core.ListFeatureSetsResponse\x12K\n\nListStores\x12\x1d.feast.core.ListStoresRequest\x1a\x1e.feast.core.ListStoresResponse\x12Z\n\x0f\x41pplyFeatureSet\x12\".feast.core.ApplyFeatureSetRequest\x1a#.feast.core.ApplyFeatureSetResponse\x12N\n\x0bUpdateStore\x12\x1e.feast.core.UpdateStoreRequest\x1a\x1f.feast.core.UpdateStoreResponse\x12T\n\rCreateProject\x12 .feast.core.CreateProjectRequest\x1a!.feast.core.CreateProjectResponse\x12W\n\x0e\x41rchiveProject\x12!.feast.core.ArchiveProjectRequest\x1a\".feast.core.ArchiveProjectResponse\x12Q\n\x0cListProjects\x12\x1f.feast.core.ListProjectsRequest\x1a .feast.core.ListProjectsResponseBO\n\nfeast.coreB\x10\x43oreServiceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3') + serialized_options=b'\n\nfeast.coreB\020CoreServiceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core', + serialized_pb=b'\n\x1c\x66\x65\x61st/core/CoreService.proto\x12\nfeast.core\x1a\x1b\x66\x65\x61st/core/FeatureSet.proto\x1a\x16\x66\x65\x61st/core/Store.proto\"F\n\x14GetFeatureSetRequest\x12\x0f\n\x07project\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\x05\"D\n\x15GetFeatureSetResponse\x12+\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x16.feast.core.FeatureSet\"\xa5\x01\n\x16ListFeatureSetsRequest\x12\x39\n\x06\x66ilter\x18\x01 \x01(\x0b\x32).feast.core.ListFeatureSetsRequest.Filter\x1aP\n\x06\x46ilter\x12\x0f\n\x07project\x18\x03 \x01(\t\x12\x18\n\x10\x66\x65\x61ture_set_name\x18\x01 \x01(\t\x12\x1b\n\x13\x66\x65\x61ture_set_version\x18\x02 \x01(\t\"G\n\x17ListFeatureSetsResponse\x12,\n\x0c\x66\x65\x61ture_sets\x18\x01 \x03(\x0b\x32\x16.feast.core.FeatureSet\"a\n\x11ListStoresRequest\x12\x34\n\x06\x66ilter\x18\x01 \x01(\x0b\x32$.feast.core.ListStoresRequest.Filter\x1a\x16\n\x06\x46ilter\x12\x0c\n\x04name\x18\x01 \x01(\t\"6\n\x12ListStoresResponse\x12 \n\x05store\x18\x01 \x03(\x0b\x32\x11.feast.core.Store\"E\n\x16\x41pplyFeatureSetRequest\x12+\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x16.feast.core.FeatureSet\"\xb3\x01\n\x17\x41pplyFeatureSetResponse\x12+\n\x0b\x66\x65\x61ture_set\x18\x01 \x01(\x0b\x32\x16.feast.core.FeatureSet\x12:\n\x06status\x18\x02 \x01(\x0e\x32*.feast.core.ApplyFeatureSetResponse.Status\"/\n\x06Status\x12\r\n\tNO_CHANGE\x10\x00\x12\x0b\n\x07\x43REATED\x10\x01\x12\t\n\x05\x45RROR\x10\x02\"\x1c\n\x1aGetFeastCoreVersionRequest\".\n\x1bGetFeastCoreVersionResponse\x12\x0f\n\x07version\x18\x01 \x01(\t\"6\n\x12UpdateStoreRequest\x12 \n\x05store\x18\x01 \x01(\x0b\x32\x11.feast.core.Store\"\x95\x01\n\x13UpdateStoreResponse\x12 \n\x05store\x18\x01 \x01(\x0b\x32\x11.feast.core.Store\x12\x36\n\x06status\x18\x02 \x01(\x0e\x32&.feast.core.UpdateStoreResponse.Status\"$\n\x06Status\x12\r\n\tNO_CHANGE\x10\x00\x12\x0b\n\x07UPDATED\x10\x01\"$\n\x14\x43reateProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x17\n\x15\x43reateProjectResponse\"%\n\x15\x41rchiveProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x18\n\x16\x41rchiveProjectResponse\"\x15\n\x13ListProjectsRequest\"(\n\x14ListProjectsResponse\x12\x10\n\x08projects\x18\x01 \x03(\t2\xa2\x06\n\x0b\x43oreService\x12\x66\n\x13GetFeastCoreVersion\x12&.feast.core.GetFeastCoreVersionRequest\x1a\'.feast.core.GetFeastCoreVersionResponse\x12T\n\rGetFeatureSet\x12 .feast.core.GetFeatureSetRequest\x1a!.feast.core.GetFeatureSetResponse\x12Z\n\x0fListFeatureSets\x12\".feast.core.ListFeatureSetsRequest\x1a#.feast.core.ListFeatureSetsResponse\x12K\n\nListStores\x12\x1d.feast.core.ListStoresRequest\x1a\x1e.feast.core.ListStoresResponse\x12Z\n\x0f\x41pplyFeatureSet\x12\".feast.core.ApplyFeatureSetRequest\x1a#.feast.core.ApplyFeatureSetResponse\x12N\n\x0bUpdateStore\x12\x1e.feast.core.UpdateStoreRequest\x1a\x1f.feast.core.UpdateStoreResponse\x12T\n\rCreateProject\x12 .feast.core.CreateProjectRequest\x1a!.feast.core.CreateProjectResponse\x12W\n\x0e\x41rchiveProject\x12!.feast.core.ArchiveProjectRequest\x1a\".feast.core.ArchiveProjectResponse\x12Q\n\x0cListProjects\x12\x1f.feast.core.ListProjectsRequest\x1a .feast.core.ListProjectsResponseBO\n\nfeast.coreB\x10\x43oreServiceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3' , dependencies=[feast_dot_core_dot_FeatureSet__pb2.DESCRIPTOR,feast_dot_core_dot_Store__pb2.DESCRIPTOR,]) @@ -87,14 +85,14 @@ _descriptor.FieldDescriptor( name='project', full_name='feast.core.GetFeatureSetRequest.project', index=0, number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='name', full_name='feast.core.GetFeatureSetRequest.name', index=1, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -163,21 +161,21 @@ _descriptor.FieldDescriptor( name='project', full_name='feast.core.ListFeatureSetsRequest.Filter.project', index=0, number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='feature_set_name', full_name='feast.core.ListFeatureSetsRequest.Filter.feature_set_name', index=1, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='feature_set_version', full_name='feast.core.ListFeatureSetsRequest.Filter.feature_set_version', index=2, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -269,7 +267,7 @@ _descriptor.FieldDescriptor( name='name', full_name='feast.core.ListStoresRequest.Filter.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -455,7 +453,7 @@ _descriptor.FieldDescriptor( name='version', full_name='feast.core.GetFeastCoreVersionResponse.version', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -556,7 +554,7 @@ _descriptor.FieldDescriptor( name='name', full_name='feast.core.CreateProjectRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -611,7 +609,7 @@ _descriptor.FieldDescriptor( name='name', full_name='feast.core.ArchiveProjectRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), diff --git a/sdk/python/feast/core/CoreService_pb2.pyi b/sdk/python/feast/core/CoreService_pb2.pyi index 645226982ad..21ec92092d7 100644 --- a/sdk/python/feast/core/CoreService_pb2.pyi +++ b/sdk/python/feast/core/CoreService_pb2.pyi @@ -28,6 +28,7 @@ from typing import ( Optional as typing___Optional, Text as typing___Text, Tuple as typing___Tuple, + Union as typing___Union, cast as typing___cast, ) @@ -36,26 +37,38 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str +if sys.version_info < (3,): + builtin___buffer = buffer + builtin___unicode = unicode + + class GetFeatureSetRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... project = ... # type: typing___Text name = ... # type: typing___Text - version = ... # type: int + version = ... # type: builtin___int def __init__(self, *, project : typing___Optional[typing___Text] = None, name : typing___Optional[typing___Text] = None, - version : typing___Optional[int] = None, + version : typing___Optional[builtin___int] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetFeatureSetRequest: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"name",u"project",u"version"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetFeatureSetRequest: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"project",b"project",u"version",b"version"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetFeatureSetRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"project",b"project",u"version",b"version"]) -> None: ... +global___GetFeatureSetRequest = GetFeatureSetRequest class GetFeatureSetResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -67,16 +80,17 @@ class GetFeatureSetResponse(google___protobuf___message___Message): *, feature_set : typing___Optional[feast___core___FeatureSet_pb2___FeatureSet] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetFeatureSetResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetFeatureSetResponse: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetFeatureSetResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> None: ... +global___GetFeatureSetResponse = GetFeatureSetResponse class ListFeatureSetsRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -92,33 +106,36 @@ class ListFeatureSetsRequest(google___protobuf___message___Message): feature_set_name : typing___Optional[typing___Text] = None, feature_set_version : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ListFeatureSetsRequest.Filter: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"feature_set_name",u"feature_set_version",u"project"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> ListFeatureSetsRequest.Filter: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"feature_set_name",b"feature_set_name",u"feature_set_version",b"feature_set_version",u"project",b"project"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ListFeatureSetsRequest.Filter: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"feature_set_name",b"feature_set_name",u"feature_set_version",b"feature_set_version",u"project",b"project"]) -> None: ... + global___Filter = Filter @property - def filter(self) -> ListFeatureSetsRequest.Filter: ... + def filter(self) -> global___ListFeatureSetsRequest.Filter: ... def __init__(self, *, - filter : typing___Optional[ListFeatureSetsRequest.Filter] = None, + filter : typing___Optional[global___ListFeatureSetsRequest.Filter] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ListFeatureSetsRequest: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"filter"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"filter"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> ListFeatureSetsRequest: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ListFeatureSetsRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> None: ... +global___ListFeatureSetsRequest = ListFeatureSetsRequest class ListFeatureSetsResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -130,14 +147,16 @@ class ListFeatureSetsResponse(google___protobuf___message___Message): *, feature_sets : typing___Optional[typing___Iterable[feast___core___FeatureSet_pb2___FeatureSet]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ListFeatureSetsResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"feature_sets"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> ListFeatureSetsResponse: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"feature_sets",b"feature_sets"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ListFeatureSetsResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"feature_sets",b"feature_sets"]) -> None: ... +global___ListFeatureSetsResponse = ListFeatureSetsResponse class ListStoresRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -149,33 +168,36 @@ class ListStoresRequest(google___protobuf___message___Message): *, name : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ListStoresRequest.Filter: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"name"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> ListStoresRequest.Filter: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ListStoresRequest.Filter: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name"]) -> None: ... + global___Filter = Filter @property - def filter(self) -> ListStoresRequest.Filter: ... + def filter(self) -> global___ListStoresRequest.Filter: ... def __init__(self, *, - filter : typing___Optional[ListStoresRequest.Filter] = None, + filter : typing___Optional[global___ListStoresRequest.Filter] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ListStoresRequest: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"filter"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"filter"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> ListStoresRequest: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ListStoresRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"filter",b"filter"]) -> None: ... +global___ListStoresRequest = ListStoresRequest class ListStoresResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -187,14 +209,16 @@ class ListStoresResponse(google___protobuf___message___Message): *, store : typing___Optional[typing___Iterable[feast___core___Store_pb2___Store]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ListStoresResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"store"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> ListStoresResponse: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ListStoresResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> None: ... +global___ListStoresResponse = ListStoresResponse class ApplyFeatureSetRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -206,39 +230,41 @@ class ApplyFeatureSetRequest(google___protobuf___message___Message): *, feature_set : typing___Optional[feast___core___FeatureSet_pb2___FeatureSet] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ApplyFeatureSetRequest: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> ApplyFeatureSetRequest: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ApplyFeatureSetRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> None: ... +global___ApplyFeatureSetRequest = ApplyFeatureSetRequest class ApplyFeatureSetResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class Status(int): + class Status(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> ApplyFeatureSetResponse.Status: ... + def Value(cls, name: builtin___str) -> 'ApplyFeatureSetResponse.Status': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[ApplyFeatureSetResponse.Status]: ... + def values(cls) -> typing___List['ApplyFeatureSetResponse.Status']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, ApplyFeatureSetResponse.Status]]: ... - NO_CHANGE = typing___cast(ApplyFeatureSetResponse.Status, 0) - CREATED = typing___cast(ApplyFeatureSetResponse.Status, 1) - ERROR = typing___cast(ApplyFeatureSetResponse.Status, 2) - NO_CHANGE = typing___cast(ApplyFeatureSetResponse.Status, 0) - CREATED = typing___cast(ApplyFeatureSetResponse.Status, 1) - ERROR = typing___cast(ApplyFeatureSetResponse.Status, 2) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'ApplyFeatureSetResponse.Status']]: ... + NO_CHANGE = typing___cast('ApplyFeatureSetResponse.Status', 0) + CREATED = typing___cast('ApplyFeatureSetResponse.Status', 1) + ERROR = typing___cast('ApplyFeatureSetResponse.Status', 2) + NO_CHANGE = typing___cast('ApplyFeatureSetResponse.Status', 0) + CREATED = typing___cast('ApplyFeatureSetResponse.Status', 1) + ERROR = typing___cast('ApplyFeatureSetResponse.Status', 2) + global___Status = Status - status = ... # type: ApplyFeatureSetResponse.Status + status = ... # type: global___ApplyFeatureSetResponse.Status @property def feature_set(self) -> feast___core___FeatureSet_pb2___FeatureSet: ... @@ -246,28 +272,34 @@ class ApplyFeatureSetResponse(google___protobuf___message___Message): def __init__(self, *, feature_set : typing___Optional[feast___core___FeatureSet_pb2___FeatureSet] = None, - status : typing___Optional[ApplyFeatureSetResponse.Status] = None, + status : typing___Optional[global___ApplyFeatureSetResponse.Status] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ApplyFeatureSetResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"feature_set"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",u"status"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> ApplyFeatureSetResponse: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set",u"status",b"status"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ApplyFeatureSetResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"feature_set",b"feature_set",u"status",b"status"]) -> None: ... +global___ApplyFeatureSetResponse = ApplyFeatureSetResponse class GetFeastCoreVersionRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... def __init__(self, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetFeastCoreVersionRequest: ... + if sys.version_info >= (3,): + @classmethod + def FromString(cls, s: builtin___bytes) -> GetFeastCoreVersionRequest: ... + else: + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetFeastCoreVersionRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... +global___GetFeastCoreVersionRequest = GetFeastCoreVersionRequest class GetFeastCoreVersionResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -277,14 +309,16 @@ class GetFeastCoreVersionResponse(google___protobuf___message___Message): *, version : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetFeastCoreVersionResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"version"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetFeastCoreVersionResponse: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"version",b"version"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetFeastCoreVersionResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"version",b"version"]) -> None: ... +global___GetFeastCoreVersionResponse = GetFeastCoreVersionResponse class UpdateStoreRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -296,37 +330,39 @@ class UpdateStoreRequest(google___protobuf___message___Message): *, store : typing___Optional[feast___core___Store_pb2___Store] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> UpdateStoreRequest: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"store"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"store"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> UpdateStoreRequest: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> UpdateStoreRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> None: ... +global___UpdateStoreRequest = UpdateStoreRequest class UpdateStoreResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class Status(int): + class Status(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> UpdateStoreResponse.Status: ... + def Value(cls, name: builtin___str) -> 'UpdateStoreResponse.Status': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[UpdateStoreResponse.Status]: ... + def values(cls) -> typing___List['UpdateStoreResponse.Status']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, UpdateStoreResponse.Status]]: ... - NO_CHANGE = typing___cast(UpdateStoreResponse.Status, 0) - UPDATED = typing___cast(UpdateStoreResponse.Status, 1) - NO_CHANGE = typing___cast(UpdateStoreResponse.Status, 0) - UPDATED = typing___cast(UpdateStoreResponse.Status, 1) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'UpdateStoreResponse.Status']]: ... + NO_CHANGE = typing___cast('UpdateStoreResponse.Status', 0) + UPDATED = typing___cast('UpdateStoreResponse.Status', 1) + NO_CHANGE = typing___cast('UpdateStoreResponse.Status', 0) + UPDATED = typing___cast('UpdateStoreResponse.Status', 1) + global___Status = Status - status = ... # type: UpdateStoreResponse.Status + status = ... # type: global___UpdateStoreResponse.Status @property def store(self) -> feast___core___Store_pb2___Store: ... @@ -334,18 +370,19 @@ class UpdateStoreResponse(google___protobuf___message___Message): def __init__(self, *, store : typing___Optional[feast___core___Store_pb2___Store] = None, - status : typing___Optional[UpdateStoreResponse.Status] = None, + status : typing___Optional[global___UpdateStoreResponse.Status] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> UpdateStoreResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"store"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"status",u"store"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> UpdateStoreResponse: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"status",b"status",u"store",b"store"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> UpdateStoreResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"store",b"store"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"status",b"status",u"store",b"store"]) -> None: ... +global___UpdateStoreResponse = UpdateStoreResponse class CreateProjectRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -355,24 +392,31 @@ class CreateProjectRequest(google___protobuf___message___Message): *, name : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> CreateProjectRequest: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"name"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> CreateProjectRequest: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> CreateProjectRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name"]) -> None: ... +global___CreateProjectRequest = CreateProjectRequest class CreateProjectResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... def __init__(self, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> CreateProjectResponse: ... + if sys.version_info >= (3,): + @classmethod + def FromString(cls, s: builtin___bytes) -> CreateProjectResponse: ... + else: + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> CreateProjectResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... +global___CreateProjectResponse = CreateProjectResponse class ArchiveProjectRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -382,34 +426,46 @@ class ArchiveProjectRequest(google___protobuf___message___Message): *, name : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ArchiveProjectRequest: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"name"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> ArchiveProjectRequest: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ArchiveProjectRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name"]) -> None: ... +global___ArchiveProjectRequest = ArchiveProjectRequest class ArchiveProjectResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... def __init__(self, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ArchiveProjectResponse: ... + if sys.version_info >= (3,): + @classmethod + def FromString(cls, s: builtin___bytes) -> ArchiveProjectResponse: ... + else: + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ArchiveProjectResponse: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... +global___ArchiveProjectResponse = ArchiveProjectResponse class ListProjectsRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... def __init__(self, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ListProjectsRequest: ... + if sys.version_info >= (3,): + @classmethod + def FromString(cls, s: builtin___bytes) -> ListProjectsRequest: ... + else: + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ListProjectsRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... +global___ListProjectsRequest = ListProjectsRequest class ListProjectsResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -419,11 +475,13 @@ class ListProjectsResponse(google___protobuf___message___Message): *, projects : typing___Optional[typing___Iterable[typing___Text]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ListProjectsResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"projects"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> ListProjectsResponse: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"projects",b"projects"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ListProjectsResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"projects",b"projects"]) -> None: ... +global___ListProjectsResponse = ListProjectsResponse diff --git a/sdk/python/feast/core/FeatureSet_pb2.py b/sdk/python/feast/core/FeatureSet_pb2.py index 991220ccae5..74747b3432f 100644 --- a/sdk/python/feast/core/FeatureSet_pb2.py +++ b/sdk/python/feast/core/FeatureSet_pb2.py @@ -2,8 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/core/FeatureSet.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message @@ -18,16 +16,17 @@ from feast.core import Source_pb2 as feast_dot_core_dot_Source__pb2 from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from tensorflow_metadata.proto.v0 import schema_pb2 as tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='feast/core/FeatureSet.proto', package='feast.core', syntax='proto3', - serialized_options=_b('\n\nfeast.coreB\017FeatureSetProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core'), - serialized_pb=_b('\n\x1b\x66\x65\x61st/core/FeatureSet.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x17\x66\x65\x61st/core/Source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"`\n\nFeatureSet\x12(\n\x04spec\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureSetSpec\x12(\n\x04meta\x18\x02 \x01(\x0b\x32\x1a.feast.core.FeatureSetMeta\"\xe5\x01\n\x0e\x46\x65\x61tureSetSpec\x12\x0f\n\x07project\x18\x07 \x01(\t\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\x05\x12(\n\x08\x65ntities\x18\x03 \x03(\x0b\x32\x16.feast.core.EntitySpec\x12)\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x17.feast.core.FeatureSpec\x12*\n\x07max_age\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\"\n\x06source\x18\x06 \x01(\x0b\x32\x12.feast.core.Source\"K\n\nEntitySpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\"L\n\x0b\x46\x65\x61tureSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\"u\n\x0e\x46\x65\x61tureSetMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x06status\x18\x02 \x01(\x0e\x32\x1c.feast.core.FeatureSetStatus*L\n\x10\x46\x65\x61tureSetStatus\x12\x12\n\x0eSTATUS_INVALID\x10\x00\x12\x12\n\x0eSTATUS_PENDING\x10\x01\x12\x10\n\x0cSTATUS_READY\x10\x02\x42N\n\nfeast.coreB\x0f\x46\x65\x61tureSetProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3') + serialized_options=b'\n\nfeast.coreB\017FeatureSetProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core', + serialized_pb=b'\n\x1b\x66\x65\x61st/core/FeatureSet.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x17\x66\x65\x61st/core/Source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a)tensorflow_metadata/proto/v0/schema.proto\"`\n\nFeatureSet\x12(\n\x04spec\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureSetSpec\x12(\n\x04meta\x18\x02 \x01(\x0b\x32\x1a.feast.core.FeatureSetMeta\"\xe5\x01\n\x0e\x46\x65\x61tureSetSpec\x12\x0f\n\x07project\x18\x07 \x01(\t\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\x05\x12(\n\x08\x65ntities\x18\x03 \x03(\x0b\x32\x16.feast.core.EntitySpec\x12)\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x17.feast.core.FeatureSpec\x12*\n\x07max_age\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\"\n\x06source\x18\x06 \x01(\x0b\x32\x12.feast.core.Source\"\xbf\x08\n\nEntitySpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12;\n\x08presence\x18\x03 \x01(\x0b\x32\'.tensorflow.metadata.v0.FeaturePresenceH\x00\x12L\n\x0egroup_presence\x18\x04 \x01(\x0b\x32\x32.tensorflow.metadata.v0.FeaturePresenceWithinGroupH\x00\x12\x33\n\x05shape\x18\x05 \x01(\x0b\x32\".tensorflow.metadata.v0.FixedShapeH\x01\x12\x39\n\x0bvalue_count\x18\x06 \x01(\x0b\x32\".tensorflow.metadata.v0.ValueCountH\x01\x12\x10\n\x06\x64omain\x18\x07 \x01(\tH\x02\x12\x37\n\nint_domain\x18\x08 \x01(\x0b\x32!.tensorflow.metadata.v0.IntDomainH\x02\x12;\n\x0c\x66loat_domain\x18\t \x01(\x0b\x32#.tensorflow.metadata.v0.FloatDomainH\x02\x12=\n\rstring_domain\x18\n \x01(\x0b\x32$.tensorflow.metadata.v0.StringDomainH\x02\x12\x39\n\x0b\x62ool_domain\x18\x0b \x01(\x0b\x32\".tensorflow.metadata.v0.BoolDomainH\x02\x12=\n\rstruct_domain\x18\x0c \x01(\x0b\x32$.tensorflow.metadata.v0.StructDomainH\x02\x12P\n\x17natural_language_domain\x18\r \x01(\x0b\x32-.tensorflow.metadata.v0.NaturalLanguageDomainH\x02\x12;\n\x0cimage_domain\x18\x0e \x01(\x0b\x32#.tensorflow.metadata.v0.ImageDomainH\x02\x12\x37\n\nmid_domain\x18\x0f \x01(\x0b\x32!.tensorflow.metadata.v0.MIDDomainH\x02\x12\x37\n\nurl_domain\x18\x10 \x01(\x0b\x32!.tensorflow.metadata.v0.URLDomainH\x02\x12\x39\n\x0btime_domain\x18\x11 \x01(\x0b\x32\".tensorflow.metadata.v0.TimeDomainH\x02\x12\x45\n\x12time_of_day_domain\x18\x12 \x01(\x0b\x32\'.tensorflow.metadata.v0.TimeOfDayDomainH\x02\x42\x16\n\x14presence_constraintsB\x0c\n\nshape_typeB\r\n\x0b\x64omain_info\"\xc0\x08\n\x0b\x46\x65\x61tureSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12;\n\x08presence\x18\x03 \x01(\x0b\x32\'.tensorflow.metadata.v0.FeaturePresenceH\x00\x12L\n\x0egroup_presence\x18\x04 \x01(\x0b\x32\x32.tensorflow.metadata.v0.FeaturePresenceWithinGroupH\x00\x12\x33\n\x05shape\x18\x05 \x01(\x0b\x32\".tensorflow.metadata.v0.FixedShapeH\x01\x12\x39\n\x0bvalue_count\x18\x06 \x01(\x0b\x32\".tensorflow.metadata.v0.ValueCountH\x01\x12\x10\n\x06\x64omain\x18\x07 \x01(\tH\x02\x12\x37\n\nint_domain\x18\x08 \x01(\x0b\x32!.tensorflow.metadata.v0.IntDomainH\x02\x12;\n\x0c\x66loat_domain\x18\t \x01(\x0b\x32#.tensorflow.metadata.v0.FloatDomainH\x02\x12=\n\rstring_domain\x18\n \x01(\x0b\x32$.tensorflow.metadata.v0.StringDomainH\x02\x12\x39\n\x0b\x62ool_domain\x18\x0b \x01(\x0b\x32\".tensorflow.metadata.v0.BoolDomainH\x02\x12=\n\rstruct_domain\x18\x0c \x01(\x0b\x32$.tensorflow.metadata.v0.StructDomainH\x02\x12P\n\x17natural_language_domain\x18\r \x01(\x0b\x32-.tensorflow.metadata.v0.NaturalLanguageDomainH\x02\x12;\n\x0cimage_domain\x18\x0e \x01(\x0b\x32#.tensorflow.metadata.v0.ImageDomainH\x02\x12\x37\n\nmid_domain\x18\x0f \x01(\x0b\x32!.tensorflow.metadata.v0.MIDDomainH\x02\x12\x37\n\nurl_domain\x18\x10 \x01(\x0b\x32!.tensorflow.metadata.v0.URLDomainH\x02\x12\x39\n\x0btime_domain\x18\x11 \x01(\x0b\x32\".tensorflow.metadata.v0.TimeDomainH\x02\x12\x45\n\x12time_of_day_domain\x18\x12 \x01(\x0b\x32\'.tensorflow.metadata.v0.TimeOfDayDomainH\x02\x42\x16\n\x14presence_constraintsB\x0c\n\nshape_typeB\r\n\x0b\x64omain_info\"u\n\x0e\x46\x65\x61tureSetMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x06status\x18\x02 \x01(\x0e\x32\x1c.feast.core.FeatureSetStatus*L\n\x10\x46\x65\x61tureSetStatus\x12\x12\n\x0eSTATUS_INVALID\x10\x00\x12\x12\n\x0eSTATUS_PENDING\x10\x01\x12\x10\n\x0cSTATUS_READY\x10\x02\x42N\n\nfeast.coreB\x0f\x46\x65\x61tureSetProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3' , - dependencies=[feast_dot_types_dot_Value__pb2.DESCRIPTOR,feast_dot_core_dot_Source__pb2.DESCRIPTOR,google_dot_protobuf_dot_duration__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,]) + dependencies=[feast_dot_types_dot_Value__pb2.DESCRIPTOR,feast_dot_core_dot_Source__pb2.DESCRIPTOR,google_dot_protobuf_dot_duration__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2.DESCRIPTOR,]) _FEATURESETSTATUS = _descriptor.EnumDescriptor( name='FeatureSetStatus', @@ -50,8 +49,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=762, - serialized_end=838, + serialized_start=2831, + serialized_end=2907, ) _sym_db.RegisterEnumDescriptor(_FEATURESETSTATUS) @@ -95,8 +94,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=158, - serialized_end=254, + serialized_start=201, + serialized_end=297, ) @@ -110,14 +109,14 @@ _descriptor.FieldDescriptor( name='project', full_name='feast.core.FeatureSetSpec.project', index=0, number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='name', full_name='feast.core.FeatureSetSpec.name', index=1, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -168,8 +167,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=257, - serialized_end=486, + serialized_start=300, + serialized_end=529, ) @@ -183,7 +182,7 @@ _descriptor.FieldDescriptor( name='name', full_name='feast.core.EntitySpec.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -194,6 +193,118 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='presence', full_name='feast.core.EntitySpec.presence', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='group_presence', full_name='feast.core.EntitySpec.group_presence', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='shape', full_name='feast.core.EntitySpec.shape', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value_count', full_name='feast.core.EntitySpec.value_count', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='domain', full_name='feast.core.EntitySpec.domain', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='int_domain', full_name='feast.core.EntitySpec.int_domain', index=7, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='float_domain', full_name='feast.core.EntitySpec.float_domain', index=8, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='string_domain', full_name='feast.core.EntitySpec.string_domain', index=9, + number=10, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='bool_domain', full_name='feast.core.EntitySpec.bool_domain', index=10, + number=11, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='struct_domain', full_name='feast.core.EntitySpec.struct_domain', index=11, + number=12, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='natural_language_domain', full_name='feast.core.EntitySpec.natural_language_domain', index=12, + number=13, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='image_domain', full_name='feast.core.EntitySpec.image_domain', index=13, + number=14, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mid_domain', full_name='feast.core.EntitySpec.mid_domain', index=14, + number=15, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='url_domain', full_name='feast.core.EntitySpec.url_domain', index=15, + number=16, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='time_domain', full_name='feast.core.EntitySpec.time_domain', index=16, + number=17, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='time_of_day_domain', full_name='feast.core.EntitySpec.time_of_day_domain', index=17, + number=18, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -205,9 +316,18 @@ syntax='proto3', extension_ranges=[], oneofs=[ + _descriptor.OneofDescriptor( + name='presence_constraints', full_name='feast.core.EntitySpec.presence_constraints', + index=0, containing_type=None, fields=[]), + _descriptor.OneofDescriptor( + name='shape_type', full_name='feast.core.EntitySpec.shape_type', + index=1, containing_type=None, fields=[]), + _descriptor.OneofDescriptor( + name='domain_info', full_name='feast.core.EntitySpec.domain_info', + index=2, containing_type=None, fields=[]), ], - serialized_start=488, - serialized_end=563, + serialized_start=532, + serialized_end=1619, ) @@ -221,7 +341,7 @@ _descriptor.FieldDescriptor( name='name', full_name='feast.core.FeatureSpec.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -232,6 +352,118 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='presence', full_name='feast.core.FeatureSpec.presence', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='group_presence', full_name='feast.core.FeatureSpec.group_presence', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='shape', full_name='feast.core.FeatureSpec.shape', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value_count', full_name='feast.core.FeatureSpec.value_count', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='domain', full_name='feast.core.FeatureSpec.domain', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='int_domain', full_name='feast.core.FeatureSpec.int_domain', index=7, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='float_domain', full_name='feast.core.FeatureSpec.float_domain', index=8, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='string_domain', full_name='feast.core.FeatureSpec.string_domain', index=9, + number=10, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='bool_domain', full_name='feast.core.FeatureSpec.bool_domain', index=10, + number=11, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='struct_domain', full_name='feast.core.FeatureSpec.struct_domain', index=11, + number=12, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='natural_language_domain', full_name='feast.core.FeatureSpec.natural_language_domain', index=12, + number=13, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='image_domain', full_name='feast.core.FeatureSpec.image_domain', index=13, + number=14, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mid_domain', full_name='feast.core.FeatureSpec.mid_domain', index=14, + number=15, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='url_domain', full_name='feast.core.FeatureSpec.url_domain', index=15, + number=16, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='time_domain', full_name='feast.core.FeatureSpec.time_domain', index=16, + number=17, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='time_of_day_domain', full_name='feast.core.FeatureSpec.time_of_day_domain', index=17, + number=18, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -243,9 +475,18 @@ syntax='proto3', extension_ranges=[], oneofs=[ + _descriptor.OneofDescriptor( + name='presence_constraints', full_name='feast.core.FeatureSpec.presence_constraints', + index=0, containing_type=None, fields=[]), + _descriptor.OneofDescriptor( + name='shape_type', full_name='feast.core.FeatureSpec.shape_type', + index=1, containing_type=None, fields=[]), + _descriptor.OneofDescriptor( + name='domain_info', full_name='feast.core.FeatureSpec.domain_info', + index=2, containing_type=None, fields=[]), ], - serialized_start=565, - serialized_end=641, + serialized_start=1622, + serialized_end=2710, ) @@ -282,8 +523,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=643, - serialized_end=760, + serialized_start=2712, + serialized_end=2829, ) _FEATURESET.fields_by_name['spec'].message_type = _FEATURESETSPEC @@ -293,7 +534,133 @@ _FEATURESETSPEC.fields_by_name['max_age'].message_type = google_dot_protobuf_dot_duration__pb2._DURATION _FEATURESETSPEC.fields_by_name['source'].message_type = feast_dot_core_dot_Source__pb2._SOURCE _ENTITYSPEC.fields_by_name['value_type'].enum_type = feast_dot_types_dot_Value__pb2._VALUETYPE_ENUM +_ENTITYSPEC.fields_by_name['presence'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FEATUREPRESENCE +_ENTITYSPEC.fields_by_name['group_presence'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FEATUREPRESENCEWITHINGROUP +_ENTITYSPEC.fields_by_name['shape'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FIXEDSHAPE +_ENTITYSPEC.fields_by_name['value_count'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._VALUECOUNT +_ENTITYSPEC.fields_by_name['int_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._INTDOMAIN +_ENTITYSPEC.fields_by_name['float_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FLOATDOMAIN +_ENTITYSPEC.fields_by_name['string_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._STRINGDOMAIN +_ENTITYSPEC.fields_by_name['bool_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._BOOLDOMAIN +_ENTITYSPEC.fields_by_name['struct_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._STRUCTDOMAIN +_ENTITYSPEC.fields_by_name['natural_language_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._NATURALLANGUAGEDOMAIN +_ENTITYSPEC.fields_by_name['image_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._IMAGEDOMAIN +_ENTITYSPEC.fields_by_name['mid_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._MIDDOMAIN +_ENTITYSPEC.fields_by_name['url_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._URLDOMAIN +_ENTITYSPEC.fields_by_name['time_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._TIMEDOMAIN +_ENTITYSPEC.fields_by_name['time_of_day_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._TIMEOFDAYDOMAIN +_ENTITYSPEC.oneofs_by_name['presence_constraints'].fields.append( + _ENTITYSPEC.fields_by_name['presence']) +_ENTITYSPEC.fields_by_name['presence'].containing_oneof = _ENTITYSPEC.oneofs_by_name['presence_constraints'] +_ENTITYSPEC.oneofs_by_name['presence_constraints'].fields.append( + _ENTITYSPEC.fields_by_name['group_presence']) +_ENTITYSPEC.fields_by_name['group_presence'].containing_oneof = _ENTITYSPEC.oneofs_by_name['presence_constraints'] +_ENTITYSPEC.oneofs_by_name['shape_type'].fields.append( + _ENTITYSPEC.fields_by_name['shape']) +_ENTITYSPEC.fields_by_name['shape'].containing_oneof = _ENTITYSPEC.oneofs_by_name['shape_type'] +_ENTITYSPEC.oneofs_by_name['shape_type'].fields.append( + _ENTITYSPEC.fields_by_name['value_count']) +_ENTITYSPEC.fields_by_name['value_count'].containing_oneof = _ENTITYSPEC.oneofs_by_name['shape_type'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['domain']) +_ENTITYSPEC.fields_by_name['domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['int_domain']) +_ENTITYSPEC.fields_by_name['int_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['float_domain']) +_ENTITYSPEC.fields_by_name['float_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['string_domain']) +_ENTITYSPEC.fields_by_name['string_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['bool_domain']) +_ENTITYSPEC.fields_by_name['bool_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['struct_domain']) +_ENTITYSPEC.fields_by_name['struct_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['natural_language_domain']) +_ENTITYSPEC.fields_by_name['natural_language_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['image_domain']) +_ENTITYSPEC.fields_by_name['image_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['mid_domain']) +_ENTITYSPEC.fields_by_name['mid_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['url_domain']) +_ENTITYSPEC.fields_by_name['url_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['time_domain']) +_ENTITYSPEC.fields_by_name['time_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] +_ENTITYSPEC.oneofs_by_name['domain_info'].fields.append( + _ENTITYSPEC.fields_by_name['time_of_day_domain']) +_ENTITYSPEC.fields_by_name['time_of_day_domain'].containing_oneof = _ENTITYSPEC.oneofs_by_name['domain_info'] _FEATURESPEC.fields_by_name['value_type'].enum_type = feast_dot_types_dot_Value__pb2._VALUETYPE_ENUM +_FEATURESPEC.fields_by_name['presence'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FEATUREPRESENCE +_FEATURESPEC.fields_by_name['group_presence'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FEATUREPRESENCEWITHINGROUP +_FEATURESPEC.fields_by_name['shape'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FIXEDSHAPE +_FEATURESPEC.fields_by_name['value_count'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._VALUECOUNT +_FEATURESPEC.fields_by_name['int_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._INTDOMAIN +_FEATURESPEC.fields_by_name['float_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._FLOATDOMAIN +_FEATURESPEC.fields_by_name['string_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._STRINGDOMAIN +_FEATURESPEC.fields_by_name['bool_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._BOOLDOMAIN +_FEATURESPEC.fields_by_name['struct_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._STRUCTDOMAIN +_FEATURESPEC.fields_by_name['natural_language_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._NATURALLANGUAGEDOMAIN +_FEATURESPEC.fields_by_name['image_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._IMAGEDOMAIN +_FEATURESPEC.fields_by_name['mid_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._MIDDOMAIN +_FEATURESPEC.fields_by_name['url_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._URLDOMAIN +_FEATURESPEC.fields_by_name['time_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._TIMEDOMAIN +_FEATURESPEC.fields_by_name['time_of_day_domain'].message_type = tensorflow__metadata_dot_proto_dot_v0_dot_schema__pb2._TIMEOFDAYDOMAIN +_FEATURESPEC.oneofs_by_name['presence_constraints'].fields.append( + _FEATURESPEC.fields_by_name['presence']) +_FEATURESPEC.fields_by_name['presence'].containing_oneof = _FEATURESPEC.oneofs_by_name['presence_constraints'] +_FEATURESPEC.oneofs_by_name['presence_constraints'].fields.append( + _FEATURESPEC.fields_by_name['group_presence']) +_FEATURESPEC.fields_by_name['group_presence'].containing_oneof = _FEATURESPEC.oneofs_by_name['presence_constraints'] +_FEATURESPEC.oneofs_by_name['shape_type'].fields.append( + _FEATURESPEC.fields_by_name['shape']) +_FEATURESPEC.fields_by_name['shape'].containing_oneof = _FEATURESPEC.oneofs_by_name['shape_type'] +_FEATURESPEC.oneofs_by_name['shape_type'].fields.append( + _FEATURESPEC.fields_by_name['value_count']) +_FEATURESPEC.fields_by_name['value_count'].containing_oneof = _FEATURESPEC.oneofs_by_name['shape_type'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['domain']) +_FEATURESPEC.fields_by_name['domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['int_domain']) +_FEATURESPEC.fields_by_name['int_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['float_domain']) +_FEATURESPEC.fields_by_name['float_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['string_domain']) +_FEATURESPEC.fields_by_name['string_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['bool_domain']) +_FEATURESPEC.fields_by_name['bool_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['struct_domain']) +_FEATURESPEC.fields_by_name['struct_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['natural_language_domain']) +_FEATURESPEC.fields_by_name['natural_language_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['image_domain']) +_FEATURESPEC.fields_by_name['image_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['mid_domain']) +_FEATURESPEC.fields_by_name['mid_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['url_domain']) +_FEATURESPEC.fields_by_name['url_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['time_domain']) +_FEATURESPEC.fields_by_name['time_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] +_FEATURESPEC.oneofs_by_name['domain_info'].fields.append( + _FEATURESPEC.fields_by_name['time_of_day_domain']) +_FEATURESPEC.fields_by_name['time_of_day_domain'].containing_oneof = _FEATURESPEC.oneofs_by_name['domain_info'] _FEATURESETMETA.fields_by_name['created_timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP _FEATURESETMETA.fields_by_name['status'].enum_type = _FEATURESETSTATUS DESCRIPTOR.message_types_by_name['FeatureSet'] = _FEATURESET diff --git a/sdk/python/feast/core/FeatureSet_pb2.pyi b/sdk/python/feast/core/FeatureSet_pb2.pyi index c663c70c682..bd8889bea3c 100644 --- a/sdk/python/feast/core/FeatureSet_pb2.pyi +++ b/sdk/python/feast/core/FeatureSet_pb2.pyi @@ -29,13 +29,33 @@ from google.protobuf.timestamp_pb2 import ( Timestamp as google___protobuf___timestamp_pb2___Timestamp, ) +from tensorflow_metadata.proto.v0.schema_pb2 import ( + BoolDomain as tensorflow_metadata___proto___v0___schema_pb2___BoolDomain, + FeaturePresence as tensorflow_metadata___proto___v0___schema_pb2___FeaturePresence, + FeaturePresenceWithinGroup as tensorflow_metadata___proto___v0___schema_pb2___FeaturePresenceWithinGroup, + FixedShape as tensorflow_metadata___proto___v0___schema_pb2___FixedShape, + FloatDomain as tensorflow_metadata___proto___v0___schema_pb2___FloatDomain, + ImageDomain as tensorflow_metadata___proto___v0___schema_pb2___ImageDomain, + IntDomain as tensorflow_metadata___proto___v0___schema_pb2___IntDomain, + MIDDomain as tensorflow_metadata___proto___v0___schema_pb2___MIDDomain, + NaturalLanguageDomain as tensorflow_metadata___proto___v0___schema_pb2___NaturalLanguageDomain, + StringDomain as tensorflow_metadata___proto___v0___schema_pb2___StringDomain, + StructDomain as tensorflow_metadata___proto___v0___schema_pb2___StructDomain, + TimeDomain as tensorflow_metadata___proto___v0___schema_pb2___TimeDomain, + TimeOfDayDomain as tensorflow_metadata___proto___v0___schema_pb2___TimeOfDayDomain, + URLDomain as tensorflow_metadata___proto___v0___schema_pb2___URLDomain, + ValueCount as tensorflow_metadata___proto___v0___schema_pb2___ValueCount, +) + from typing import ( Iterable as typing___Iterable, List as typing___List, Optional as typing___Optional, Text as typing___Text, Tuple as typing___Tuple, + Union as typing___Union, cast as typing___cast, + overload as typing___overload, ) from typing_extensions import ( @@ -43,61 +63,73 @@ from typing_extensions import ( ) -class FeatureSetStatus(int): +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str +if sys.version_info < (3,): + builtin___buffer = buffer + builtin___unicode = unicode + + +class FeatureSetStatus(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> FeatureSetStatus: ... + def Value(cls, name: builtin___str) -> 'FeatureSetStatus': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[FeatureSetStatus]: ... + def values(cls) -> typing___List['FeatureSetStatus']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, FeatureSetStatus]]: ... - STATUS_INVALID = typing___cast(FeatureSetStatus, 0) - STATUS_PENDING = typing___cast(FeatureSetStatus, 1) - STATUS_READY = typing___cast(FeatureSetStatus, 2) -STATUS_INVALID = typing___cast(FeatureSetStatus, 0) -STATUS_PENDING = typing___cast(FeatureSetStatus, 1) -STATUS_READY = typing___cast(FeatureSetStatus, 2) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'FeatureSetStatus']]: ... + STATUS_INVALID = typing___cast('FeatureSetStatus', 0) + STATUS_PENDING = typing___cast('FeatureSetStatus', 1) + STATUS_READY = typing___cast('FeatureSetStatus', 2) +STATUS_INVALID = typing___cast('FeatureSetStatus', 0) +STATUS_PENDING = typing___cast('FeatureSetStatus', 1) +STATUS_READY = typing___cast('FeatureSetStatus', 2) +global___FeatureSetStatus = FeatureSetStatus class FeatureSet(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property - def spec(self) -> FeatureSetSpec: ... + def spec(self) -> global___FeatureSetSpec: ... @property - def meta(self) -> FeatureSetMeta: ... + def meta(self) -> global___FeatureSetMeta: ... def __init__(self, *, - spec : typing___Optional[FeatureSetSpec] = None, - meta : typing___Optional[FeatureSetMeta] = None, + spec : typing___Optional[global___FeatureSetSpec] = None, + meta : typing___Optional[global___FeatureSetMeta] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> FeatureSet: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"meta",u"spec"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"meta",u"spec"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> FeatureSet: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"meta",b"meta",u"spec",b"spec"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"meta",b"meta",u"spec",b"spec"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> FeatureSet: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"meta",b"meta",u"spec",b"spec"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"meta",b"meta",u"spec",b"spec"]) -> None: ... +global___FeatureSet = FeatureSet class FeatureSetSpec(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... project = ... # type: typing___Text name = ... # type: typing___Text - version = ... # type: int + version = ... # type: builtin___int @property - def entities(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[EntitySpec]: ... + def entities(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[global___EntitySpec]: ... @property - def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[FeatureSpec]: ... + def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[global___FeatureSpec]: ... @property def max_age(self) -> google___protobuf___duration_pb2___Duration: ... @@ -109,64 +141,207 @@ class FeatureSetSpec(google___protobuf___message___Message): *, project : typing___Optional[typing___Text] = None, name : typing___Optional[typing___Text] = None, - version : typing___Optional[int] = None, - entities : typing___Optional[typing___Iterable[EntitySpec]] = None, - features : typing___Optional[typing___Iterable[FeatureSpec]] = None, + version : typing___Optional[builtin___int] = None, + entities : typing___Optional[typing___Iterable[global___EntitySpec]] = None, + features : typing___Optional[typing___Iterable[global___FeatureSpec]] = None, max_age : typing___Optional[google___protobuf___duration_pb2___Duration] = None, source : typing___Optional[feast___core___Source_pb2___Source] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> FeatureSetSpec: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"max_age",u"source"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"entities",u"features",u"max_age",u"name",u"project",u"source",u"version"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> FeatureSetSpec: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age",u"source",b"source"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"entities",b"entities",u"features",b"features",u"max_age",b"max_age",u"name",b"name",u"project",b"project",u"source",b"source",u"version",b"version"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> FeatureSetSpec: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age",u"source",b"source"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"entities",b"entities",u"features",b"features",u"max_age",b"max_age",u"name",b"name",u"project",b"project",u"source",b"source",u"version",b"version"]) -> None: ... +global___FeatureSetSpec = FeatureSetSpec class EntitySpec(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... name = ... # type: typing___Text value_type = ... # type: feast___types___Value_pb2___ValueType.Enum + domain = ... # type: typing___Text + + @property + def presence(self) -> tensorflow_metadata___proto___v0___schema_pb2___FeaturePresence: ... + + @property + def group_presence(self) -> tensorflow_metadata___proto___v0___schema_pb2___FeaturePresenceWithinGroup: ... + + @property + def shape(self) -> tensorflow_metadata___proto___v0___schema_pb2___FixedShape: ... + + @property + def value_count(self) -> tensorflow_metadata___proto___v0___schema_pb2___ValueCount: ... + + @property + def int_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___IntDomain: ... + + @property + def float_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___FloatDomain: ... + + @property + def string_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___StringDomain: ... + + @property + def bool_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___BoolDomain: ... + + @property + def struct_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___StructDomain: ... + + @property + def natural_language_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___NaturalLanguageDomain: ... + + @property + def image_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___ImageDomain: ... + + @property + def mid_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___MIDDomain: ... + + @property + def url_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___URLDomain: ... + + @property + def time_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___TimeDomain: ... + + @property + def time_of_day_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___TimeOfDayDomain: ... def __init__(self, *, name : typing___Optional[typing___Text] = None, value_type : typing___Optional[feast___types___Value_pb2___ValueType.Enum] = None, + presence : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FeaturePresence] = None, + group_presence : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FeaturePresenceWithinGroup] = None, + shape : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FixedShape] = None, + value_count : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___ValueCount] = None, + domain : typing___Optional[typing___Text] = None, + int_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___IntDomain] = None, + float_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FloatDomain] = None, + string_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___StringDomain] = None, + bool_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___BoolDomain] = None, + struct_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___StructDomain] = None, + natural_language_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___NaturalLanguageDomain] = None, + image_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___ImageDomain] = None, + mid_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___MIDDomain] = None, + url_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___URLDomain] = None, + time_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___TimeDomain] = None, + time_of_day_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___TimeOfDayDomain] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> EntitySpec: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"name",u"value_type"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> EntitySpec: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"value_type",b"value_type"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> EntitySpec: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"bool_domain",b"bool_domain",u"domain",b"domain",u"domain_info",b"domain_info",u"float_domain",b"float_domain",u"group_presence",b"group_presence",u"image_domain",b"image_domain",u"int_domain",b"int_domain",u"mid_domain",b"mid_domain",u"natural_language_domain",b"natural_language_domain",u"presence",b"presence",u"presence_constraints",b"presence_constraints",u"shape",b"shape",u"shape_type",b"shape_type",u"string_domain",b"string_domain",u"struct_domain",b"struct_domain",u"time_domain",b"time_domain",u"time_of_day_domain",b"time_of_day_domain",u"url_domain",b"url_domain",u"value_count",b"value_count"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bool_domain",b"bool_domain",u"domain",b"domain",u"domain_info",b"domain_info",u"float_domain",b"float_domain",u"group_presence",b"group_presence",u"image_domain",b"image_domain",u"int_domain",b"int_domain",u"mid_domain",b"mid_domain",u"name",b"name",u"natural_language_domain",b"natural_language_domain",u"presence",b"presence",u"presence_constraints",b"presence_constraints",u"shape",b"shape",u"shape_type",b"shape_type",u"string_domain",b"string_domain",u"struct_domain",b"struct_domain",u"time_domain",b"time_domain",u"time_of_day_domain",b"time_of_day_domain",u"url_domain",b"url_domain",u"value_count",b"value_count",u"value_type",b"value_type"]) -> None: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"domain_info",b"domain_info"]) -> typing_extensions___Literal["domain","int_domain","float_domain","string_domain","bool_domain","struct_domain","natural_language_domain","image_domain","mid_domain","url_domain","time_domain","time_of_day_domain"]: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"presence_constraints",b"presence_constraints"]) -> typing_extensions___Literal["presence","group_presence"]: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"shape_type",b"shape_type"]) -> typing_extensions___Literal["shape","value_count"]: ... +global___EntitySpec = EntitySpec class FeatureSpec(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... name = ... # type: typing___Text value_type = ... # type: feast___types___Value_pb2___ValueType.Enum + domain = ... # type: typing___Text + + @property + def presence(self) -> tensorflow_metadata___proto___v0___schema_pb2___FeaturePresence: ... + + @property + def group_presence(self) -> tensorflow_metadata___proto___v0___schema_pb2___FeaturePresenceWithinGroup: ... + + @property + def shape(self) -> tensorflow_metadata___proto___v0___schema_pb2___FixedShape: ... + + @property + def value_count(self) -> tensorflow_metadata___proto___v0___schema_pb2___ValueCount: ... + + @property + def int_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___IntDomain: ... + + @property + def float_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___FloatDomain: ... + + @property + def string_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___StringDomain: ... + + @property + def bool_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___BoolDomain: ... + + @property + def struct_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___StructDomain: ... + + @property + def natural_language_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___NaturalLanguageDomain: ... + + @property + def image_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___ImageDomain: ... + + @property + def mid_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___MIDDomain: ... + + @property + def url_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___URLDomain: ... + + @property + def time_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___TimeDomain: ... + + @property + def time_of_day_domain(self) -> tensorflow_metadata___proto___v0___schema_pb2___TimeOfDayDomain: ... def __init__(self, *, name : typing___Optional[typing___Text] = None, value_type : typing___Optional[feast___types___Value_pb2___ValueType.Enum] = None, + presence : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FeaturePresence] = None, + group_presence : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FeaturePresenceWithinGroup] = None, + shape : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FixedShape] = None, + value_count : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___ValueCount] = None, + domain : typing___Optional[typing___Text] = None, + int_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___IntDomain] = None, + float_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___FloatDomain] = None, + string_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___StringDomain] = None, + bool_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___BoolDomain] = None, + struct_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___StructDomain] = None, + natural_language_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___NaturalLanguageDomain] = None, + image_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___ImageDomain] = None, + mid_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___MIDDomain] = None, + url_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___URLDomain] = None, + time_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___TimeDomain] = None, + time_of_day_domain : typing___Optional[tensorflow_metadata___proto___v0___schema_pb2___TimeOfDayDomain] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> FeatureSpec: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"name",u"value_type"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> FeatureSpec: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"value_type",b"value_type"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> FeatureSpec: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"bool_domain",b"bool_domain",u"domain",b"domain",u"domain_info",b"domain_info",u"float_domain",b"float_domain",u"group_presence",b"group_presence",u"image_domain",b"image_domain",u"int_domain",b"int_domain",u"mid_domain",b"mid_domain",u"natural_language_domain",b"natural_language_domain",u"presence",b"presence",u"presence_constraints",b"presence_constraints",u"shape",b"shape",u"shape_type",b"shape_type",u"string_domain",b"string_domain",u"struct_domain",b"struct_domain",u"time_domain",b"time_domain",u"time_of_day_domain",b"time_of_day_domain",u"url_domain",b"url_domain",u"value_count",b"value_count"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bool_domain",b"bool_domain",u"domain",b"domain",u"domain_info",b"domain_info",u"float_domain",b"float_domain",u"group_presence",b"group_presence",u"image_domain",b"image_domain",u"int_domain",b"int_domain",u"mid_domain",b"mid_domain",u"name",b"name",u"natural_language_domain",b"natural_language_domain",u"presence",b"presence",u"presence_constraints",b"presence_constraints",u"shape",b"shape",u"shape_type",b"shape_type",u"string_domain",b"string_domain",u"struct_domain",b"struct_domain",u"time_domain",b"time_domain",u"time_of_day_domain",b"time_of_day_domain",u"url_domain",b"url_domain",u"value_count",b"value_count",u"value_type",b"value_type"]) -> None: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"domain_info",b"domain_info"]) -> typing_extensions___Literal["domain","int_domain","float_domain","string_domain","bool_domain","struct_domain","natural_language_domain","image_domain","mid_domain","url_domain","time_domain","time_of_day_domain"]: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"presence_constraints",b"presence_constraints"]) -> typing_extensions___Literal["presence","group_presence"]: ... + @typing___overload + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"shape_type",b"shape_type"]) -> typing_extensions___Literal["shape","value_count"]: ... +global___FeatureSpec = FeatureSpec class FeatureSetMeta(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - status = ... # type: FeatureSetStatus + status = ... # type: global___FeatureSetStatus @property def created_timestamp(self) -> google___protobuf___timestamp_pb2___Timestamp: ... @@ -174,15 +349,16 @@ class FeatureSetMeta(google___protobuf___message___Message): def __init__(self, *, created_timestamp : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, - status : typing___Optional[FeatureSetStatus] = None, + status : typing___Optional[global___FeatureSetStatus] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> FeatureSetMeta: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"created_timestamp"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"created_timestamp",u"status"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> FeatureSetMeta: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"created_timestamp",b"created_timestamp"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"created_timestamp",b"created_timestamp",u"status",b"status"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> FeatureSetMeta: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"created_timestamp",b"created_timestamp"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"created_timestamp",b"created_timestamp",u"status",b"status"]) -> None: ... +global___FeatureSetMeta = FeatureSetMeta diff --git a/sdk/python/feast/core/Source_pb2.py b/sdk/python/feast/core/Source_pb2.py index e0d0dd64313..356a7df89c8 100644 --- a/sdk/python/feast/core/Source_pb2.py +++ b/sdk/python/feast/core/Source_pb2.py @@ -2,8 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/core/Source.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message @@ -20,8 +18,8 @@ name='feast/core/Source.proto', package='feast.core', syntax='proto3', - serialized_options=_b('\n\nfeast.coreB\013SourceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core'), - serialized_pb=_b('\n\x17\x66\x65\x61st/core/Source.proto\x12\nfeast.core\"}\n\x06Source\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.feast.core.SourceType\x12<\n\x13kafka_source_config\x18\x02 \x01(\x0b\x32\x1d.feast.core.KafkaSourceConfigH\x00\x42\x0f\n\rsource_config\"=\n\x11KafkaSourceConfig\x12\x19\n\x11\x62ootstrap_servers\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t*$\n\nSourceType\x12\x0b\n\x07INVALID\x10\x00\x12\t\n\x05KAFKA\x10\x01\x42J\n\nfeast.coreB\x0bSourceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3') + serialized_options=b'\n\nfeast.coreB\013SourceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core', + serialized_pb=b'\n\x17\x66\x65\x61st/core/Source.proto\x12\nfeast.core\"}\n\x06Source\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.feast.core.SourceType\x12<\n\x13kafka_source_config\x18\x02 \x01(\x0b\x32\x1d.feast.core.KafkaSourceConfigH\x00\x42\x0f\n\rsource_config\"=\n\x11KafkaSourceConfig\x12\x19\n\x11\x62ootstrap_servers\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t*$\n\nSourceType\x12\x0b\n\x07INVALID\x10\x00\x12\t\n\x05KAFKA\x10\x01\x42J\n\nfeast.coreB\x0bSourceProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3' ) _SOURCETYPE = _descriptor.EnumDescriptor( @@ -103,14 +101,14 @@ _descriptor.FieldDescriptor( name='bootstrap_servers', full_name='feast.core.KafkaSourceConfig.bootstrap_servers', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='topic', full_name='feast.core.KafkaSourceConfig.topic', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), diff --git a/sdk/python/feast/core/Source_pb2.pyi b/sdk/python/feast/core/Source_pb2.pyi index 0521ac34f80..1a0e87e7ba3 100644 --- a/sdk/python/feast/core/Source_pb2.pyi +++ b/sdk/python/feast/core/Source_pb2.pyi @@ -14,6 +14,7 @@ from typing import ( Optional as typing___Optional, Text as typing___Text, Tuple as typing___Tuple, + Union as typing___Union, cast as typing___cast, ) @@ -22,46 +23,58 @@ from typing_extensions import ( ) -class SourceType(int): +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str +if sys.version_info < (3,): + builtin___buffer = buffer + builtin___unicode = unicode + + +class SourceType(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> SourceType: ... + def Value(cls, name: builtin___str) -> 'SourceType': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[SourceType]: ... + def values(cls) -> typing___List['SourceType']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, SourceType]]: ... - INVALID = typing___cast(SourceType, 0) - KAFKA = typing___cast(SourceType, 1) -INVALID = typing___cast(SourceType, 0) -KAFKA = typing___cast(SourceType, 1) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'SourceType']]: ... + INVALID = typing___cast('SourceType', 0) + KAFKA = typing___cast('SourceType', 1) +INVALID = typing___cast('SourceType', 0) +KAFKA = typing___cast('SourceType', 1) +global___SourceType = SourceType class Source(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - type = ... # type: SourceType + type = ... # type: global___SourceType @property - def kafka_source_config(self) -> KafkaSourceConfig: ... + def kafka_source_config(self) -> global___KafkaSourceConfig: ... def __init__(self, *, - type : typing___Optional[SourceType] = None, - kafka_source_config : typing___Optional[KafkaSourceConfig] = None, + type : typing___Optional[global___SourceType] = None, + kafka_source_config : typing___Optional[global___KafkaSourceConfig] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Source: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"kafka_source_config",u"source_config"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"kafka_source_config",u"source_config",u"type"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Source: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"kafka_source_config",b"kafka_source_config",u"source_config",b"source_config"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"kafka_source_config",b"kafka_source_config",u"source_config",b"source_config",u"type",b"type"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Source: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"kafka_source_config",b"kafka_source_config",u"source_config",b"source_config"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"kafka_source_config",b"kafka_source_config",u"source_config",b"source_config",u"type",b"type"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"source_config",b"source_config"]) -> typing_extensions___Literal["kafka_source_config"]: ... +global___Source = Source class KafkaSourceConfig(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -73,11 +86,13 @@ class KafkaSourceConfig(google___protobuf___message___Message): bootstrap_servers : typing___Optional[typing___Text] = None, topic : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> KafkaSourceConfig: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"bootstrap_servers",u"topic"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> KafkaSourceConfig: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"bootstrap_servers",b"bootstrap_servers",u"topic",b"topic"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> KafkaSourceConfig: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bootstrap_servers",b"bootstrap_servers",u"topic",b"topic"]) -> None: ... +global___KafkaSourceConfig = KafkaSourceConfig diff --git a/sdk/python/feast/core/Store_pb2.py b/sdk/python/feast/core/Store_pb2.py index 716a597b9a3..7360f8f9924 100644 --- a/sdk/python/feast/core/Store_pb2.py +++ b/sdk/python/feast/core/Store_pb2.py @@ -2,8 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/core/Store.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -13,15 +11,17 @@ _sym_db = _symbol_database.Default() +from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='feast/core/Store.proto', package='feast.core', syntax='proto3', - serialized_options=_b('\n\nfeast.coreB\nStoreProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core'), - serialized_pb=_b('\n\x16\x66\x65\x61st/core/Store.proto\x12\nfeast.core\"\xca\x04\n\x05Store\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\x04type\x18\x02 \x01(\x0e\x32\x1b.feast.core.Store.StoreType\x12\x35\n\rsubscriptions\x18\x04 \x03(\x0b\x32\x1e.feast.core.Store.Subscription\x12\x35\n\x0credis_config\x18\x0b \x01(\x0b\x32\x1d.feast.core.Store.RedisConfigH\x00\x12;\n\x0f\x62igquery_config\x18\x0c \x01(\x0b\x32 .feast.core.Store.BigQueryConfigH\x00\x12=\n\x10\x63\x61ssandra_config\x18\r \x01(\x0b\x32!.feast.core.Store.CassandraConfigH\x00\x1a)\n\x0bRedisConfig\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x1a\x38\n\x0e\x42igQueryConfig\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x12\n\ndataset_id\x18\x02 \x01(\t\x1a-\n\x0f\x43\x61ssandraConfig\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x1a>\n\x0cSubscription\x12\x0f\n\x07project\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"@\n\tStoreType\x12\x0b\n\x07INVALID\x10\x00\x12\t\n\x05REDIS\x10\x01\x12\x0c\n\x08\x42IGQUERY\x10\x02\x12\r\n\tCASSANDRA\x10\x03\x42\x08\n\x06\x63onfigBI\n\nfeast.coreB\nStoreProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3') -) + serialized_options=b'\n\nfeast.coreB\nStoreProtoZ/github.com/gojek/feast/sdk/go/protos/feast/core', + serialized_pb=b'\n\x16\x66\x65\x61st/core/Store.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\"\x9a\x07\n\x05Store\x12\x0c\n\x04name\x18\x01 \x01(\t\x12)\n\x04type\x18\x02 \x01(\x0e\x32\x1b.feast.core.Store.StoreType\x12\x35\n\rsubscriptions\x18\x04 \x03(\x0b\x32\x1e.feast.core.Store.Subscription\x12\x35\n\x0credis_config\x18\x0b \x01(\x0b\x32\x1d.feast.core.Store.RedisConfigH\x00\x12;\n\x0f\x62igquery_config\x18\x0c \x01(\x0b\x32 .feast.core.Store.BigQueryConfigH\x00\x12=\n\x10\x63\x61ssandra_config\x18\r \x01(\x0b\x32!.feast.core.Store.CassandraConfigH\x00\x1aZ\n\x0bRedisConfig\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x12\x1a\n\x12initial_backoff_ms\x18\x03 \x01(\x05\x12\x13\n\x0bmax_retries\x18\x04 \x01(\x05\x1a\x38\n\x0e\x42igQueryConfig\x12\x12\n\nproject_id\x18\x01 \x01(\t\x12\x12\n\ndataset_id\x18\x02 \x01(\t\x1a\xcb\x02\n\x0f\x43\x61ssandraConfig\x12\x17\n\x0f\x62ootstrap_hosts\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x12\x10\n\x08keyspace\x18\x03 \x01(\t\x12\x12\n\ntable_name\x18\x04 \x01(\t\x12V\n\x13replication_options\x18\x05 \x03(\x0b\x32\x39.feast.core.Store.CassandraConfig.ReplicationOptionsEntry\x12.\n\x0b\x64\x65\x66\x61ult_ttl\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bversionless\x18\x07 \x01(\x08\x12\x13\n\x0b\x63onsistency\x18\x08 \x01(\t\x1a\x39\n\x17ReplicationOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a>\n\x0cSubscription\x12\x0f\n\x07project\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\"@\n\tStoreType\x12\x0b\n\x07INVALID\x10\x00\x12\t\n\x05REDIS\x10\x01\x12\x0c\n\x08\x42IGQUERY\x10\x02\x12\r\n\tCASSANDRA\x10\x03\x42\x08\n\x06\x63onfigBI\n\nfeast.coreB\nStoreProtoZ/github.com/gojek/feast/sdk/go/protos/feast/coreb\x06proto3' + , + dependencies=[google_dot_protobuf_dot_duration__pb2.DESCRIPTOR,]) @@ -50,8 +50,8 @@ ], containing_type=None, serialized_options=None, - serialized_start=551, - serialized_end=615, + serialized_start=919, + serialized_end=983, ) _sym_db.RegisterEnumDescriptor(_STORE_STORETYPE) @@ -66,7 +66,7 @@ _descriptor.FieldDescriptor( name='host', full_name='feast.core.Store.RedisConfig.host', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -77,6 +77,20 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='initial_backoff_ms', full_name='feast.core.Store.RedisConfig.initial_backoff_ms', index=2, + number=3, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='max_retries', full_name='feast.core.Store.RedisConfig.max_retries', index=3, + number=4, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -89,8 +103,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=339, - serialized_end=380, + serialized_start=371, + serialized_end=461, ) _STORE_BIGQUERYCONFIG = _descriptor.Descriptor( @@ -103,14 +117,14 @@ _descriptor.FieldDescriptor( name='project_id', full_name='feast.core.Store.BigQueryConfig.project_id', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='dataset_id', full_name='feast.core.Store.BigQueryConfig.dataset_id', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -126,8 +140,45 @@ extension_ranges=[], oneofs=[ ], - serialized_start=382, - serialized_end=438, + serialized_start=463, + serialized_end=519, +) + +_STORE_CASSANDRACONFIG_REPLICATIONOPTIONSENTRY = _descriptor.Descriptor( + name='ReplicationOptionsEntry', + full_name='feast.core.Store.CassandraConfig.ReplicationOptionsEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='feast.core.Store.CassandraConfig.ReplicationOptionsEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='feast.core.Store.CassandraConfig.ReplicationOptionsEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=796, + serialized_end=853, ) _STORE_CASSANDRACONFIG = _descriptor.Descriptor( @@ -138,9 +189,9 @@ containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='host', full_name='feast.core.Store.CassandraConfig.host', index=0, + name='bootstrap_hosts', full_name='feast.core.Store.CassandraConfig.bootstrap_hosts', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -151,10 +202,52 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='keyspace', full_name='feast.core.Store.CassandraConfig.keyspace', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='table_name', full_name='feast.core.Store.CassandraConfig.table_name', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='replication_options', full_name='feast.core.Store.CassandraConfig.replication_options', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='default_ttl', full_name='feast.core.Store.CassandraConfig.default_ttl', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='versionless', full_name='feast.core.Store.CassandraConfig.versionless', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='consistency', full_name='feast.core.Store.CassandraConfig.consistency', index=7, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], - nested_types=[], + nested_types=[_STORE_CASSANDRACONFIG_REPLICATIONOPTIONSENTRY, ], enum_types=[ ], serialized_options=None, @@ -163,8 +256,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=440, - serialized_end=485, + serialized_start=522, + serialized_end=853, ) _STORE_SUBSCRIPTION = _descriptor.Descriptor( @@ -177,21 +270,21 @@ _descriptor.FieldDescriptor( name='project', full_name='feast.core.Store.Subscription.project', index=0, number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='name', full_name='feast.core.Store.Subscription.name', index=1, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='version', full_name='feast.core.Store.Subscription.version', index=2, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -207,8 +300,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=487, - serialized_end=549, + serialized_start=855, + serialized_end=917, ) _STORE = _descriptor.Descriptor( @@ -221,7 +314,7 @@ _descriptor.FieldDescriptor( name='name', full_name='feast.core.Store.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -276,12 +369,15 @@ name='config', full_name='feast.core.Store.config', index=0, containing_type=None, fields=[]), ], - serialized_start=39, - serialized_end=625, + serialized_start=71, + serialized_end=993, ) _STORE_REDISCONFIG.containing_type = _STORE _STORE_BIGQUERYCONFIG.containing_type = _STORE +_STORE_CASSANDRACONFIG_REPLICATIONOPTIONSENTRY.containing_type = _STORE_CASSANDRACONFIG +_STORE_CASSANDRACONFIG.fields_by_name['replication_options'].message_type = _STORE_CASSANDRACONFIG_REPLICATIONOPTIONSENTRY +_STORE_CASSANDRACONFIG.fields_by_name['default_ttl'].message_type = google_dot_protobuf_dot_duration__pb2._DURATION _STORE_CASSANDRACONFIG.containing_type = _STORE _STORE_SUBSCRIPTION.containing_type = _STORE _STORE.fields_by_name['type'].enum_type = _STORE_STORETYPE @@ -319,6 +415,13 @@ , 'CassandraConfig' : _reflection.GeneratedProtocolMessageType('CassandraConfig', (_message.Message,), { + + 'ReplicationOptionsEntry' : _reflection.GeneratedProtocolMessageType('ReplicationOptionsEntry', (_message.Message,), { + 'DESCRIPTOR' : _STORE_CASSANDRACONFIG_REPLICATIONOPTIONSENTRY, + '__module__' : 'feast.core.Store_pb2' + # @@protoc_insertion_point(class_scope:feast.core.Store.CassandraConfig.ReplicationOptionsEntry) + }) + , 'DESCRIPTOR' : _STORE_CASSANDRACONFIG, '__module__' : 'feast.core.Store_pb2' # @@protoc_insertion_point(class_scope:feast.core.Store.CassandraConfig) @@ -339,8 +442,10 @@ _sym_db.RegisterMessage(Store.RedisConfig) _sym_db.RegisterMessage(Store.BigQueryConfig) _sym_db.RegisterMessage(Store.CassandraConfig) +_sym_db.RegisterMessage(Store.CassandraConfig.ReplicationOptionsEntry) _sym_db.RegisterMessage(Store.Subscription) DESCRIPTOR._options = None +_STORE_CASSANDRACONFIG_REPLICATIONOPTIONSENTRY._options = None # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/core/Store_pb2.pyi b/sdk/python/feast/core/Store_pb2.pyi index 541bcd329bb..81bbee04756 100644 --- a/sdk/python/feast/core/Store_pb2.pyi +++ b/sdk/python/feast/core/Store_pb2.pyi @@ -5,6 +5,10 @@ from google.protobuf.descriptor import ( EnumDescriptor as google___protobuf___descriptor___EnumDescriptor, ) +from google.protobuf.duration_pb2 import ( + Duration as google___protobuf___duration_pb2___Duration, +) + from google.protobuf.internal.containers import ( RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, ) @@ -16,9 +20,12 @@ from google.protobuf.message import ( from typing import ( Iterable as typing___Iterable, List as typing___List, + Mapping as typing___Mapping, + MutableMapping as typing___MutableMapping, Optional as typing___Optional, Text as typing___Text, Tuple as typing___Tuple, + Union as typing___Union, cast as typing___cast, ) @@ -27,47 +34,64 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str +if sys.version_info < (3,): + builtin___buffer = buffer + builtin___unicode = unicode + + class Store(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class StoreType(int): + class StoreType(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> Store.StoreType: ... + def Value(cls, name: builtin___str) -> 'Store.StoreType': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[Store.StoreType]: ... + def values(cls) -> typing___List['Store.StoreType']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, Store.StoreType]]: ... - INVALID = typing___cast(Store.StoreType, 0) - REDIS = typing___cast(Store.StoreType, 1) - BIGQUERY = typing___cast(Store.StoreType, 2) - CASSANDRA = typing___cast(Store.StoreType, 3) - INVALID = typing___cast(Store.StoreType, 0) - REDIS = typing___cast(Store.StoreType, 1) - BIGQUERY = typing___cast(Store.StoreType, 2) - CASSANDRA = typing___cast(Store.StoreType, 3) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'Store.StoreType']]: ... + INVALID = typing___cast('Store.StoreType', 0) + REDIS = typing___cast('Store.StoreType', 1) + BIGQUERY = typing___cast('Store.StoreType', 2) + CASSANDRA = typing___cast('Store.StoreType', 3) + INVALID = typing___cast('Store.StoreType', 0) + REDIS = typing___cast('Store.StoreType', 1) + BIGQUERY = typing___cast('Store.StoreType', 2) + CASSANDRA = typing___cast('Store.StoreType', 3) + global___StoreType = StoreType class RedisConfig(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... host = ... # type: typing___Text - port = ... # type: int + port = ... # type: builtin___int + initial_backoff_ms = ... # type: builtin___int + max_retries = ... # type: builtin___int def __init__(self, *, host : typing___Optional[typing___Text] = None, - port : typing___Optional[int] = None, + port : typing___Optional[builtin___int] = None, + initial_backoff_ms : typing___Optional[builtin___int] = None, + max_retries : typing___Optional[builtin___int] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Store.RedisConfig: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"host",u"port"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Store.RedisConfig: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"host",b"host",u"port",b"port"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Store.RedisConfig: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"host",b"host",u"initial_backoff_ms",b"initial_backoff_ms",u"max_retries",b"max_retries",u"port",b"port"]) -> None: ... + global___RedisConfig = RedisConfig class BigQueryConfig(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -79,33 +103,75 @@ class Store(google___protobuf___message___Message): project_id : typing___Optional[typing___Text] = None, dataset_id : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Store.BigQueryConfig: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"dataset_id",u"project_id"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Store.BigQueryConfig: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"dataset_id",b"dataset_id",u"project_id",b"project_id"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Store.BigQueryConfig: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"dataset_id",b"dataset_id",u"project_id",b"project_id"]) -> None: ... + global___BigQueryConfig = BigQueryConfig class CassandraConfig(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - host = ... # type: typing___Text - port = ... # type: int + class ReplicationOptionsEntry(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + key = ... # type: typing___Text + value = ... # type: typing___Text + + def __init__(self, + *, + key : typing___Optional[typing___Text] = None, + value : typing___Optional[typing___Text] = None, + ) -> None: ... + if sys.version_info >= (3,): + @classmethod + def FromString(cls, s: builtin___bytes) -> Store.CassandraConfig.ReplicationOptionsEntry: ... + else: + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Store.CassandraConfig.ReplicationOptionsEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ... + global___ReplicationOptionsEntry = ReplicationOptionsEntry + + bootstrap_hosts = ... # type: typing___Text + port = ... # type: builtin___int + keyspace = ... # type: typing___Text + table_name = ... # type: typing___Text + versionless = ... # type: builtin___bool + consistency = ... # type: typing___Text + + @property + def replication_options(self) -> typing___MutableMapping[typing___Text, typing___Text]: ... + + @property + def default_ttl(self) -> google___protobuf___duration_pb2___Duration: ... def __init__(self, *, - host : typing___Optional[typing___Text] = None, - port : typing___Optional[int] = None, + bootstrap_hosts : typing___Optional[typing___Text] = None, + port : typing___Optional[builtin___int] = None, + keyspace : typing___Optional[typing___Text] = None, + table_name : typing___Optional[typing___Text] = None, + replication_options : typing___Optional[typing___Mapping[typing___Text, typing___Text]] = None, + default_ttl : typing___Optional[google___protobuf___duration_pb2___Duration] = None, + versionless : typing___Optional[builtin___bool] = None, + consistency : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Store.CassandraConfig: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"host",u"port"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Store.CassandraConfig: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"host",b"host",u"port",b"port"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Store.CassandraConfig: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"default_ttl",b"default_ttl"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bootstrap_hosts",b"bootstrap_hosts",u"consistency",b"consistency",u"default_ttl",b"default_ttl",u"keyspace",b"keyspace",u"port",b"port",u"replication_options",b"replication_options",u"table_name",b"table_name",u"versionless",b"versionless"]) -> None: ... + global___CassandraConfig = CassandraConfig class Subscription(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -119,47 +185,50 @@ class Store(google___protobuf___message___Message): name : typing___Optional[typing___Text] = None, version : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Store.Subscription: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"name",u"project",u"version"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Store.Subscription: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"project",b"project",u"version",b"version"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Store.Subscription: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"project",b"project",u"version",b"version"]) -> None: ... + global___Subscription = Subscription name = ... # type: typing___Text - type = ... # type: Store.StoreType + type = ... # type: global___Store.StoreType @property - def subscriptions(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Store.Subscription]: ... + def subscriptions(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[global___Store.Subscription]: ... @property - def redis_config(self) -> Store.RedisConfig: ... + def redis_config(self) -> global___Store.RedisConfig: ... @property - def bigquery_config(self) -> Store.BigQueryConfig: ... + def bigquery_config(self) -> global___Store.BigQueryConfig: ... @property - def cassandra_config(self) -> Store.CassandraConfig: ... + def cassandra_config(self) -> global___Store.CassandraConfig: ... def __init__(self, *, name : typing___Optional[typing___Text] = None, - type : typing___Optional[Store.StoreType] = None, - subscriptions : typing___Optional[typing___Iterable[Store.Subscription]] = None, - redis_config : typing___Optional[Store.RedisConfig] = None, - bigquery_config : typing___Optional[Store.BigQueryConfig] = None, - cassandra_config : typing___Optional[Store.CassandraConfig] = None, + type : typing___Optional[global___Store.StoreType] = None, + subscriptions : typing___Optional[typing___Iterable[global___Store.Subscription]] = None, + redis_config : typing___Optional[global___Store.RedisConfig] = None, + bigquery_config : typing___Optional[global___Store.BigQueryConfig] = None, + cassandra_config : typing___Optional[global___Store.CassandraConfig] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Store: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"bigquery_config",u"cassandra_config",u"config",u"redis_config"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"bigquery_config",u"cassandra_config",u"config",u"name",u"redis_config",u"subscriptions",u"type"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Store: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"bigquery_config",b"bigquery_config",u"cassandra_config",b"cassandra_config",u"config",b"config",u"redis_config",b"redis_config"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"bigquery_config",b"bigquery_config",u"cassandra_config",b"cassandra_config",u"config",b"config",u"name",b"name",u"redis_config",b"redis_config",u"subscriptions",b"subscriptions",u"type",b"type"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Store: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"bigquery_config",b"bigquery_config",u"cassandra_config",b"cassandra_config",u"config",b"config",u"redis_config",b"redis_config"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bigquery_config",b"bigquery_config",u"cassandra_config",b"cassandra_config",u"config",b"config",u"name",b"name",u"redis_config",b"redis_config",u"subscriptions",b"subscriptions",u"type",b"type"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"config",b"config"]) -> typing_extensions___Literal["redis_config","bigquery_config","cassandra_config"]: ... +global___Store = Store diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index d5576607513..c47c51e5a21 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -689,8 +689,6 @@ def from_dict(cls, fs_dict): Returns a FeatureSet object based on the feature set dict """ - if ("kind" not in fs_dict) and (fs_dict["kind"].strip() != "feature_set"): - raise Exception(f"Resource kind is not a feature set {str(fs_dict)}") feature_set_proto = json_format.ParseDict( fs_dict, FeatureSetProto(), ignore_unknown_fields=True ) diff --git a/sdk/python/feast/loaders/yaml.py b/sdk/python/feast/loaders/yaml.py index 4cbe15dfaaf..130a71a3d02 100644 --- a/sdk/python/feast/loaders/yaml.py +++ b/sdk/python/feast/loaders/yaml.py @@ -53,7 +53,7 @@ def _get_yaml_contents(yml: str) -> str: with open(yml, "r") as f: yml_content = f.read() - elif isinstance(yml, str) and "kind" in yml.lower(): + elif isinstance(yml, str): yml_content = yml else: raise Exception( @@ -73,7 +73,4 @@ def _yaml_to_dict(yaml_string): Dictionary containing the same object """ - yaml_dict = yaml.safe_load(yaml_string) - if not isinstance(yaml_dict, dict) or not "kind" in yaml_dict: - raise Exception(f"Could not detect YAML kind from resource: ${yaml_string}") - return yaml_dict + return yaml.safe_load(yaml_string) diff --git a/sdk/python/feast/resource.py b/sdk/python/feast/resource.py deleted file mode 100644 index 17a65291667..00000000000 --- a/sdk/python/feast/resource.py +++ /dev/null @@ -1,10 +0,0 @@ -from feast.feature_set import FeatureSet - -# TODO: This factory adds no value. It should be removed asap. -class ResourceFactory: - @staticmethod - def get_resource(kind): - if kind == "feature_set": - return FeatureSet - else: - raise ValueError(kind) diff --git a/sdk/python/feast/serving/ServingService_pb2.py b/sdk/python/feast/serving/ServingService_pb2.py index 9d0d55f2ab4..42e92d1f6a5 100644 --- a/sdk/python/feast/serving/ServingService_pb2.py +++ b/sdk/python/feast/serving/ServingService_pb2.py @@ -2,8 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/serving/ServingService.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message @@ -23,8 +21,8 @@ name='feast/serving/ServingService.proto', package='feast.serving', syntax='proto3', - serialized_options=_b('\n\rfeast.servingB\017ServingAPIProtoZ2github.com/gojek/feast/sdk/go/protos/feast/serving'), - serialized_pb=_b('\n\"feast/serving/ServingService.proto\x12\rfeast.serving\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"\x1c\n\x1aGetFeastServingInfoRequest\"{\n\x1bGetFeastServingInfoResponse\x12\x0f\n\x07version\x18\x01 \x01(\t\x12-\n\x04type\x18\x02 \x01(\x0e\x32\x1f.feast.serving.FeastServingType\x12\x1c\n\x14job_staging_location\x18\n \x01(\t\"n\n\x10\x46\x65\x61tureReference\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\x05\x12*\n\x07max_age\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\"\x8e\x03\n\x18GetOnlineFeaturesRequest\x12\x31\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x1f.feast.serving.FeatureReference\x12\x46\n\x0b\x65ntity_rows\x18\x02 \x03(\x0b\x32\x31.feast.serving.GetOnlineFeaturesRequest.EntityRow\x12!\n\x19omit_entities_in_response\x18\x03 \x01(\x08\x1a\xd3\x01\n\tEntityRow\x12\x34\n\x10\x65ntity_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12M\n\x06\x66ields\x18\x02 \x03(\x0b\x32=.feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry\x1a\x41\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.feast.types.Value:\x02\x38\x01\"\x82\x01\n\x17GetBatchFeaturesRequest\x12\x31\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32\x1f.feast.serving.FeatureReference\x12\x34\n\x0e\x64\x61taset_source\x18\x02 \x01(\x0b\x32\x1c.feast.serving.DatasetSource\"\x8c\x02\n\x19GetOnlineFeaturesResponse\x12J\n\x0c\x66ield_values\x18\x01 \x03(\x0b\x32\x34.feast.serving.GetOnlineFeaturesResponse.FieldValues\x1a\xa2\x01\n\x0b\x46ieldValues\x12P\n\x06\x66ields\x18\x01 \x03(\x0b\x32@.feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry\x1a\x41\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.feast.types.Value:\x02\x38\x01\";\n\x18GetBatchFeaturesResponse\x12\x1f\n\x03job\x18\x01 \x01(\x0b\x32\x12.feast.serving.Job\"0\n\rGetJobRequest\x12\x1f\n\x03job\x18\x01 \x01(\x0b\x32\x12.feast.serving.Job\"1\n\x0eGetJobResponse\x12\x1f\n\x03job\x18\x01 \x01(\x0b\x32\x12.feast.serving.Job\"\xb3\x01\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12$\n\x04type\x18\x02 \x01(\x0e\x32\x16.feast.serving.JobType\x12(\n\x06status\x18\x03 \x01(\x0e\x32\x18.feast.serving.JobStatus\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x11\n\tfile_uris\x18\x05 \x03(\t\x12.\n\x0b\x64\x61ta_format\x18\x06 \x01(\x0e\x32\x19.feast.serving.DataFormat\"\xb2\x01\n\rDatasetSource\x12>\n\x0b\x66ile_source\x18\x01 \x01(\x0b\x32\'.feast.serving.DatasetSource.FileSourceH\x00\x1aO\n\nFileSource\x12\x11\n\tfile_uris\x18\x01 \x03(\t\x12.\n\x0b\x64\x61ta_format\x18\x02 \x01(\x0e\x32\x19.feast.serving.DataFormatB\x10\n\x0e\x64\x61taset_source*o\n\x10\x46\x65\x61stServingType\x12\x1e\n\x1a\x46\x45\x41ST_SERVING_TYPE_INVALID\x10\x00\x12\x1d\n\x19\x46\x45\x41ST_SERVING_TYPE_ONLINE\x10\x01\x12\x1c\n\x18\x46\x45\x41ST_SERVING_TYPE_BATCH\x10\x02*6\n\x07JobType\x12\x14\n\x10JOB_TYPE_INVALID\x10\x00\x12\x15\n\x11JOB_TYPE_DOWNLOAD\x10\x01*h\n\tJobStatus\x12\x16\n\x12JOB_STATUS_INVALID\x10\x00\x12\x16\n\x12JOB_STATUS_PENDING\x10\x01\x12\x16\n\x12JOB_STATUS_RUNNING\x10\x02\x12\x13\n\x0fJOB_STATUS_DONE\x10\x03*;\n\nDataFormat\x12\x17\n\x13\x44\x41TA_FORMAT_INVALID\x10\x00\x12\x14\n\x10\x44\x41TA_FORMAT_AVRO\x10\x01\x32\x92\x03\n\x0eServingService\x12l\n\x13GetFeastServingInfo\x12).feast.serving.GetFeastServingInfoRequest\x1a*.feast.serving.GetFeastServingInfoResponse\x12\x66\n\x11GetOnlineFeatures\x12\'.feast.serving.GetOnlineFeaturesRequest\x1a(.feast.serving.GetOnlineFeaturesResponse\x12\x63\n\x10GetBatchFeatures\x12&.feast.serving.GetBatchFeaturesRequest\x1a\'.feast.serving.GetBatchFeaturesResponse\x12\x45\n\x06GetJob\x12\x1c.feast.serving.GetJobRequest\x1a\x1d.feast.serving.GetJobResponseBT\n\rfeast.servingB\x0fServingAPIProtoZ2github.com/gojek/feast/sdk/go/protos/feast/servingb\x06proto3') + serialized_options=b'\n\rfeast.servingB\017ServingAPIProtoZ2github.com/gojek/feast/sdk/go/protos/feast/serving', + serialized_pb=b'\n\"feast/serving/ServingService.proto\x12\rfeast.serving\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"\x1c\n\x1aGetFeastServingInfoRequest\"{\n\x1bGetFeastServingInfoResponse\x12\x0f\n\x07version\x18\x01 \x01(\t\x12-\n\x04type\x18\x02 \x01(\x0e\x32\x1f.feast.serving.FeastServingType\x12\x1c\n\x14job_staging_location\x18\n \x01(\t\"n\n\x10\x46\x65\x61tureReference\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\x05\x12*\n\x07max_age\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\"\x8e\x03\n\x18GetOnlineFeaturesRequest\x12\x31\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x1f.feast.serving.FeatureReference\x12\x46\n\x0b\x65ntity_rows\x18\x02 \x03(\x0b\x32\x31.feast.serving.GetOnlineFeaturesRequest.EntityRow\x12!\n\x19omit_entities_in_response\x18\x03 \x01(\x08\x1a\xd3\x01\n\tEntityRow\x12\x34\n\x10\x65ntity_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12M\n\x06\x66ields\x18\x02 \x03(\x0b\x32=.feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry\x1a\x41\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.feast.types.Value:\x02\x38\x01\"\x82\x01\n\x17GetBatchFeaturesRequest\x12\x31\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32\x1f.feast.serving.FeatureReference\x12\x34\n\x0e\x64\x61taset_source\x18\x02 \x01(\x0b\x32\x1c.feast.serving.DatasetSource\"\x8c\x02\n\x19GetOnlineFeaturesResponse\x12J\n\x0c\x66ield_values\x18\x01 \x03(\x0b\x32\x34.feast.serving.GetOnlineFeaturesResponse.FieldValues\x1a\xa2\x01\n\x0b\x46ieldValues\x12P\n\x06\x66ields\x18\x01 \x03(\x0b\x32@.feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry\x1a\x41\n\x0b\x46ieldsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.feast.types.Value:\x02\x38\x01\";\n\x18GetBatchFeaturesResponse\x12\x1f\n\x03job\x18\x01 \x01(\x0b\x32\x12.feast.serving.Job\"0\n\rGetJobRequest\x12\x1f\n\x03job\x18\x01 \x01(\x0b\x32\x12.feast.serving.Job\"1\n\x0eGetJobResponse\x12\x1f\n\x03job\x18\x01 \x01(\x0b\x32\x12.feast.serving.Job\"\xb3\x01\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12$\n\x04type\x18\x02 \x01(\x0e\x32\x16.feast.serving.JobType\x12(\n\x06status\x18\x03 \x01(\x0e\x32\x18.feast.serving.JobStatus\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x11\n\tfile_uris\x18\x05 \x03(\t\x12.\n\x0b\x64\x61ta_format\x18\x06 \x01(\x0e\x32\x19.feast.serving.DataFormat\"\xb2\x01\n\rDatasetSource\x12>\n\x0b\x66ile_source\x18\x01 \x01(\x0b\x32\'.feast.serving.DatasetSource.FileSourceH\x00\x1aO\n\nFileSource\x12\x11\n\tfile_uris\x18\x01 \x03(\t\x12.\n\x0b\x64\x61ta_format\x18\x02 \x01(\x0e\x32\x19.feast.serving.DataFormatB\x10\n\x0e\x64\x61taset_source*o\n\x10\x46\x65\x61stServingType\x12\x1e\n\x1a\x46\x45\x41ST_SERVING_TYPE_INVALID\x10\x00\x12\x1d\n\x19\x46\x45\x41ST_SERVING_TYPE_ONLINE\x10\x01\x12\x1c\n\x18\x46\x45\x41ST_SERVING_TYPE_BATCH\x10\x02*6\n\x07JobType\x12\x14\n\x10JOB_TYPE_INVALID\x10\x00\x12\x15\n\x11JOB_TYPE_DOWNLOAD\x10\x01*h\n\tJobStatus\x12\x16\n\x12JOB_STATUS_INVALID\x10\x00\x12\x16\n\x12JOB_STATUS_PENDING\x10\x01\x12\x16\n\x12JOB_STATUS_RUNNING\x10\x02\x12\x13\n\x0fJOB_STATUS_DONE\x10\x03*;\n\nDataFormat\x12\x17\n\x13\x44\x41TA_FORMAT_INVALID\x10\x00\x12\x14\n\x10\x44\x41TA_FORMAT_AVRO\x10\x01\x32\x92\x03\n\x0eServingService\x12l\n\x13GetFeastServingInfo\x12).feast.serving.GetFeastServingInfoRequest\x1a*.feast.serving.GetFeastServingInfoResponse\x12\x66\n\x11GetOnlineFeatures\x12\'.feast.serving.GetOnlineFeaturesRequest\x1a(.feast.serving.GetOnlineFeaturesResponse\x12\x63\n\x10GetBatchFeatures\x12&.feast.serving.GetBatchFeaturesRequest\x1a\'.feast.serving.GetBatchFeaturesResponse\x12\x45\n\x06GetJob\x12\x1c.feast.serving.GetJobRequest\x1a\x1d.feast.serving.GetJobResponseBT\n\rfeast.servingB\x0fServingAPIProtoZ2github.com/gojek/feast/sdk/go/protos/feast/servingb\x06proto3' , dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,google_dot_protobuf_dot_duration__pb2.DESCRIPTOR,feast_dot_types_dot_Value__pb2.DESCRIPTOR,]) @@ -180,7 +178,7 @@ _descriptor.FieldDescriptor( name='version', full_name='feast.serving.GetFeastServingInfoResponse.version', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -194,7 +192,7 @@ _descriptor.FieldDescriptor( name='job_staging_location', full_name='feast.serving.GetFeastServingInfoResponse.job_staging_location', index=2, number=10, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -225,14 +223,14 @@ _descriptor.FieldDescriptor( name='project', full_name='feast.serving.FeatureReference.project', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='name', full_name='feast.serving.FeatureReference.name', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -277,7 +275,7 @@ _descriptor.FieldDescriptor( name='key', full_name='feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry.key', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -294,7 +292,7 @@ nested_types=[], enum_types=[ ], - serialized_options=_b('8\001'), + serialized_options=b'8\001', is_extendable=False, syntax='proto3', extension_ranges=[], @@ -434,7 +432,7 @@ _descriptor.FieldDescriptor( name='key', full_name='feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry.key', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -451,7 +449,7 @@ nested_types=[], enum_types=[ ], - serialized_options=_b('8\001'), + serialized_options=b'8\001', is_extendable=False, syntax='proto3', extension_ranges=[], @@ -625,7 +623,7 @@ _descriptor.FieldDescriptor( name='id', full_name='feast.serving.Job.id', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), @@ -646,7 +644,7 @@ _descriptor.FieldDescriptor( name='error', full_name='feast.serving.Job.error', index=3, number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), diff --git a/sdk/python/feast/serving/ServingService_pb2.pyi b/sdk/python/feast/serving/ServingService_pb2.pyi index e10245d6c7a..8a58633b5ff 100644 --- a/sdk/python/feast/serving/ServingService_pb2.pyi +++ b/sdk/python/feast/serving/ServingService_pb2.pyi @@ -34,6 +34,7 @@ from typing import ( Optional as typing___Optional, Text as typing___Text, Tuple as typing___Tuple, + Union as typing___Union, cast as typing___cast, ) @@ -42,116 +43,137 @@ from typing_extensions import ( ) -class FeastServingType(int): +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str +if sys.version_info < (3,): + builtin___buffer = buffer + builtin___unicode = unicode + + +class FeastServingType(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> FeastServingType: ... + def Value(cls, name: builtin___str) -> 'FeastServingType': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[FeastServingType]: ... + def values(cls) -> typing___List['FeastServingType']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, FeastServingType]]: ... - FEAST_SERVING_TYPE_INVALID = typing___cast(FeastServingType, 0) - FEAST_SERVING_TYPE_ONLINE = typing___cast(FeastServingType, 1) - FEAST_SERVING_TYPE_BATCH = typing___cast(FeastServingType, 2) -FEAST_SERVING_TYPE_INVALID = typing___cast(FeastServingType, 0) -FEAST_SERVING_TYPE_ONLINE = typing___cast(FeastServingType, 1) -FEAST_SERVING_TYPE_BATCH = typing___cast(FeastServingType, 2) - -class JobType(int): + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'FeastServingType']]: ... + FEAST_SERVING_TYPE_INVALID = typing___cast('FeastServingType', 0) + FEAST_SERVING_TYPE_ONLINE = typing___cast('FeastServingType', 1) + FEAST_SERVING_TYPE_BATCH = typing___cast('FeastServingType', 2) +FEAST_SERVING_TYPE_INVALID = typing___cast('FeastServingType', 0) +FEAST_SERVING_TYPE_ONLINE = typing___cast('FeastServingType', 1) +FEAST_SERVING_TYPE_BATCH = typing___cast('FeastServingType', 2) +global___FeastServingType = FeastServingType + +class JobType(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> JobType: ... + def Value(cls, name: builtin___str) -> 'JobType': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[JobType]: ... + def values(cls) -> typing___List['JobType']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, JobType]]: ... - JOB_TYPE_INVALID = typing___cast(JobType, 0) - JOB_TYPE_DOWNLOAD = typing___cast(JobType, 1) -JOB_TYPE_INVALID = typing___cast(JobType, 0) -JOB_TYPE_DOWNLOAD = typing___cast(JobType, 1) - -class JobStatus(int): + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'JobType']]: ... + JOB_TYPE_INVALID = typing___cast('JobType', 0) + JOB_TYPE_DOWNLOAD = typing___cast('JobType', 1) +JOB_TYPE_INVALID = typing___cast('JobType', 0) +JOB_TYPE_DOWNLOAD = typing___cast('JobType', 1) +global___JobType = JobType + +class JobStatus(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> JobStatus: ... + def Value(cls, name: builtin___str) -> 'JobStatus': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[JobStatus]: ... + def values(cls) -> typing___List['JobStatus']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, JobStatus]]: ... - JOB_STATUS_INVALID = typing___cast(JobStatus, 0) - JOB_STATUS_PENDING = typing___cast(JobStatus, 1) - JOB_STATUS_RUNNING = typing___cast(JobStatus, 2) - JOB_STATUS_DONE = typing___cast(JobStatus, 3) -JOB_STATUS_INVALID = typing___cast(JobStatus, 0) -JOB_STATUS_PENDING = typing___cast(JobStatus, 1) -JOB_STATUS_RUNNING = typing___cast(JobStatus, 2) -JOB_STATUS_DONE = typing___cast(JobStatus, 3) - -class DataFormat(int): + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'JobStatus']]: ... + JOB_STATUS_INVALID = typing___cast('JobStatus', 0) + JOB_STATUS_PENDING = typing___cast('JobStatus', 1) + JOB_STATUS_RUNNING = typing___cast('JobStatus', 2) + JOB_STATUS_DONE = typing___cast('JobStatus', 3) +JOB_STATUS_INVALID = typing___cast('JobStatus', 0) +JOB_STATUS_PENDING = typing___cast('JobStatus', 1) +JOB_STATUS_RUNNING = typing___cast('JobStatus', 2) +JOB_STATUS_DONE = typing___cast('JobStatus', 3) +global___JobStatus = JobStatus + +class DataFormat(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> DataFormat: ... + def Value(cls, name: builtin___str) -> 'DataFormat': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[DataFormat]: ... + def values(cls) -> typing___List['DataFormat']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, DataFormat]]: ... - DATA_FORMAT_INVALID = typing___cast(DataFormat, 0) - DATA_FORMAT_AVRO = typing___cast(DataFormat, 1) -DATA_FORMAT_INVALID = typing___cast(DataFormat, 0) -DATA_FORMAT_AVRO = typing___cast(DataFormat, 1) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'DataFormat']]: ... + DATA_FORMAT_INVALID = typing___cast('DataFormat', 0) + DATA_FORMAT_AVRO = typing___cast('DataFormat', 1) +DATA_FORMAT_INVALID = typing___cast('DataFormat', 0) +DATA_FORMAT_AVRO = typing___cast('DataFormat', 1) +global___DataFormat = DataFormat class GetFeastServingInfoRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... def __init__(self, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetFeastServingInfoRequest: ... + if sys.version_info >= (3,): + @classmethod + def FromString(cls, s: builtin___bytes) -> GetFeastServingInfoRequest: ... + else: + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetFeastServingInfoRequest: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... +global___GetFeastServingInfoRequest = GetFeastServingInfoRequest class GetFeastServingInfoResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... version = ... # type: typing___Text - type = ... # type: FeastServingType + type = ... # type: global___FeastServingType job_staging_location = ... # type: typing___Text def __init__(self, *, version : typing___Optional[typing___Text] = None, - type : typing___Optional[FeastServingType] = None, + type : typing___Optional[global___FeastServingType] = None, job_staging_location : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetFeastServingInfoResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"job_staging_location",u"type",u"version"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetFeastServingInfoResponse: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"job_staging_location",b"job_staging_location",u"type",b"type",u"version",b"version"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetFeastServingInfoResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"job_staging_location",b"job_staging_location",u"type",b"type",u"version",b"version"]) -> None: ... +global___GetFeastServingInfoResponse = GetFeastServingInfoResponse class FeatureReference(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... project = ... # type: typing___Text name = ... # type: typing___Text - version = ... # type: int + version = ... # type: builtin___int @property def max_age(self) -> google___protobuf___duration_pb2___Duration: ... @@ -160,19 +182,20 @@ class FeatureReference(google___protobuf___message___Message): *, project : typing___Optional[typing___Text] = None, name : typing___Optional[typing___Text] = None, - version : typing___Optional[int] = None, + version : typing___Optional[builtin___int] = None, max_age : typing___Optional[google___protobuf___duration_pb2___Duration] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> FeatureReference: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"max_age"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"max_age",u"name",u"project",u"version"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> FeatureReference: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age",u"name",b"name",u"project",b"project",u"version",b"version"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> FeatureReference: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"max_age",b"max_age",u"name",b"name",u"project",b"project",u"version",b"version"]) -> None: ... +global___FeatureReference = FeatureReference class GetOnlineFeaturesRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -190,16 +213,17 @@ class GetOnlineFeaturesRequest(google___protobuf___message___Message): key : typing___Optional[typing___Text] = None, value : typing___Optional[feast___types___Value_pb2___Value] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesRequest.EntityRow.FieldsEntry: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesRequest.EntityRow.FieldsEntry: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetOnlineFeaturesRequest.EntityRow.FieldsEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ... + global___FieldsEntry = FieldsEntry @property @@ -213,64 +237,68 @@ class GetOnlineFeaturesRequest(google___protobuf___message___Message): entity_timestamp : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, fields : typing___Optional[typing___Mapping[typing___Text, feast___types___Value_pb2___Value]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesRequest.EntityRow: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"entity_timestamp"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"entity_timestamp",u"fields"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesRequest.EntityRow: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"entity_timestamp",b"entity_timestamp"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"entity_timestamp",b"entity_timestamp",u"fields",b"fields"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetOnlineFeaturesRequest.EntityRow: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"entity_timestamp",b"entity_timestamp"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"entity_timestamp",b"entity_timestamp",u"fields",b"fields"]) -> None: ... + global___EntityRow = EntityRow - omit_entities_in_response = ... # type: bool + omit_entities_in_response = ... # type: builtin___bool @property - def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[FeatureReference]: ... + def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[global___FeatureReference]: ... @property - def entity_rows(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[GetOnlineFeaturesRequest.EntityRow]: ... + def entity_rows(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[global___GetOnlineFeaturesRequest.EntityRow]: ... def __init__(self, *, - features : typing___Optional[typing___Iterable[FeatureReference]] = None, - entity_rows : typing___Optional[typing___Iterable[GetOnlineFeaturesRequest.EntityRow]] = None, - omit_entities_in_response : typing___Optional[bool] = None, + features : typing___Optional[typing___Iterable[global___FeatureReference]] = None, + entity_rows : typing___Optional[typing___Iterable[global___GetOnlineFeaturesRequest.EntityRow]] = None, + omit_entities_in_response : typing___Optional[builtin___bool] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesRequest: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"entity_rows",u"features",u"omit_entities_in_response"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesRequest: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"entity_rows",b"entity_rows",u"features",b"features",u"omit_entities_in_response",b"omit_entities_in_response"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetOnlineFeaturesRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"entity_rows",b"entity_rows",u"features",b"features",u"omit_entities_in_response",b"omit_entities_in_response"]) -> None: ... +global___GetOnlineFeaturesRequest = GetOnlineFeaturesRequest class GetBatchFeaturesRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property - def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[FeatureReference]: ... + def features(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[global___FeatureReference]: ... @property - def dataset_source(self) -> DatasetSource: ... + def dataset_source(self) -> global___DatasetSource: ... def __init__(self, *, - features : typing___Optional[typing___Iterable[FeatureReference]] = None, - dataset_source : typing___Optional[DatasetSource] = None, + features : typing___Optional[typing___Iterable[global___FeatureReference]] = None, + dataset_source : typing___Optional[global___DatasetSource] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetBatchFeaturesRequest: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"dataset_source"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"dataset_source",u"features"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetBatchFeaturesRequest: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source",u"features",b"features"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetBatchFeaturesRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source",u"features",b"features"]) -> None: ... +global___GetBatchFeaturesRequest = GetBatchFeaturesRequest class GetOnlineFeaturesResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -288,16 +316,17 @@ class GetOnlineFeaturesResponse(google___protobuf___message___Message): key : typing___Optional[typing___Text] = None, value : typing___Optional[feast___types___Value_pb2___Value] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesResponse.FieldValues.FieldsEntry: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"key",u"value"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesResponse.FieldValues.FieldsEntry: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetOnlineFeaturesResponse.FieldValues.FieldsEntry: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"key",b"key",u"value",b"value"]) -> None: ... + global___FieldsEntry = FieldsEntry @property @@ -307,159 +336,171 @@ class GetOnlineFeaturesResponse(google___protobuf___message___Message): *, fields : typing___Optional[typing___Mapping[typing___Text, feast___types___Value_pb2___Value]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesResponse.FieldValues: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"fields"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesResponse.FieldValues: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"fields",b"fields"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetOnlineFeaturesResponse.FieldValues: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"fields",b"fields"]) -> None: ... + global___FieldValues = FieldValues @property - def field_values(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[GetOnlineFeaturesResponse.FieldValues]: ... + def field_values(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[global___GetOnlineFeaturesResponse.FieldValues]: ... def __init__(self, *, - field_values : typing___Optional[typing___Iterable[GetOnlineFeaturesResponse.FieldValues]] = None, + field_values : typing___Optional[typing___Iterable[global___GetOnlineFeaturesResponse.FieldValues]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetOnlineFeaturesResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"field_values"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetOnlineFeaturesResponse: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"field_values",b"field_values"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetOnlineFeaturesResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"field_values",b"field_values"]) -> None: ... +global___GetOnlineFeaturesResponse = GetOnlineFeaturesResponse class GetBatchFeaturesResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property - def job(self) -> Job: ... + def job(self) -> global___Job: ... def __init__(self, *, - job : typing___Optional[Job] = None, + job : typing___Optional[global___Job] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetBatchFeaturesResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"job"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"job"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetBatchFeaturesResponse: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetBatchFeaturesResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> None: ... +global___GetBatchFeaturesResponse = GetBatchFeaturesResponse class GetJobRequest(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property - def job(self) -> Job: ... + def job(self) -> global___Job: ... def __init__(self, *, - job : typing___Optional[Job] = None, + job : typing___Optional[global___Job] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetJobRequest: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"job"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"job"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetJobRequest: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetJobRequest: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> None: ... +global___GetJobRequest = GetJobRequest class GetJobResponse(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @property - def job(self) -> Job: ... + def job(self) -> global___Job: ... def __init__(self, *, - job : typing___Optional[Job] = None, + job : typing___Optional[global___Job] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> GetJobResponse: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"job"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"job"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> GetJobResponse: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> GetJobResponse: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"job",b"job"]) -> None: ... +global___GetJobResponse = GetJobResponse class Job(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... id = ... # type: typing___Text - type = ... # type: JobType - status = ... # type: JobStatus + type = ... # type: global___JobType + status = ... # type: global___JobStatus error = ... # type: typing___Text file_uris = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] - data_format = ... # type: DataFormat + data_format = ... # type: global___DataFormat def __init__(self, *, id : typing___Optional[typing___Text] = None, - type : typing___Optional[JobType] = None, - status : typing___Optional[JobStatus] = None, + type : typing___Optional[global___JobType] = None, + status : typing___Optional[global___JobStatus] = None, error : typing___Optional[typing___Text] = None, file_uris : typing___Optional[typing___Iterable[typing___Text]] = None, - data_format : typing___Optional[DataFormat] = None, + data_format : typing___Optional[global___DataFormat] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Job: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"data_format",u"error",u"file_uris",u"id",u"status",u"type"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Job: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"data_format",b"data_format",u"error",b"error",u"file_uris",b"file_uris",u"id",b"id",u"status",b"status",u"type",b"type"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Job: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"data_format",b"data_format",u"error",b"error",u"file_uris",b"file_uris",u"id",b"id",u"status",b"status",u"type",b"type"]) -> None: ... +global___Job = Job class DatasetSource(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... class FileSource(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... file_uris = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] - data_format = ... # type: DataFormat + data_format = ... # type: global___DataFormat def __init__(self, *, file_uris : typing___Optional[typing___Iterable[typing___Text]] = None, - data_format : typing___Optional[DataFormat] = None, + data_format : typing___Optional[global___DataFormat] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> DatasetSource.FileSource: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"data_format",u"file_uris"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> DatasetSource.FileSource: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"data_format",b"data_format",u"file_uris",b"file_uris"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> DatasetSource.FileSource: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"data_format",b"data_format",u"file_uris",b"file_uris"]) -> None: ... + global___FileSource = FileSource @property - def file_source(self) -> DatasetSource.FileSource: ... + def file_source(self) -> global___DatasetSource.FileSource: ... def __init__(self, *, - file_source : typing___Optional[DatasetSource.FileSource] = None, + file_source : typing___Optional[global___DatasetSource.FileSource] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> DatasetSource: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",u"file_source"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"dataset_source",u"file_source"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> DatasetSource: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source",u"file_source",b"file_source"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source",u"file_source",b"file_source"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> DatasetSource: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source",u"file_source",b"file_source"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"dataset_source",b"dataset_source",u"file_source",b"file_source"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"dataset_source",b"dataset_source"]) -> typing_extensions___Literal["file_source"]: ... +global___DatasetSource = DatasetSource diff --git a/sdk/python/feast/storage/Redis_pb2.py b/sdk/python/feast/storage/Redis_pb2.py index 49b0b793781..2823225d749 100644 --- a/sdk/python/feast/storage/Redis_pb2.py +++ b/sdk/python/feast/storage/Redis_pb2.py @@ -2,8 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/storage/Redis.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -20,8 +18,8 @@ name='feast/storage/Redis.proto', package='feast.storage', syntax='proto3', - serialized_options=_b('\n\rfeast.storageB\nRedisProtoZ2github.com/gojek/feast/sdk/go/protos/feast/storage'), - serialized_pb=_b('\n\x19\x66\x65\x61st/storage/Redis.proto\x12\rfeast.storage\x1a\x17\x66\x65\x61st/types/Field.proto\"E\n\x08RedisKey\x12\x13\n\x0b\x66\x65\x61ture_set\x18\x02 \x01(\t\x12$\n\x08\x65ntities\x18\x03 \x03(\x0b\x32\x12.feast.types.FieldBO\n\rfeast.storageB\nRedisProtoZ2github.com/gojek/feast/sdk/go/protos/feast/storageb\x06proto3') + serialized_options=b'\n\rfeast.storageB\nRedisProtoZ2github.com/gojek/feast/sdk/go/protos/feast/storage', + serialized_pb=b'\n\x19\x66\x65\x61st/storage/Redis.proto\x12\rfeast.storage\x1a\x17\x66\x65\x61st/types/Field.proto\"E\n\x08RedisKey\x12\x13\n\x0b\x66\x65\x61ture_set\x18\x02 \x01(\t\x12$\n\x08\x65ntities\x18\x03 \x03(\x0b\x32\x12.feast.types.FieldBO\n\rfeast.storageB\nRedisProtoZ2github.com/gojek/feast/sdk/go/protos/feast/storageb\x06proto3' , dependencies=[feast_dot_types_dot_Field__pb2.DESCRIPTOR,]) @@ -38,7 +36,7 @@ _descriptor.FieldDescriptor( name='feature_set', full_name='feast.storage.RedisKey.feature_set', index=0, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), diff --git a/sdk/python/feast/storage/Redis_pb2.pyi b/sdk/python/feast/storage/Redis_pb2.pyi index 717aae79db2..4235978f55b 100644 --- a/sdk/python/feast/storage/Redis_pb2.pyi +++ b/sdk/python/feast/storage/Redis_pb2.pyi @@ -20,6 +20,7 @@ from typing import ( Iterable as typing___Iterable, Optional as typing___Optional, Text as typing___Text, + Union as typing___Union, ) from typing_extensions import ( @@ -27,6 +28,15 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +if sys.version_info < (3,): + builtin___buffer = buffer + builtin___unicode = unicode + + class RedisKey(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... feature_set = ... # type: typing___Text @@ -39,11 +49,13 @@ class RedisKey(google___protobuf___message___Message): feature_set : typing___Optional[typing___Text] = None, entities : typing___Optional[typing___Iterable[feast___types___Field_pb2___Field]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> RedisKey: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"entities",u"feature_set"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> RedisKey: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"entities",b"entities",u"feature_set",b"feature_set"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> RedisKey: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"entities",b"entities",u"feature_set",b"feature_set"]) -> None: ... +global___RedisKey = RedisKey diff --git a/sdk/python/feast/types/FeatureRowExtended_pb2.py b/sdk/python/feast/types/FeatureRowExtended_pb2.py index e7372958168..6634163339f 100644 --- a/sdk/python/feast/types/FeatureRowExtended_pb2.py +++ b/sdk/python/feast/types/FeatureRowExtended_pb2.py @@ -2,8 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/types/FeatureRowExtended.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -21,8 +19,8 @@ name='feast/types/FeatureRowExtended.proto', package='feast.types', syntax='proto3', - serialized_options=_b('\n\013feast.typesB\027FeatureRowExtendedProtoZ0github.com/gojek/feast/sdk/go/protos/feast/types'), - serialized_pb=_b('\n$feast/types/FeatureRowExtended.proto\x12\x0b\x66\x65\x61st.types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1c\x66\x65\x61st/types/FeatureRow.proto\"O\n\x05\x45rror\x12\r\n\x05\x63\x61use\x18\x01 \x01(\t\x12\x11\n\ttransform\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12\x13\n\x0bstack_trace\x18\x04 \x01(\t\">\n\x07\x41ttempt\x12\x10\n\x08\x61ttempts\x18\x01 \x01(\x05\x12!\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x12.feast.types.Error\"\x96\x01\n\x12\x46\x65\x61tureRowExtended\x12$\n\x03row\x18\x01 \x01(\x0b\x32\x17.feast.types.FeatureRow\x12*\n\x0clast_attempt\x18\x02 \x01(\x0b\x32\x14.feast.types.Attempt\x12.\n\nfirst_seen\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampBX\n\x0b\x66\x65\x61st.typesB\x17\x46\x65\x61tureRowExtendedProtoZ0github.com/gojek/feast/sdk/go/protos/feast/typesb\x06proto3') + serialized_options=b'\n\013feast.typesB\027FeatureRowExtendedProtoZ0github.com/gojek/feast/sdk/go/protos/feast/types', + serialized_pb=b'\n$feast/types/FeatureRowExtended.proto\x12\x0b\x66\x65\x61st.types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1c\x66\x65\x61st/types/FeatureRow.proto\"O\n\x05\x45rror\x12\r\n\x05\x63\x61use\x18\x01 \x01(\t\x12\x11\n\ttransform\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\x12\x13\n\x0bstack_trace\x18\x04 \x01(\t\">\n\x07\x41ttempt\x12\x10\n\x08\x61ttempts\x18\x01 \x01(\x05\x12!\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x12.feast.types.Error\"\x96\x01\n\x12\x46\x65\x61tureRowExtended\x12$\n\x03row\x18\x01 \x01(\x0b\x32\x17.feast.types.FeatureRow\x12*\n\x0clast_attempt\x18\x02 \x01(\x0b\x32\x14.feast.types.Attempt\x12.\n\nfirst_seen\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampBX\n\x0b\x66\x65\x61st.typesB\x17\x46\x65\x61tureRowExtendedProtoZ0github.com/gojek/feast/sdk/go/protos/feast/typesb\x06proto3' , dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,feast_dot_types_dot_FeatureRow__pb2.DESCRIPTOR,]) @@ -39,28 +37,28 @@ _descriptor.FieldDescriptor( name='cause', full_name='feast.types.Error.cause', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='transform', full_name='feast.types.Error.transform', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='message', full_name='feast.types.Error.message', index=2, number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='stack_trace', full_name='feast.types.Error.stack_trace', index=3, number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), diff --git a/sdk/python/feast/types/FeatureRowExtended_pb2.pyi b/sdk/python/feast/types/FeatureRowExtended_pb2.pyi index 4f3d02c8ee6..0e2599fa530 100644 --- a/sdk/python/feast/types/FeatureRowExtended_pb2.pyi +++ b/sdk/python/feast/types/FeatureRowExtended_pb2.pyi @@ -19,6 +19,7 @@ from google.protobuf.timestamp_pb2 import ( from typing import ( Optional as typing___Optional, Text as typing___Text, + Union as typing___Union, ) from typing_extensions import ( @@ -26,6 +27,15 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +if sys.version_info < (3,): + builtin___buffer = buffer + builtin___unicode = unicode + + class Error(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... cause = ... # type: typing___Text @@ -40,37 +50,40 @@ class Error(google___protobuf___message___Message): message : typing___Optional[typing___Text] = None, stack_trace : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Error: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"cause",u"message",u"stack_trace",u"transform"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Error: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"cause",b"cause",u"message",b"message",u"stack_trace",b"stack_trace",u"transform",b"transform"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Error: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"cause",b"cause",u"message",b"message",u"stack_trace",b"stack_trace",u"transform",b"transform"]) -> None: ... +global___Error = Error class Attempt(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - attempts = ... # type: int + attempts = ... # type: builtin___int @property - def error(self) -> Error: ... + def error(self) -> global___Error: ... def __init__(self, *, - attempts : typing___Optional[int] = None, - error : typing___Optional[Error] = None, + attempts : typing___Optional[builtin___int] = None, + error : typing___Optional[global___Error] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Attempt: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"error"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"attempts",u"error"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Attempt: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"error",b"error"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"attempts",b"attempts",u"error",b"error"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Attempt: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"error",b"error"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"attempts",b"attempts",u"error",b"error"]) -> None: ... +global___Attempt = Attempt class FeatureRowExtended(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -79,7 +92,7 @@ class FeatureRowExtended(google___protobuf___message___Message): def row(self) -> feast___types___FeatureRow_pb2___FeatureRow: ... @property - def last_attempt(self) -> Attempt: ... + def last_attempt(self) -> global___Attempt: ... @property def first_seen(self) -> google___protobuf___timestamp_pb2___Timestamp: ... @@ -87,16 +100,17 @@ class FeatureRowExtended(google___protobuf___message___Message): def __init__(self, *, row : typing___Optional[feast___types___FeatureRow_pb2___FeatureRow] = None, - last_attempt : typing___Optional[Attempt] = None, + last_attempt : typing___Optional[global___Attempt] = None, first_seen : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> FeatureRowExtended: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"first_seen",u"last_attempt",u"row"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"first_seen",u"last_attempt",u"row"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> FeatureRowExtended: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"first_seen",b"first_seen",u"last_attempt",b"last_attempt",u"row",b"row"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"first_seen",b"first_seen",u"last_attempt",b"last_attempt",u"row",b"row"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> FeatureRowExtended: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"first_seen",b"first_seen",u"last_attempt",b"last_attempt",u"row",b"row"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"first_seen",b"first_seen",u"last_attempt",b"last_attempt",u"row",b"row"]) -> None: ... +global___FeatureRowExtended = FeatureRowExtended diff --git a/sdk/python/feast/types/FeatureRow_pb2.py b/sdk/python/feast/types/FeatureRow_pb2.py index 1b6c16910f2..ff5ac8a956d 100644 --- a/sdk/python/feast/types/FeatureRow_pb2.py +++ b/sdk/python/feast/types/FeatureRow_pb2.py @@ -2,8 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/types/FeatureRow.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -21,8 +19,8 @@ name='feast/types/FeatureRow.proto', package='feast.types', syntax='proto3', - serialized_options=_b('\n\013feast.typesB\017FeatureRowProtoZ0github.com/gojek/feast/sdk/go/protos/feast/types'), - serialized_pb=_b('\n\x1c\x66\x65\x61st/types/FeatureRow.proto\x12\x0b\x66\x65\x61st.types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17\x66\x65\x61st/types/Field.proto\"z\n\nFeatureRow\x12\"\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x12.feast.types.Field\x12\x33\n\x0f\x65vent_timestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x13\n\x0b\x66\x65\x61ture_set\x18\x06 \x01(\tBP\n\x0b\x66\x65\x61st.typesB\x0f\x46\x65\x61tureRowProtoZ0github.com/gojek/feast/sdk/go/protos/feast/typesb\x06proto3') + serialized_options=b'\n\013feast.typesB\017FeatureRowProtoZ0github.com/gojek/feast/sdk/go/protos/feast/types', + serialized_pb=b'\n\x1c\x66\x65\x61st/types/FeatureRow.proto\x12\x0b\x66\x65\x61st.types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17\x66\x65\x61st/types/Field.proto\"z\n\nFeatureRow\x12\"\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x12.feast.types.Field\x12\x33\n\x0f\x65vent_timestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x13\n\x0b\x66\x65\x61ture_set\x18\x06 \x01(\tBP\n\x0b\x66\x65\x61st.typesB\x0f\x46\x65\x61tureRowProtoZ0github.com/gojek/feast/sdk/go/protos/feast/typesb\x06proto3' , dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,feast_dot_types_dot_Field__pb2.DESCRIPTOR,]) @@ -53,7 +51,7 @@ _descriptor.FieldDescriptor( name='feature_set', full_name='feast.types.FeatureRow.feature_set', index=2, number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), diff --git a/sdk/python/feast/types/FeatureRow_pb2.pyi b/sdk/python/feast/types/FeatureRow_pb2.pyi index 9bf745f9130..ca464c86d99 100644 --- a/sdk/python/feast/types/FeatureRow_pb2.pyi +++ b/sdk/python/feast/types/FeatureRow_pb2.pyi @@ -24,6 +24,7 @@ from typing import ( Iterable as typing___Iterable, Optional as typing___Optional, Text as typing___Text, + Union as typing___Union, ) from typing_extensions import ( @@ -31,6 +32,15 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +if sys.version_info < (3,): + builtin___buffer = buffer + builtin___unicode = unicode + + class FeatureRow(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... feature_set = ... # type: typing___Text @@ -47,13 +57,14 @@ class FeatureRow(google___protobuf___message___Message): event_timestamp : typing___Optional[google___protobuf___timestamp_pb2___Timestamp] = None, feature_set : typing___Optional[typing___Text] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> FeatureRow: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"event_timestamp"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"event_timestamp",u"feature_set",u"fields"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> FeatureRow: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"event_timestamp",b"event_timestamp"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"event_timestamp",b"event_timestamp",u"feature_set",b"feature_set",u"fields",b"fields"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> FeatureRow: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"event_timestamp",b"event_timestamp"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"event_timestamp",b"event_timestamp",u"feature_set",b"feature_set",u"fields",b"fields"]) -> None: ... +global___FeatureRow = FeatureRow diff --git a/sdk/python/feast/types/Field_pb2.py b/sdk/python/feast/types/Field_pb2.py index 95bcf38cf9d..67cee04f961 100644 --- a/sdk/python/feast/types/Field_pb2.py +++ b/sdk/python/feast/types/Field_pb2.py @@ -2,8 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/types/Field.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -20,8 +18,8 @@ name='feast/types/Field.proto', package='feast.types', syntax='proto3', - serialized_options=_b('\n\013feast.typesB\nFieldProtoZ0github.com/gojek/feast/sdk/go/protos/feast/types'), - serialized_pb=_b('\n\x17\x66\x65\x61st/types/Field.proto\x12\x0b\x66\x65\x61st.types\x1a\x17\x66\x65\x61st/types/Value.proto\"8\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.feast.types.ValueBK\n\x0b\x66\x65\x61st.typesB\nFieldProtoZ0github.com/gojek/feast/sdk/go/protos/feast/typesb\x06proto3') + serialized_options=b'\n\013feast.typesB\nFieldProtoZ0github.com/gojek/feast/sdk/go/protos/feast/types', + serialized_pb=b'\n\x17\x66\x65\x61st/types/Field.proto\x12\x0b\x66\x65\x61st.types\x1a\x17\x66\x65\x61st/types/Value.proto\"8\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.feast.types.ValueBK\n\x0b\x66\x65\x61st.typesB\nFieldProtoZ0github.com/gojek/feast/sdk/go/protos/feast/typesb\x06proto3' , dependencies=[feast_dot_types_dot_Value__pb2.DESCRIPTOR,]) @@ -38,7 +36,7 @@ _descriptor.FieldDescriptor( name='name', full_name='feast.types.Field.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), diff --git a/sdk/python/feast/types/Field_pb2.pyi b/sdk/python/feast/types/Field_pb2.pyi index 1305503fab7..97239b00710 100644 --- a/sdk/python/feast/types/Field_pb2.pyi +++ b/sdk/python/feast/types/Field_pb2.pyi @@ -15,6 +15,7 @@ from google.protobuf.message import ( from typing import ( Optional as typing___Optional, Text as typing___Text, + Union as typing___Union, ) from typing_extensions import ( @@ -22,6 +23,15 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +if sys.version_info < (3,): + builtin___buffer = buffer + builtin___unicode = unicode + + class Field(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... name = ... # type: typing___Text @@ -34,13 +44,14 @@ class Field(google___protobuf___message___Message): name : typing___Optional[typing___Text] = None, value : typing___Optional[feast___types___Value_pb2___Value] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Field: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"value"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"name",u"value"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Field: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"value",b"value"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Field: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"value",b"value"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"value",b"value"]) -> None: ... +global___Field = Field diff --git a/sdk/python/feast/types/Value_pb2.py b/sdk/python/feast/types/Value_pb2.py index fe2cd125ca5..7796c7e4161 100644 --- a/sdk/python/feast/types/Value_pb2.py +++ b/sdk/python/feast/types/Value_pb2.py @@ -2,8 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: feast/types/Value.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -19,8 +17,8 @@ name='feast/types/Value.proto', package='feast.types', syntax='proto3', - serialized_options=_b('\n\013feast.typesB\nValueProtoZ0github.com/gojek/feast/sdk/go/protos/feast/types'), - serialized_pb=_b('\n\x17\x66\x65\x61st/types/Value.proto\x12\x0b\x66\x65\x61st.types\"\xe0\x01\n\tValueType\"\xd2\x01\n\x04\x45num\x12\x0b\n\x07INVALID\x10\x00\x12\t\n\x05\x42YTES\x10\x01\x12\n\n\x06STRING\x10\x02\x12\t\n\x05INT32\x10\x03\x12\t\n\x05INT64\x10\x04\x12\n\n\x06\x44OUBLE\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\x08\n\x04\x42OOL\x10\x07\x12\x0e\n\nBYTES_LIST\x10\x0b\x12\x0f\n\x0bSTRING_LIST\x10\x0c\x12\x0e\n\nINT32_LIST\x10\r\x12\x0e\n\nINT64_LIST\x10\x0e\x12\x0f\n\x0b\x44OUBLE_LIST\x10\x0f\x12\x0e\n\nFLOAT_LIST\x10\x10\x12\r\n\tBOOL_LIST\x10\x11\"\x82\x04\n\x05Value\x12\x13\n\tbytes_val\x18\x01 \x01(\x0cH\x00\x12\x14\n\nstring_val\x18\x02 \x01(\tH\x00\x12\x13\n\tint32_val\x18\x03 \x01(\x05H\x00\x12\x13\n\tint64_val\x18\x04 \x01(\x03H\x00\x12\x14\n\ndouble_val\x18\x05 \x01(\x01H\x00\x12\x13\n\tfloat_val\x18\x06 \x01(\x02H\x00\x12\x12\n\x08\x62ool_val\x18\x07 \x01(\x08H\x00\x12\x30\n\x0e\x62ytes_list_val\x18\x0b \x01(\x0b\x32\x16.feast.types.BytesListH\x00\x12\x32\n\x0fstring_list_val\x18\x0c \x01(\x0b\x32\x17.feast.types.StringListH\x00\x12\x30\n\x0eint32_list_val\x18\r \x01(\x0b\x32\x16.feast.types.Int32ListH\x00\x12\x30\n\x0eint64_list_val\x18\x0e \x01(\x0b\x32\x16.feast.types.Int64ListH\x00\x12\x32\n\x0f\x64ouble_list_val\x18\x0f \x01(\x0b\x32\x17.feast.types.DoubleListH\x00\x12\x30\n\x0e\x66loat_list_val\x18\x10 \x01(\x0b\x32\x16.feast.types.FloatListH\x00\x12.\n\rbool_list_val\x18\x11 \x01(\x0b\x32\x15.feast.types.BoolListH\x00\x42\x05\n\x03val\"\x18\n\tBytesList\x12\x0b\n\x03val\x18\x01 \x03(\x0c\"\x19\n\nStringList\x12\x0b\n\x03val\x18\x01 \x03(\t\"\x18\n\tInt32List\x12\x0b\n\x03val\x18\x01 \x03(\x05\"\x18\n\tInt64List\x12\x0b\n\x03val\x18\x01 \x03(\x03\"\x19\n\nDoubleList\x12\x0b\n\x03val\x18\x01 \x03(\x01\"\x18\n\tFloatList\x12\x0b\n\x03val\x18\x01 \x03(\x02\"\x17\n\x08\x42oolList\x12\x0b\n\x03val\x18\x01 \x03(\x08\x42K\n\x0b\x66\x65\x61st.typesB\nValueProtoZ0github.com/gojek/feast/sdk/go/protos/feast/typesb\x06proto3') + serialized_options=b'\n\013feast.typesB\nValueProtoZ0github.com/gojek/feast/sdk/go/protos/feast/types', + serialized_pb=b'\n\x17\x66\x65\x61st/types/Value.proto\x12\x0b\x66\x65\x61st.types\"\xe0\x01\n\tValueType\"\xd2\x01\n\x04\x45num\x12\x0b\n\x07INVALID\x10\x00\x12\t\n\x05\x42YTES\x10\x01\x12\n\n\x06STRING\x10\x02\x12\t\n\x05INT32\x10\x03\x12\t\n\x05INT64\x10\x04\x12\n\n\x06\x44OUBLE\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\x08\n\x04\x42OOL\x10\x07\x12\x0e\n\nBYTES_LIST\x10\x0b\x12\x0f\n\x0bSTRING_LIST\x10\x0c\x12\x0e\n\nINT32_LIST\x10\r\x12\x0e\n\nINT64_LIST\x10\x0e\x12\x0f\n\x0b\x44OUBLE_LIST\x10\x0f\x12\x0e\n\nFLOAT_LIST\x10\x10\x12\r\n\tBOOL_LIST\x10\x11\"\x82\x04\n\x05Value\x12\x13\n\tbytes_val\x18\x01 \x01(\x0cH\x00\x12\x14\n\nstring_val\x18\x02 \x01(\tH\x00\x12\x13\n\tint32_val\x18\x03 \x01(\x05H\x00\x12\x13\n\tint64_val\x18\x04 \x01(\x03H\x00\x12\x14\n\ndouble_val\x18\x05 \x01(\x01H\x00\x12\x13\n\tfloat_val\x18\x06 \x01(\x02H\x00\x12\x12\n\x08\x62ool_val\x18\x07 \x01(\x08H\x00\x12\x30\n\x0e\x62ytes_list_val\x18\x0b \x01(\x0b\x32\x16.feast.types.BytesListH\x00\x12\x32\n\x0fstring_list_val\x18\x0c \x01(\x0b\x32\x17.feast.types.StringListH\x00\x12\x30\n\x0eint32_list_val\x18\r \x01(\x0b\x32\x16.feast.types.Int32ListH\x00\x12\x30\n\x0eint64_list_val\x18\x0e \x01(\x0b\x32\x16.feast.types.Int64ListH\x00\x12\x32\n\x0f\x64ouble_list_val\x18\x0f \x01(\x0b\x32\x17.feast.types.DoubleListH\x00\x12\x30\n\x0e\x66loat_list_val\x18\x10 \x01(\x0b\x32\x16.feast.types.FloatListH\x00\x12.\n\rbool_list_val\x18\x11 \x01(\x0b\x32\x15.feast.types.BoolListH\x00\x42\x05\n\x03val\"\x18\n\tBytesList\x12\x0b\n\x03val\x18\x01 \x03(\x0c\"\x19\n\nStringList\x12\x0b\n\x03val\x18\x01 \x03(\t\"\x18\n\tInt32List\x12\x0b\n\x03val\x18\x01 \x03(\x05\"\x18\n\tInt64List\x12\x0b\n\x03val\x18\x01 \x03(\x03\"\x19\n\nDoubleList\x12\x0b\n\x03val\x18\x01 \x03(\x01\"\x18\n\tFloatList\x12\x0b\n\x03val\x18\x01 \x03(\x02\"\x17\n\x08\x42oolList\x12\x0b\n\x03val\x18\x01 \x03(\x08\x42K\n\x0b\x66\x65\x61st.typesB\nValueProtoZ0github.com/gojek/feast/sdk/go/protos/feast/typesb\x06proto3' ) @@ -135,14 +133,14 @@ _descriptor.FieldDescriptor( name='bytes_val', full_name='feast.types.Value.bytes_val', index=0, number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='string_val', full_name='feast.types.Value.string_val', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), diff --git a/sdk/python/feast/types/Value_pb2.pyi b/sdk/python/feast/types/Value_pb2.pyi index d8b8a73dd36..9b8c450ca03 100644 --- a/sdk/python/feast/types/Value_pb2.pyi +++ b/sdk/python/feast/types/Value_pb2.pyi @@ -19,6 +19,7 @@ from typing import ( Optional as typing___Optional, Text as typing___Text, Tuple as typing___Tuple, + Union as typing___Union, cast as typing___cast, ) @@ -27,135 +28,154 @@ from typing_extensions import ( ) +builtin___bool = bool +builtin___bytes = bytes +builtin___float = float +builtin___int = int +builtin___str = str +if sys.version_info < (3,): + builtin___buffer = buffer + builtin___unicode = unicode + + class ValueType(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - class Enum(int): + class Enum(builtin___int): DESCRIPTOR: google___protobuf___descriptor___EnumDescriptor = ... @classmethod - def Name(cls, number: int) -> str: ... + def Name(cls, number: builtin___int) -> builtin___str: ... @classmethod - def Value(cls, name: str) -> ValueType.Enum: ... + def Value(cls, name: builtin___str) -> 'ValueType.Enum': ... @classmethod - def keys(cls) -> typing___List[str]: ... + def keys(cls) -> typing___List[builtin___str]: ... @classmethod - def values(cls) -> typing___List[ValueType.Enum]: ... + def values(cls) -> typing___List['ValueType.Enum']: ... @classmethod - def items(cls) -> typing___List[typing___Tuple[str, ValueType.Enum]]: ... - INVALID = typing___cast(ValueType.Enum, 0) - BYTES = typing___cast(ValueType.Enum, 1) - STRING = typing___cast(ValueType.Enum, 2) - INT32 = typing___cast(ValueType.Enum, 3) - INT64 = typing___cast(ValueType.Enum, 4) - DOUBLE = typing___cast(ValueType.Enum, 5) - FLOAT = typing___cast(ValueType.Enum, 6) - BOOL = typing___cast(ValueType.Enum, 7) - BYTES_LIST = typing___cast(ValueType.Enum, 11) - STRING_LIST = typing___cast(ValueType.Enum, 12) - INT32_LIST = typing___cast(ValueType.Enum, 13) - INT64_LIST = typing___cast(ValueType.Enum, 14) - DOUBLE_LIST = typing___cast(ValueType.Enum, 15) - FLOAT_LIST = typing___cast(ValueType.Enum, 16) - BOOL_LIST = typing___cast(ValueType.Enum, 17) - INVALID = typing___cast(ValueType.Enum, 0) - BYTES = typing___cast(ValueType.Enum, 1) - STRING = typing___cast(ValueType.Enum, 2) - INT32 = typing___cast(ValueType.Enum, 3) - INT64 = typing___cast(ValueType.Enum, 4) - DOUBLE = typing___cast(ValueType.Enum, 5) - FLOAT = typing___cast(ValueType.Enum, 6) - BOOL = typing___cast(ValueType.Enum, 7) - BYTES_LIST = typing___cast(ValueType.Enum, 11) - STRING_LIST = typing___cast(ValueType.Enum, 12) - INT32_LIST = typing___cast(ValueType.Enum, 13) - INT64_LIST = typing___cast(ValueType.Enum, 14) - DOUBLE_LIST = typing___cast(ValueType.Enum, 15) - FLOAT_LIST = typing___cast(ValueType.Enum, 16) - BOOL_LIST = typing___cast(ValueType.Enum, 17) + def items(cls) -> typing___List[typing___Tuple[builtin___str, 'ValueType.Enum']]: ... + INVALID = typing___cast('ValueType.Enum', 0) + BYTES = typing___cast('ValueType.Enum', 1) + STRING = typing___cast('ValueType.Enum', 2) + INT32 = typing___cast('ValueType.Enum', 3) + INT64 = typing___cast('ValueType.Enum', 4) + DOUBLE = typing___cast('ValueType.Enum', 5) + FLOAT = typing___cast('ValueType.Enum', 6) + BOOL = typing___cast('ValueType.Enum', 7) + BYTES_LIST = typing___cast('ValueType.Enum', 11) + STRING_LIST = typing___cast('ValueType.Enum', 12) + INT32_LIST = typing___cast('ValueType.Enum', 13) + INT64_LIST = typing___cast('ValueType.Enum', 14) + DOUBLE_LIST = typing___cast('ValueType.Enum', 15) + FLOAT_LIST = typing___cast('ValueType.Enum', 16) + BOOL_LIST = typing___cast('ValueType.Enum', 17) + INVALID = typing___cast('ValueType.Enum', 0) + BYTES = typing___cast('ValueType.Enum', 1) + STRING = typing___cast('ValueType.Enum', 2) + INT32 = typing___cast('ValueType.Enum', 3) + INT64 = typing___cast('ValueType.Enum', 4) + DOUBLE = typing___cast('ValueType.Enum', 5) + FLOAT = typing___cast('ValueType.Enum', 6) + BOOL = typing___cast('ValueType.Enum', 7) + BYTES_LIST = typing___cast('ValueType.Enum', 11) + STRING_LIST = typing___cast('ValueType.Enum', 12) + INT32_LIST = typing___cast('ValueType.Enum', 13) + INT64_LIST = typing___cast('ValueType.Enum', 14) + DOUBLE_LIST = typing___cast('ValueType.Enum', 15) + FLOAT_LIST = typing___cast('ValueType.Enum', 16) + BOOL_LIST = typing___cast('ValueType.Enum', 17) + global___Enum = Enum def __init__(self, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> ValueType: ... + if sys.version_info >= (3,): + @classmethod + def FromString(cls, s: builtin___bytes) -> ValueType: ... + else: + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> ValueType: ... def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... +global___ValueType = ValueType class Value(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - bytes_val = ... # type: bytes + bytes_val = ... # type: builtin___bytes string_val = ... # type: typing___Text - int32_val = ... # type: int - int64_val = ... # type: int - double_val = ... # type: float - float_val = ... # type: float - bool_val = ... # type: bool + int32_val = ... # type: builtin___int + int64_val = ... # type: builtin___int + double_val = ... # type: builtin___float + float_val = ... # type: builtin___float + bool_val = ... # type: builtin___bool @property - def bytes_list_val(self) -> BytesList: ... + def bytes_list_val(self) -> global___BytesList: ... @property - def string_list_val(self) -> StringList: ... + def string_list_val(self) -> global___StringList: ... @property - def int32_list_val(self) -> Int32List: ... + def int32_list_val(self) -> global___Int32List: ... @property - def int64_list_val(self) -> Int64List: ... + def int64_list_val(self) -> global___Int64List: ... @property - def double_list_val(self) -> DoubleList: ... + def double_list_val(self) -> global___DoubleList: ... @property - def float_list_val(self) -> FloatList: ... + def float_list_val(self) -> global___FloatList: ... @property - def bool_list_val(self) -> BoolList: ... + def bool_list_val(self) -> global___BoolList: ... def __init__(self, *, - bytes_val : typing___Optional[bytes] = None, + bytes_val : typing___Optional[builtin___bytes] = None, string_val : typing___Optional[typing___Text] = None, - int32_val : typing___Optional[int] = None, - int64_val : typing___Optional[int] = None, - double_val : typing___Optional[float] = None, - float_val : typing___Optional[float] = None, - bool_val : typing___Optional[bool] = None, - bytes_list_val : typing___Optional[BytesList] = None, - string_list_val : typing___Optional[StringList] = None, - int32_list_val : typing___Optional[Int32List] = None, - int64_list_val : typing___Optional[Int64List] = None, - double_list_val : typing___Optional[DoubleList] = None, - float_list_val : typing___Optional[FloatList] = None, - bool_list_val : typing___Optional[BoolList] = None, + int32_val : typing___Optional[builtin___int] = None, + int64_val : typing___Optional[builtin___int] = None, + double_val : typing___Optional[builtin___float] = None, + float_val : typing___Optional[builtin___float] = None, + bool_val : typing___Optional[builtin___bool] = None, + bytes_list_val : typing___Optional[global___BytesList] = None, + string_list_val : typing___Optional[global___StringList] = None, + int32_list_val : typing___Optional[global___Int32List] = None, + int64_list_val : typing___Optional[global___Int64List] = None, + double_list_val : typing___Optional[global___DoubleList] = None, + float_list_val : typing___Optional[global___FloatList] = None, + bool_list_val : typing___Optional[global___BoolList] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Value: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"bool_list_val",u"bool_val",u"bytes_list_val",u"bytes_val",u"double_list_val",u"double_val",u"float_list_val",u"float_val",u"int32_list_val",u"int32_val",u"int64_list_val",u"int64_val",u"string_list_val",u"string_val",u"val"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"bool_list_val",u"bool_val",u"bytes_list_val",u"bytes_val",u"double_list_val",u"double_val",u"float_list_val",u"float_val",u"int32_list_val",u"int32_val",u"int64_list_val",u"int64_val",u"string_list_val",u"string_val",u"val"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Value: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"bool_list_val",b"bool_list_val",u"bool_val",b"bool_val",u"bytes_list_val",b"bytes_list_val",u"bytes_val",b"bytes_val",u"double_list_val",b"double_list_val",u"double_val",b"double_val",u"float_list_val",b"float_list_val",u"float_val",b"float_val",u"int32_list_val",b"int32_list_val",u"int32_val",b"int32_val",u"int64_list_val",b"int64_list_val",u"int64_val",b"int64_val",u"string_list_val",b"string_list_val",u"string_val",b"string_val",u"val",b"val"]) -> bool: ... - def ClearField(self, field_name: typing_extensions___Literal[u"bool_list_val",b"bool_list_val",u"bool_val",b"bool_val",u"bytes_list_val",b"bytes_list_val",u"bytes_val",b"bytes_val",u"double_list_val",b"double_list_val",u"double_val",b"double_val",u"float_list_val",b"float_list_val",u"float_val",b"float_val",u"int32_list_val",b"int32_list_val",u"int32_val",b"int32_val",u"int64_list_val",b"int64_list_val",u"int64_val",b"int64_val",u"string_list_val",b"string_list_val",u"string_val",b"string_val",u"val",b"val"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Value: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def HasField(self, field_name: typing_extensions___Literal[u"bool_list_val",b"bool_list_val",u"bool_val",b"bool_val",u"bytes_list_val",b"bytes_list_val",u"bytes_val",b"bytes_val",u"double_list_val",b"double_list_val",u"double_val",b"double_val",u"float_list_val",b"float_list_val",u"float_val",b"float_val",u"int32_list_val",b"int32_list_val",u"int32_val",b"int32_val",u"int64_list_val",b"int64_list_val",u"int64_val",b"int64_val",u"string_list_val",b"string_list_val",u"string_val",b"string_val",u"val",b"val"]) -> builtin___bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"bool_list_val",b"bool_list_val",u"bool_val",b"bool_val",u"bytes_list_val",b"bytes_list_val",u"bytes_val",b"bytes_val",u"double_list_val",b"double_list_val",u"double_val",b"double_val",u"float_list_val",b"float_list_val",u"float_val",b"float_val",u"int32_list_val",b"int32_list_val",u"int32_val",b"int32_val",u"int64_list_val",b"int64_list_val",u"int64_val",b"int64_val",u"string_list_val",b"string_list_val",u"string_val",b"string_val",u"val",b"val"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions___Literal[u"val",b"val"]) -> typing_extensions___Literal["bytes_val","string_val","int32_val","int64_val","double_val","float_val","bool_val","bytes_list_val","string_list_val","int32_list_val","int64_list_val","double_list_val","float_list_val","bool_list_val"]: ... +global___Value = Value class BytesList(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[bytes] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___bytes] def __init__(self, *, - val : typing___Optional[typing___Iterable[bytes]] = None, + val : typing___Optional[typing___Iterable[builtin___bytes]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> BytesList: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> BytesList: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> BytesList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... +global___BytesList = BytesList class StringList(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... @@ -165,96 +185,108 @@ class StringList(google___protobuf___message___Message): *, val : typing___Optional[typing___Iterable[typing___Text]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> StringList: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> StringList: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> StringList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... +global___StringList = StringList class Int32List(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[int] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] def __init__(self, *, - val : typing___Optional[typing___Iterable[int]] = None, + val : typing___Optional[typing___Iterable[builtin___int]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Int32List: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Int32List: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Int32List: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... +global___Int32List = Int32List class Int64List(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[int] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___int] def __init__(self, *, - val : typing___Optional[typing___Iterable[int]] = None, + val : typing___Optional[typing___Iterable[builtin___int]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> Int64List: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> Int64List: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> Int64List: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... +global___Int64List = Int64List class DoubleList(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[float] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float] def __init__(self, *, - val : typing___Optional[typing___Iterable[float]] = None, + val : typing___Optional[typing___Iterable[builtin___float]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> DoubleList: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> DoubleList: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> DoubleList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... +global___DoubleList = DoubleList class FloatList(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[float] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___float] def __init__(self, *, - val : typing___Optional[typing___Iterable[float]] = None, + val : typing___Optional[typing___Iterable[builtin___float]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> FloatList: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> FloatList: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> FloatList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... +global___FloatList = FloatList class BoolList(google___protobuf___message___Message): DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... - val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[bool] + val = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[builtin___bool] def __init__(self, *, - val : typing___Optional[typing___Iterable[bool]] = None, + val : typing___Optional[typing___Iterable[builtin___bool]] = None, ) -> None: ... - @classmethod - def FromString(cls, s: bytes) -> BoolList: ... - def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... - def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def ClearField(self, field_name: typing_extensions___Literal[u"val"]) -> None: ... + @classmethod + def FromString(cls, s: builtin___bytes) -> BoolList: ... else: - def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... + @classmethod + def FromString(cls, s: typing___Union[builtin___bytes, builtin___buffer, builtin___unicode]) -> BoolList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def ClearField(self, field_name: typing_extensions___Literal[u"val",b"val"]) -> None: ... +global___BoolList = BoolList diff --git a/sdk/python/requirements-ci.txt b/sdk/python/requirements-ci.txt index f3df60a02ec..31818ba7f7b 100644 --- a/sdk/python/requirements-ci.txt +++ b/sdk/python/requirements-ci.txt @@ -1,3 +1,4 @@ +Click==7.* google-api-core==1.* google-auth==1.* google-cloud-bigquery==1.* @@ -11,12 +12,20 @@ mock==2.0.0 pandas==0.* protobuf==3.* pytest +pytest-lazy-fixture==0.6.3 pytest-mock pytest-timeout -PyYAML==5.1.2 -fastavro==0.21.* +PyYAML==5.1.* +fastavro==0.* grpcio-testing==1.* pytest-ordering==0.6.* pyarrow Sphinx -sphinx-rtd-theme \ No newline at end of file +sphinx-rtd-theme +toml==0.10.* +tqdm==4.* +confluent_kafka +google +pandavro==1.5.* +kafka-python==1.* +tabulate==0.8.* \ No newline at end of file diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 1617f83852f..9d8a3786505 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import subprocess from setuptools import find_packages, setup @@ -36,7 +37,7 @@ "pandavro==1.5.*", "protobuf>=3.10", "PyYAML==5.1.*", - "fastavro==0.*", + "fastavro>=0.22.11,<0.23", "kafka-python==1.*", "tabulate==0.8.*", "toml==0.10.*", @@ -48,7 +49,13 @@ ] # README file from Feast repo root directory -README_FILE = os.path.join(os.path.dirname(__file__), "..", "..", "README.md") +repo_root = ( + subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE) + .communicate()[0] + .rstrip() + .decode("utf-8") +) +README_FILE = os.path.join(repo_root, "README.md") with open(os.path.join(README_FILE), "r") as f: LONG_DESCRIPTION = f.read() diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py new file mode 100644 index 00000000000..b564eeaa5b1 --- /dev/null +++ b/sdk/python/tests/conftest.py @@ -0,0 +1,22 @@ +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from sys import platform +import multiprocessing + + +def pytest_configure(config): + if platform in ["darwin", "windows"]: + multiprocessing.set_start_method("spawn") + else: + multiprocessing.set_start_method("fork") diff --git a/sdk/python/tests/data/localhost.crt b/sdk/python/tests/data/localhost.crt new file mode 100644 index 00000000000..1f471506aab --- /dev/null +++ b/sdk/python/tests/data/localhost.crt @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC5zCCAc+gAwIBAgIJAKzukpnyuwsVMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0yMDAyMTcxMTE4NDNaGA8zMDE5MDYyMDExMTg0M1ow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAqoanhiy4EUZjPA/m8IWk50OyTjKAnqZvEW5glqmTHP6lQbfyWQnzj3Ny +c++4Xn901FO2v07h+7lE3BScjgCX6klsLOHRnWcLX8lQygR6zzO+Oey1yXuCebBA +yhrsqgTDC/8zoCxe0W3t0vqvE4AJs3tJHq5Y1ba/X9OiKKsDZuMSSsbdd4qVEL6y +BD8PRNLT/iiD84Kq58GZtOI3fJls8E/bYbvksugcPI3kmlU4Plg3VrVplMl3DcMz +7BbvQP6jmVqdPtUT7+lL0C5CsNqbdDOIwg09+Gwus+A/g8PerBBd+ZCmdvSa9LYJ +OmlJszgZPIL9AagXLfuGQvNN2Y6WowIDAQABozowODAUBgNVHREEDTALgglsb2Nh +bGhvc3QwCwYDVR0PBAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3 +DQEBCwUAA4IBAQAuF1/VeQL73Y1FKrBX4bAb/Rdh2+Dadpi+w1pgEOi3P4udmQ+y +Xn9GwwLRQmHRLjyCT5KT8lNHdldPdlBamqPGGku449aCAjA/YHVHhcHaXl0MtPGq +BfKhHYSsvI2sIymlzZIvvIaf04yuJ1g+L0j8Px4Ecor9YwcKDZmpnIXLgdUtUrIQ +5Omrb4jImX6q8jp6Bjplb4H3o4TqKoa74NLOWUiH5/Rix3Lo8MRoEVbX2GhKk+8n +0eD3AuyrI1i+ce7zY8qGJKKFHGLDWPA/+006ZIS4j/Hr2FWo07CPFQ4/3gdJ8Erw +SzgO9vvIhQrBJn2CIH4+P5Cb1ktdobNWW9XK +-----END CERTIFICATE----- diff --git a/sdk/python/tests/data/localhost.key b/sdk/python/tests/data/localhost.key new file mode 100644 index 00000000000..dbd9cda062c --- /dev/null +++ b/sdk/python/tests/data/localhost.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCqhqeGLLgRRmM8 +D+bwhaTnQ7JOMoCepm8RbmCWqZMc/qVBt/JZCfOPc3Jz77hef3TUU7a/TuH7uUTc +FJyOAJfqSWws4dGdZwtfyVDKBHrPM7457LXJe4J5sEDKGuyqBMML/zOgLF7Rbe3S ++q8TgAmze0kerljVtr9f06IoqwNm4xJKxt13ipUQvrIEPw9E0tP+KIPzgqrnwZm0 +4jd8mWzwT9thu+Sy6Bw8jeSaVTg+WDdWtWmUyXcNwzPsFu9A/qOZWp0+1RPv6UvQ +LkKw2pt0M4jCDT34bC6z4D+Dw96sEF35kKZ29Jr0tgk6aUmzOBk8gv0BqBct+4ZC +803ZjpajAgMBAAECggEADE4FHphxe8WheX8IQgjSumFXJ29bepc14oMdcyGvXOM/ +F3vnf+dI7Ov+sUD2A9OcoYmc4TcW9WwL/Pl7xn9iduRvatmsn3gFCRdkvf8OwY7R +Riq/f1drNc6zDiJdO3N2g5IZrpAlE2WkSJoQMg8GJC5cO1uHS3yRWJ/Tzq1wZGcW +Dot9hAFgN0qNdP0xFkOsPM5ptC3DjLqsZWboJhIM19hgsIYaWQWHvcYlCcWTVhkj +FYzvLj5GrzAgyE89RpdXus670q5E2R2Rlnja21TfcxK0UOdIrKghZ0jxZMsXEwdB +8V7kIzL5kh//RhT/dIt0mHNMSdLFFx3yMTb2wTzpWQKBgQDRiCRslDSjiNSFySkn +6IivAwJtV2gLSxV05D9u9lrrlskHogrZUJkpVF1VzSnwv/ASaCZX4AGTtNPaz+vy +yDviwfjADsuum8jkzoxKCHnR1HVMyX+vm/g+pE20PMskTUuDE4zROtrqo9Ky0afv +94mJrf93Q815rsbEM5osugaeBQKBgQDQWAPTKy1wcG7edwfu3EaLYHPZ8pW9MldP +FvCLTMwSDkSzU+wA4BGE/5Tuu0WHSAfUc5C1LnMQXKBQXun+YCaBR6GZjUAmntz3 +poBIOYaxe651zqzCmo4ip1h5wIfPvynsyGmhsbpDSNhvXFgH2mF3XSY1nduKSRHu +389cHk3ahwKBgA4gAWSYcRv9I2aJcw7PrDcwGr/IPqlUPHQO1v/h96seFRtAnz6b +IlgY6dnY5NTn+4UiJEOUREbyz71Weu949CCLNvurg6uXsOlLy0VKYPv2OJoek08B +UrDWXq6h0of19fs2HC4Wq59Zv+ByJcIVi94OLsSZe4aSc6/SUrhlKgEJAoGBAIvR +5Y88NNx2uBEYdPx6W+WBr34e7Rrxw+JSFNCHk5SyeqyWr5XOyjMliv/EMl8dmhOc +Ewtkxte+MeB+Mi8CvBSay/rO7rR8fPK+jOzrnldSF7z8HLjlHGppQFlFOl/TfQFp +ZmqbadNp+caShImQp0SCAPiOnh1p+F0FWpYJyFnVAoGAKhSRP0iUmd+tId94px2m +G248BhcM9/0r+Y3yRX1eBx5eBzlzPUPcW1MSbhiZ1DIyLZ/MyObl98A1oNBGun11 +H/7Mq0E8BcJoXmt/6Z+2NhREBV9tDNuINyS/coYBV7H50pnSqyPpREPxNmu3Ukbm +u7ggLRfH+DexDysbpbCZ9l4= +-----END PRIVATE KEY----- diff --git a/sdk/python/tests/data/localhost.pem b/sdk/python/tests/data/localhost.pem new file mode 100644 index 00000000000..1f471506aab --- /dev/null +++ b/sdk/python/tests/data/localhost.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC5zCCAc+gAwIBAgIJAKzukpnyuwsVMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0yMDAyMTcxMTE4NDNaGA8zMDE5MDYyMDExMTg0M1ow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAqoanhiy4EUZjPA/m8IWk50OyTjKAnqZvEW5glqmTHP6lQbfyWQnzj3Ny +c++4Xn901FO2v07h+7lE3BScjgCX6klsLOHRnWcLX8lQygR6zzO+Oey1yXuCebBA +yhrsqgTDC/8zoCxe0W3t0vqvE4AJs3tJHq5Y1ba/X9OiKKsDZuMSSsbdd4qVEL6y +BD8PRNLT/iiD84Kq58GZtOI3fJls8E/bYbvksugcPI3kmlU4Plg3VrVplMl3DcMz +7BbvQP6jmVqdPtUT7+lL0C5CsNqbdDOIwg09+Gwus+A/g8PerBBd+ZCmdvSa9LYJ +OmlJszgZPIL9AagXLfuGQvNN2Y6WowIDAQABozowODAUBgNVHREEDTALgglsb2Nh +bGhvc3QwCwYDVR0PBAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3 +DQEBCwUAA4IBAQAuF1/VeQL73Y1FKrBX4bAb/Rdh2+Dadpi+w1pgEOi3P4udmQ+y +Xn9GwwLRQmHRLjyCT5KT8lNHdldPdlBamqPGGku449aCAjA/YHVHhcHaXl0MtPGq +BfKhHYSsvI2sIymlzZIvvIaf04yuJ1g+L0j8Px4Ecor9YwcKDZmpnIXLgdUtUrIQ +5Omrb4jImX6q8jp6Bjplb4H3o4TqKoa74NLOWUiH5/Rix3Lo8MRoEVbX2GhKk+8n +0eD3AuyrI1i+ce7zY8qGJKKFHGLDWPA/+006ZIS4j/Hr2FWo07CPFQ4/3gdJ8Erw +SzgO9vvIhQrBJn2CIH4+P5Cb1ktdobNWW9XK +-----END CERTIFICATE----- diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 123cbe47fd6..2724fff52e3 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -11,9 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import pkgutil from datetime import datetime import tempfile +from unittest import mock + import grpc import pandas as pd from google.protobuf.duration_pb2 import Duration @@ -63,10 +66,38 @@ CORE_URL = "core.feast.example.com" SERVING_URL = "serving.example.com" +_PRIVATE_KEY_RESOURCE_PATH = 'data/localhost.key' +_CERTIFICATE_CHAIN_RESOURCE_PATH = 'data/localhost.pem' +_ROOT_CERTIFICATE_RESOURCE_PATH = 'data/localhost.crt' class TestClient: - @pytest.fixture(scope="function") + + @pytest.fixture + def secure_mock_client(self, mocker): + client = Client(core_url=CORE_URL, serving_url=SERVING_URL, core_secure=True, serving_secure=True) + mocker.patch.object(client, "_connect_core") + mocker.patch.object(client, "_connect_serving") + client._core_url = CORE_URL + client._serving_url = SERVING_URL + return client + + @pytest.fixture + def mock_client(self, mocker): + client = Client(core_url=CORE_URL, serving_url=SERVING_URL) + mocker.patch.object(client, "_connect_core") + mocker.patch.object(client, "_connect_serving") + client._core_url = CORE_URL + client._serving_url = SERVING_URL + return client + + @pytest.fixture + def server_credentials(self): + private_key = pkgutil.get_data(__name__, _PRIVATE_KEY_RESOURCE_PATH) + certificate_chain = pkgutil.get_data(__name__, _CERTIFICATE_CHAIN_RESOURCE_PATH) + return grpc.ssl_server_credentials(((private_key, certificate_chain),)) + + @pytest.fixture def core_server(self): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) Core.add_CoreServiceServicer_to_server(CoreServicer(), server) @@ -75,7 +106,7 @@ def core_server(self): yield server server.stop(0) - @pytest.fixture(scope="function") + @pytest.fixture def serving_server(self): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) Serving.add_ServingServiceServicer_to_server(ServingServicer(), server) @@ -85,48 +116,73 @@ def serving_server(self): server.stop(0) @pytest.fixture - def mock_client(self, mocker): - client = Client(core_url=CORE_URL, serving_url=SERVING_URL) - mocker.patch.object(client, "_connect_core") - mocker.patch.object(client, "_connect_serving") - client._core_url = CORE_URL - client._serving_url = SERVING_URL - return client + def secure_core_server(self, server_credentials): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + Core.add_CoreServiceServicer_to_server(CoreServicer(), server) + server.add_secure_port("[::]:50053", server_credentials) + server.start() + yield server + server.stop(0) + + @pytest.fixture + def secure_serving_server(self, server_credentials): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + Serving.add_ServingServiceServicer_to_server(ServingServicer(), server) + + server.add_secure_port("[::]:50054", server_credentials) + server.start() + yield server + server.stop(0) + + @pytest.fixture + def secure_client(self, secure_core_server, secure_serving_server): + root_certificate_credentials = pkgutil.get_data(__name__, _ROOT_CERTIFICATE_RESOURCE_PATH) + # this is needed to establish a secure connection using self-signed certificates, for the purpose of the test + ssl_channel_credentials = grpc.ssl_channel_credentials(root_certificates=root_certificate_credentials) + with mock.patch("grpc.ssl_channel_credentials", MagicMock(return_value=ssl_channel_credentials)): + yield Client(core_url="localhost:50053", serving_url="localhost:50054", core_secure=True, + serving_secure=True) @pytest.fixture def client(self, core_server, serving_server): return Client(core_url="localhost:50051", serving_url="localhost:50052") - def test_version(self, mock_client, mocker): - mock_client._core_service_stub = Core.CoreServiceStub(grpc.insecure_channel("")) - mock_client._serving_service_stub = Serving.ServingServiceStub( + @pytest.mark.parametrize("mocked_client", [pytest.lazy_fixture("mock_client"), + pytest.lazy_fixture("secure_mock_client") + ]) + def test_version(self, mocked_client, mocker): + mocked_client._core_service_stub = Core.CoreServiceStub(grpc.insecure_channel("")) + mocked_client._serving_service_stub = Serving.ServingServiceStub( grpc.insecure_channel("") ) mocker.patch.object( - mock_client._core_service_stub, + mocked_client._core_service_stub, "GetFeastCoreVersion", return_value=GetFeastCoreVersionResponse(version="0.3.2"), ) mocker.patch.object( - mock_client._serving_service_stub, + mocked_client._serving_service_stub, "GetFeastServingInfo", return_value=GetFeastServingInfoResponse(version="0.3.2"), ) - status = mock_client.version() + status = mocked_client.version() assert ( - status["core"]["url"] == CORE_URL - and status["core"]["version"] == "0.3.2" - and status["serving"]["url"] == SERVING_URL - and status["serving"]["version"] == "0.3.2" + status["core"]["url"] == CORE_URL + and status["core"]["version"] == "0.3.2" + and status["serving"]["url"] == SERVING_URL + and status["serving"]["version"] == "0.3.2" ) - def test_get_online_features(self, mock_client, mocker): + @pytest.mark.parametrize("mocked_client", [pytest.lazy_fixture("mock_client"), + pytest.lazy_fixture("secure_mock_client") + ]) + def test_get_online_features(self, mocked_client, mocker): ROW_COUNT = 300 - mock_client._serving_service_stub = Serving.ServingServiceStub( + mocked_client._serving_service_stub = Serving.ServingServiceStub( grpc.insecure_channel("") ) @@ -148,12 +204,12 @@ def test_get_online_features(self, mock_client, mocker): ) mocker.patch.object( - mock_client._serving_service_stub, + mocked_client._serving_service_stub, "GetOnlineFeatures", return_value=response, ) - response = mock_client.get_online_features( + response = mocked_client.get_online_features( entity_rows=entity_rows, feature_refs=[ "my_project/feature_1:1", @@ -169,17 +225,20 @@ def test_get_online_features(self, mock_client, mocker): ) # type: GetOnlineFeaturesResponse assert ( - response.field_values[0].fields["my_project/feature_1:1"].int64_val == 1 - and response.field_values[0].fields["my_project/feature_9:1"].int64_val == 9 + response.field_values[0].fields["my_project/feature_1:1"].int64_val == 1 + and response.field_values[0].fields["my_project/feature_9:1"].int64_val == 9 ) - def test_get_feature_set(self, mock_client, mocker): - mock_client._core_service_stub = Core.CoreServiceStub(grpc.insecure_channel("")) + @pytest.mark.parametrize("mocked_client", [pytest.lazy_fixture("mock_client"), + pytest.lazy_fixture("secure_mock_client") + ]) + def test_get_feature_set(self, mocked_client, mocker): + mocked_client._core_service_stub = Core.CoreServiceStub(grpc.insecure_channel("")) from google.protobuf.duration_pb2 import Duration mocker.patch.object( - mock_client._core_service_stub, + mocked_client._core_service_stub, "GetFeatureSet", return_value=GetFeatureSetResponse( feature_set=FeatureSetProto( @@ -214,29 +273,32 @@ def test_get_feature_set(self, mock_client, mocker): ) ), ) - mock_client.set_project("my_project") - feature_set = mock_client.get_feature_set("my_feature_set", version=2) + mocked_client.set_project("my_project") + feature_set = mocked_client.get_feature_set("my_feature_set", version=2) assert ( - feature_set.name == "my_feature_set" - and feature_set.version == 2 - and feature_set.fields["my_feature_1"].name == "my_feature_1" - and feature_set.fields["my_feature_1"].dtype == ValueType.FLOAT - and feature_set.fields["my_entity_1"].name == "my_entity_1" - and feature_set.fields["my_entity_1"].dtype == ValueType.INT64 - and len(feature_set.features) == 2 - and len(feature_set.entities) == 1 + feature_set.name == "my_feature_set" + and feature_set.version == 2 + and feature_set.fields["my_feature_1"].name == "my_feature_1" + and feature_set.fields["my_feature_1"].dtype == ValueType.FLOAT + and feature_set.fields["my_entity_1"].name == "my_entity_1" + and feature_set.fields["my_entity_1"].dtype == ValueType.INT64 + and len(feature_set.features) == 2 + and len(feature_set.entities) == 1 ) - def test_get_batch_features(self, mock_client, mocker): + @pytest.mark.parametrize("mocked_client", [pytest.lazy_fixture("mock_client"), + pytest.lazy_fixture("secure_mock_client") + ]) + def test_get_batch_features(self, mocked_client, mocker): - mock_client._serving_service_stub = Serving.ServingServiceStub( + mocked_client._serving_service_stub = Serving.ServingServiceStub( grpc.insecure_channel("") ) - mock_client._core_service_stub = Core.CoreServiceStub(grpc.insecure_channel("")) + mocked_client._core_service_stub = Core.CoreServiceStub(grpc.insecure_channel("")) mocker.patch.object( - mock_client._core_service_stub, + mocked_client._core_service_stub, "GetFeatureSet", return_value=GetFeatureSetResponse( feature_set=FeatureSetProto( @@ -283,7 +345,7 @@ def test_get_batch_features(self, mock_client, mocker): to_avro(file_path_or_buffer=final_results, df=expected_dataframe) mocker.patch.object( - mock_client._serving_service_stub, + mocked_client._serving_service_stub, "GetBatchFeatures", return_value=GetBatchFeaturesResponse( job=BatchFeaturesJob( @@ -297,7 +359,7 @@ def test_get_batch_features(self, mock_client, mocker): ) mocker.patch.object( - mock_client._serving_service_stub, + mocked_client._serving_service_stub, "GetJob", return_value=GetJobResponse( job=BatchFeaturesJob( @@ -311,7 +373,7 @@ def test_get_batch_features(self, mock_client, mocker): ) mocker.patch.object( - mock_client._serving_service_stub, + mocked_client._serving_service_stub, "GetFeastServingInfo", return_value=GetFeastServingInfoResponse( job_staging_location=f"file://{tempfile.mkdtemp()}/", @@ -319,8 +381,8 @@ def test_get_batch_features(self, mock_client, mocker): ), ) - mock_client.set_project("project1") - response = mock_client.get_batch_features( + mocked_client.set_project("project1") + response = mocked_client.get_batch_features( entity_rows=pd.DataFrame( { "datetime": [ @@ -348,9 +410,12 @@ def test_get_batch_features(self, mock_client, mocker): ] ) - def test_apply_feature_set_success(self, client): + @pytest.mark.parametrize("test_client", [pytest.lazy_fixture("client"), + pytest.lazy_fixture("secure_client") + ]) + def test_apply_feature_set_success(self, test_client): - client.set_project("project1") + test_client.set_project("project1") # Create Feature Sets fs1 = FeatureSet("my-feature-set-1") @@ -364,23 +429,24 @@ def test_apply_feature_set_success(self, client): fs2.add(Entity(name="fs2-my-entity-1", dtype=ValueType.INT64)) # Register Feature Set with Core - client.apply(fs1) - client.apply(fs2) + test_client.apply(fs1) + test_client.apply(fs2) - feature_sets = client.list_feature_sets() + feature_sets = test_client.list_feature_sets() # List Feature Sets assert ( - len(feature_sets) == 2 - and feature_sets[0].name == "my-feature-set-1" - and feature_sets[0].features[0].name == "fs1-my-feature-1" - and feature_sets[0].features[0].dtype == ValueType.INT64 - and feature_sets[1].features[1].dtype == ValueType.BYTES_LIST + len(feature_sets) == 2 + and feature_sets[0].name == "my-feature-set-1" + and feature_sets[0].features[0].name == "fs1-my-feature-1" + and feature_sets[0].features[0].dtype == ValueType.INT64 + and feature_sets[1].features[1].dtype == ValueType.BYTES_LIST ) - @pytest.mark.parametrize("dataframe", [dataframes.GOOD]) - def test_feature_set_ingest_success(self, dataframe, client, mocker): - client.set_project("project1") + @pytest.mark.parametrize("dataframe,test_client", [(dataframes.GOOD, pytest.lazy_fixture("client")), + (dataframes.GOOD, pytest.lazy_fixture("secure_client"))]) + def test_feature_set_ingest_success(self, dataframe, test_client, mocker): + test_client.set_project("project1") driver_fs = FeatureSet( "driver-feature-set", source=KafkaSource(brokers="kafka:9092", topic="test") ) @@ -390,12 +456,12 @@ def test_feature_set_ingest_success(self, dataframe, client, mocker): driver_fs.add(Entity(name="entity_id", dtype=ValueType.INT64)) # Register with Feast core - client.apply(driver_fs) + test_client.apply(driver_fs) driver_fs = driver_fs.to_proto() driver_fs.meta.status = FeatureSetStatusProto.STATUS_READY mocker.patch.object( - client._core_service_stub, + test_client._core_service_stub, "GetFeatureSet", return_value=GetFeatureSetResponse(feature_set=driver_fs), ) @@ -403,14 +469,16 @@ def test_feature_set_ingest_success(self, dataframe, client, mocker): # Need to create a mock producer with patch("feast.client.get_producer") as mocked_queue: # Ingest data into Feast - client.ingest("driver-feature-set", dataframe) + test_client.ingest("driver-feature-set", dataframe) - @pytest.mark.parametrize("dataframe,exception", [(dataframes.GOOD, TimeoutError)]) + @pytest.mark.parametrize("dataframe,exception,test_client", + [(dataframes.GOOD, TimeoutError, pytest.lazy_fixture("client")), + (dataframes.GOOD, TimeoutError, pytest.lazy_fixture("secure_client"))]) def test_feature_set_ingest_fail_if_pending( - self, dataframe, exception, client, mocker + self, dataframe, exception, test_client, mocker ): with pytest.raises(exception): - client.set_project("project1") + test_client.set_project("project1") driver_fs = FeatureSet( "driver-feature-set", source=KafkaSource(brokers="kafka:9092", topic="test"), @@ -421,12 +489,12 @@ def test_feature_set_ingest_fail_if_pending( driver_fs.add(Entity(name="entity_id", dtype=ValueType.INT64)) # Register with Feast core - client.apply(driver_fs) + test_client.apply(driver_fs) driver_fs = driver_fs.to_proto() driver_fs.meta.status = FeatureSetStatusProto.STATUS_PENDING mocker.patch.object( - client._core_service_stub, + test_client._core_service_stub, "GetFeatureSet", return_value=GetFeatureSetResponse(feature_set=driver_fs), ) @@ -434,18 +502,22 @@ def test_feature_set_ingest_fail_if_pending( # Need to create a mock producer with patch("feast.client.get_producer") as mocked_queue: # Ingest data into Feast - client.ingest("driver-feature-set", dataframe, timeout=1) + test_client.ingest("driver-feature-set", dataframe, timeout=1) @pytest.mark.parametrize( - "dataframe,exception", + "dataframe,exception,test_client", [ - (dataframes.BAD_NO_DATETIME, Exception), - (dataframes.BAD_INCORRECT_DATETIME_TYPE, Exception), - (dataframes.BAD_NO_ENTITY, Exception), - (dataframes.NO_FEATURES, Exception), + (dataframes.BAD_NO_DATETIME, Exception, pytest.lazy_fixture("client")), + (dataframes.BAD_INCORRECT_DATETIME_TYPE, Exception, pytest.lazy_fixture("client")), + (dataframes.BAD_NO_ENTITY, Exception, pytest.lazy_fixture("client")), + (dataframes.NO_FEATURES, Exception, pytest.lazy_fixture("client")), + (dataframes.BAD_NO_DATETIME, Exception, pytest.lazy_fixture("secure_client")), + (dataframes.BAD_INCORRECT_DATETIME_TYPE, Exception, pytest.lazy_fixture("secure_client")), + (dataframes.BAD_NO_ENTITY, Exception, pytest.lazy_fixture("secure_client")), + (dataframes.NO_FEATURES, Exception, pytest.lazy_fixture("secure_client")), ], ) - def test_feature_set_ingest_failure(self, client, dataframe, exception): + def test_feature_set_ingest_failure(self, test_client, dataframe, exception): with pytest.raises(exception): # Create feature set driver_fs = FeatureSet("driver-feature-set") @@ -454,15 +526,16 @@ def test_feature_set_ingest_failure(self, client, dataframe, exception): driver_fs.infer_fields_from_df(dataframe) # Register with Feast core - client.apply(driver_fs) + test_client.apply(driver_fs) # Ingest data into Feast - client.ingest(driver_fs, dataframe=dataframe) + test_client.ingest(driver_fs, dataframe=dataframe) - @pytest.mark.parametrize("dataframe", [dataframes.ALL_TYPES]) - def test_feature_set_types_success(self, client, dataframe, mocker): + @pytest.mark.parametrize("dataframe,test_client", [(dataframes.ALL_TYPES, pytest.lazy_fixture("client")), + (dataframes.ALL_TYPES, pytest.lazy_fixture("secure_client"))]) + def test_feature_set_types_success(self, test_client, dataframe, mocker): - client.set_project("project1") + test_client.set_project("project1") all_types_fs = FeatureSet( name="all_types", @@ -489,10 +562,10 @@ def test_feature_set_types_success(self, client, dataframe, mocker): ) # Register with Feast core - client.apply(all_types_fs) + test_client.apply(all_types_fs) mocker.patch.object( - client._core_service_stub, + test_client._core_service_stub, "GetFeatureSet", return_value=GetFeatureSetResponse(feature_set=all_types_fs.to_proto()), ) @@ -500,4 +573,38 @@ def test_feature_set_types_success(self, client, dataframe, mocker): # Need to create a mock producer with patch("feast.client.get_producer") as mocked_queue: # Ingest data into Feast - client.ingest(all_types_fs, dataframe) + test_client.ingest(all_types_fs, dataframe) + + @patch("grpc.channel_ready_future") + def test_secure_channel_creation_with_secure_client(self, _mocked_obj): + client = Client(core_url="localhost:50051", serving_url="localhost:50052", serving_secure=True, + core_secure=True) + with mock.patch("grpc.secure_channel") as _grpc_mock, \ + mock.patch("grpc.ssl_channel_credentials", MagicMock(return_value="test")) as _mocked_credentials: + client._connect_serving() + _grpc_mock.assert_called_with(client.serving_url, _mocked_credentials.return_value) + + @mock.patch("grpc.channel_ready_future") + def test_secure_channel_creation_with_secure_serving_url(self, _mocked_obj, ): + client = Client(core_url="localhost:50051", serving_url="localhost:443") + with mock.patch("grpc.secure_channel") as _grpc_mock, \ + mock.patch("grpc.ssl_channel_credentials", MagicMock(return_value="test")) as _mocked_credentials: + client._connect_serving() + _grpc_mock.assert_called_with(client.serving_url, _mocked_credentials.return_value) + + @patch("grpc.channel_ready_future") + def test_secure_channel_creation_with_secure_client(self, _mocked_obj): + client = Client(core_url="localhost:50053", serving_url="localhost:50054", serving_secure=True, + core_secure=True) + with mock.patch("grpc.secure_channel") as _grpc_mock, \ + mock.patch("grpc.ssl_channel_credentials", MagicMock(return_value="test")) as _mocked_credentials: + client._connect_core() + _grpc_mock.assert_called_with(client.core_url, _mocked_credentials.return_value) + + @patch("grpc.channel_ready_future") + def test_secure_channel_creation_with_secure_core_url(self, _mocked_obj): + client = Client(core_url="localhost:443", serving_url="localhost:50054") + with mock.patch("grpc.secure_channel") as _grpc_mock, \ + mock.patch("grpc.ssl_channel_credentials", MagicMock(return_value="test")) as _mocked_credentials: + client._connect_core() + _grpc_mock.assert_called_with(client.core_url, _mocked_credentials.return_value) \ No newline at end of file diff --git a/serving/pom.xml b/serving/pom.xml index dc3391df62f..baf41eede8a 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -47,10 +47,6 @@ false - - org.xolstice.maven.plugins - protobuf-maven-plugin - org.apache.maven.plugins maven-failsafe-plugin @@ -74,6 +70,12 @@ + + dev.feast + datatypes-java + ${project.version} + + org.slf4j @@ -96,6 +98,10 @@ org.springframework.boot spring-boot-starter-log4j2 + + org.apache.logging.log4j + log4j-web + org.springframework.boot @@ -136,12 +142,27 @@ 3.1.0 - - redis.clients - jedis + io.lettuce + lettuce-core + + + com.datastax.oss + java-driver-core + 4.5.0 - + + com.datastax.oss + java-driver-query-builder + 4.5.0 + + + + com.datastax.oss + java-driver-mapper-runtime + 4.5.0 + + com.google.guava guava @@ -227,6 +248,13 @@ spring-boot-starter-test test + + + org.cassandraunit + cassandra-unit-shaded + 3.11.2.0 + test + @@ -248,6 +276,12 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml + + + com.github.kstyrc + embedded-redis + test + diff --git a/serving/sample_cassandra_config.yml b/serving/sample_cassandra_config.yml new file mode 100644 index 00000000000..ca3d4cbbcca --- /dev/null +++ b/serving/sample_cassandra_config.yml @@ -0,0 +1,13 @@ +name: serving +type: CASSANDRA +cassandra_config: + bootstrap_hosts: localhost + port: 9042 + keyspace: feast + table_name: feature_store + replication_options: + class: SimpleStrategy + replication_factor: 1 +subscriptions: + - name: "*" + version: ">0" diff --git a/serving/src/main/java/feast/serving/FeastProperties.java b/serving/src/main/java/feast/serving/FeastProperties.java index e511835b0aa..718ca78669c 100644 --- a/serving/src/main/java/feast/serving/FeastProperties.java +++ b/serving/src/main/java/feast/serving/FeastProperties.java @@ -85,6 +85,17 @@ public static class StoreProperties { private String configPath; private int redisPoolMaxSize; private int redisPoolMaxIdle; + private String cassandraDcName; + private int cassandraDcReplicas; + private int cassandraPoolCoreLocalConnections; + private int cassandraPoolMaxLocalConnections; + private int cassandraPoolCoreRemoteConnections; + private int cassandraPoolMaxRemoteConnections; + private int cassandraPoolMaxRequestsLocalConnection; + private int cassandraPoolMaxRequestsRemoteConnection; + private int cassandraPoolNewLocalConnectionThreshold; + private int cassandraPoolNewRemoteConnectionThreshold; + private int cassandraPoolTimeoutMillis; public String getConfigPath() { return this.configPath; @@ -98,6 +109,50 @@ public int getRedisPoolMaxIdle() { return this.redisPoolMaxIdle; } + public String getCassandraDcName() { + return this.cassandraDcName; + } + + public int getCassandraDcReplicas() { + return this.cassandraDcReplicas; + } + + public int getCassandraPoolCoreLocalConnections() { + return this.cassandraPoolCoreLocalConnections; + } + + public int getCassandraPoolMaxLocalConnections() { + return this.cassandraPoolMaxLocalConnections; + } + + public int getCassandraPoolCoreRemoteConnections() { + return this.cassandraPoolCoreRemoteConnections; + } + + public int getCassandraPoolMaxRemoteConnections() { + return this.cassandraPoolMaxRemoteConnections; + } + + public int getCassandraPoolMaxRequestsLocalConnection() { + return this.cassandraPoolMaxRequestsLocalConnection; + } + + public int getCassandraPoolMaxRequestsRemoteConnection() { + return this.cassandraPoolMaxRequestsRemoteConnection; + } + + public int getCassandraPoolNewLocalConnectionThreshold() { + return this.cassandraPoolNewLocalConnectionThreshold; + } + + public int getCassandraPoolNewRemoteConnectionThreshold() { + return this.cassandraPoolNewRemoteConnectionThreshold; + } + + public int getCassandraPoolTimeoutMillis() { + return this.cassandraPoolTimeoutMillis; + } + public void setConfigPath(String configPath) { this.configPath = configPath; } @@ -109,10 +164,61 @@ public void setRedisPoolMaxSize(int redisPoolMaxSize) { public void setRedisPoolMaxIdle(int redisPoolMaxIdle) { this.redisPoolMaxIdle = redisPoolMaxIdle; } + + public void setCassandraDcName(String cassandraDcName) { + this.cassandraDcName = cassandraDcName; + } + + public void setCassandraDcReplicas(int cassandraDcReplicas) { + this.cassandraDcReplicas = cassandraDcReplicas; + } + + public void setCassandraPoolCoreLocalConnections(int cassandraPoolCoreLocalConnections) { + this.cassandraPoolCoreLocalConnections = cassandraPoolCoreLocalConnections; + } + + public void setCassandraPoolMaxLocalConnections(int cassandraPoolMaxLocalConnections) { + this.cassandraPoolMaxLocalConnections = cassandraPoolMaxLocalConnections; + } + + public void setCassandraPoolCoreRemoteConnections(int cassandraPoolCoreRemoteConnections) { + this.cassandraPoolCoreRemoteConnections = cassandraPoolCoreRemoteConnections; + } + + public void setCassandraPoolMaxRemoteConnections(int cassandraPoolMaxRemoteConnections) { + this.cassandraPoolMaxRemoteConnections = cassandraPoolMaxRemoteConnections; + } + + public void setCassandraPoolMaxRequestsLocalConnection( + int cassandraPoolMaxRequestsLocalConnection) { + this.cassandraPoolMaxRequestsLocalConnection = cassandraPoolMaxRequestsLocalConnection; + } + + public void setCassandraPoolMaxRequestsRemoteConnection( + int cassandraPoolMaxRequestsRemoteConnection) { + this.cassandraPoolMaxRequestsRemoteConnection = cassandraPoolMaxRequestsRemoteConnection; + } + + public void setCassandraPoolNewLocalConnectionThreshold( + int cassandraPoolNewLocalConnectionThreshold) { + this.cassandraPoolNewLocalConnectionThreshold = cassandraPoolNewLocalConnectionThreshold; + } + + public void setCassandraPoolNewRemoteConnectionThreshold( + int cassandraPoolNewRemoteConnectionThreshold) { + this.cassandraPoolNewRemoteConnectionThreshold = cassandraPoolNewRemoteConnectionThreshold; + } + + public void setCassandraPoolTimeoutMillis(int cassandraPoolTimeoutMillis) { + this.cassandraPoolTimeoutMillis = cassandraPoolTimeoutMillis; + } } public static class JobProperties { private String stagingLocation; + private String stagingProject; + private int bigqueryInitialRetryDelaySecs; + private int bigqueryTotalTimeoutSecs; private String storeType; private Map storeOptions; @@ -120,6 +226,18 @@ public String getStagingLocation() { return this.stagingLocation; } + public String getStagingProject() { + return this.stagingProject; + } + + public int getBigqueryInitialRetryDelaySecs() { + return bigqueryInitialRetryDelaySecs; + } + + public int getBigqueryTotalTimeoutSecs() { + return bigqueryTotalTimeoutSecs; + } + public String getStoreType() { return this.storeType; } @@ -132,6 +250,18 @@ public void setStagingLocation(String stagingLocation) { this.stagingLocation = stagingLocation; } + public void setStagingProject(String stagingProject) { + this.stagingProject = stagingProject; + } + + public void setBigqueryInitialRetryDelaySecs(int bigqueryInitialRetryDelaySecs) { + this.bigqueryInitialRetryDelaySecs = bigqueryInitialRetryDelaySecs; + } + + public void setBigqueryTotalTimeoutSecs(int bigqueryTotalTimeoutSecs) { + this.bigqueryTotalTimeoutSecs = bigqueryTotalTimeoutSecs; + } + public void setStoreType(String storeType) { this.storeType = storeType; } diff --git a/serving/src/main/java/feast/serving/configuration/JobServiceConfig.java b/serving/src/main/java/feast/serving/configuration/JobServiceConfig.java index 2afbdaf90d1..179d9aadfd9 100644 --- a/serving/src/main/java/feast/serving/configuration/JobServiceConfig.java +++ b/serving/src/main/java/feast/serving/configuration/JobServiceConfig.java @@ -16,40 +16,112 @@ */ package feast.serving.configuration; -import feast.core.StoreProto.Store; -import feast.core.StoreProto.Store.RedisConfig; +import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.*; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; +import com.datastax.oss.driver.api.core.type.DataTypes; +import com.datastax.oss.driver.api.querybuilder.SchemaBuilder; +import com.datastax.oss.driver.api.querybuilder.schema.*; +import feast.core.StoreProto; import feast.core.StoreProto.Store.StoreType; +import feast.serving.FeastProperties; +import feast.serving.service.CassandraBackedJobService; import feast.serving.service.JobService; import feast.serving.service.NoopJobService; import feast.serving.service.RedisBackedJobService; import feast.serving.specs.CachedSpecService; +import java.net.InetSocketAddress; +import java.util.*; +import java.util.stream.Collectors; +import org.slf4j.Logger; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import redis.clients.jedis.Jedis; @Configuration public class JobServiceConfig { + private static final Logger log = org.slf4j.LoggerFactory.getLogger(JobServiceConfig.class); @Bean - public JobService jobService(Store jobStore, CachedSpecService specService) { + public JobService jobService( + FeastProperties feastProperties, + CachedSpecService specService, + StoreConfiguration storeConfiguration) { if (!specService.getStore().getType().equals(StoreType.BIGQUERY)) { return new NoopJobService(); } - - switch (jobStore.getType()) { + StoreType storeType = StoreType.valueOf(feastProperties.getJobs().getStoreType()); + switch (storeType) { case REDIS: - RedisConfig redisConfig = jobStore.getRedisConfig(); - Jedis jedis = new Jedis(redisConfig.getHost(), redisConfig.getPort()); - return new RedisBackedJobService(jedis); + return new RedisBackedJobService(storeConfiguration.getJobStoreRedisConnection()); case INVALID: case BIGQUERY: case CASSANDRA: + StoreProto.Store store = specService.getStore(); + StoreProto.Store.CassandraConfig cassandraConfig = store.getCassandraConfig(); + Map storeOptions = feastProperties.getJobs().getStoreOptions(); + List contactPoints = + Arrays.stream(storeOptions.get("bootstrapHosts").split(",")) + .map(h -> new InetSocketAddress(h, Integer.parseInt(storeOptions.get("port")))) + .collect(Collectors.toList()); + CqlSession cluster = CqlSession.builder().addContactPoints(contactPoints).build(); + // Session in Cassandra is thread-safe and maintains connections to cluster nodes internally + // Recommended to use one session per keyspace instead of open and close connection for each + // request + log.info(String.format("Cluster name: %s", cluster.getName())); + log.info(String.format("Cluster keyspace: %s", cluster.getMetadata().getKeyspaces())); + log.info(String.format("Cluster nodes: %s", cluster.getMetadata().getNodes())); + log.info(String.format("Cluster tokenmap: %s", cluster.getMetadata().getTokenMap())); + log.info( + String.format( + "Cluster default profile: %s", + cluster.getContext().getConfig().getDefaultProfile().toString())); + log.info( + String.format( + "Cluster lb policies: %s", cluster.getContext().getLoadBalancingPolicies())); + // Session in Cassandra is thread-safe and maintains connections to cluster nodes internally + // Recommended to use one session per keyspace instead of open and close connection for each + // request + try { + String keyspace = storeOptions.get("keyspace"); + Optional keyspaceMetadata = cluster.getMetadata().getKeyspace(keyspace); + if (keyspaceMetadata.isEmpty()) { + log.info("Creating keyspace '{}'", keyspace); + Map replicationOptions = new HashMap<>(); + replicationOptions.put( + storeOptions.get("replicationOptionsDc"), + Integer.parseInt(storeOptions.get("replicationOptionsReplicas"))); + CreateKeyspace createKeyspace = + SchemaBuilder.createKeyspace(keyspace) + .ifNotExists() + .withNetworkTopologyStrategy(replicationOptions); + createKeyspace.withDurableWrites(true); + cluster.execute(createKeyspace.asCql()); + } + CreateTable createTable = + SchemaBuilder.createTable(keyspace, storeOptions.get("tableName")) + .ifNotExists() + .withPartitionKey("job_uuid", DataTypes.TEXT) + .withClusteringColumn("timestamp", DataTypes.TIMESTAMP) + .withColumn("job_data", DataTypes.TEXT); + log.info("Create Cassandra table if not exists.."); + cluster.execute(createTable.asCql()); + + } catch (RuntimeException e) { + throw new RuntimeException( + String.format( + "Failed to connect to Cassandra at bootstrap hosts: '%s' port: '%s'. Please check that your Cassandra is running and accessible from Feast.", + contactPoints.stream() + .map(InetSocketAddress::getHostName) + .collect(Collectors.joining(",")), + cassandraConfig.getPort()), + e); + } + return new CassandraBackedJobService(cluster); case UNRECOGNIZED: default: throw new IllegalArgumentException( - String.format( - "Unsupported store type '%s' for job store name '%s'", - jobStore.getType(), jobStore.getName())); + String.format("Unsupported store type '%s' for job store", storeType)); } } } diff --git a/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java b/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java index 9380ded4c4e..82b634d4584 100644 --- a/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java +++ b/serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java @@ -16,49 +16,46 @@ */ package feast.serving.configuration; +import com.datastax.oss.driver.api.core.ConsistencyLevel; +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.*; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import feast.core.StoreProto.Store; import feast.core.StoreProto.Store.BigQueryConfig; -import feast.core.StoreProto.Store.Builder; +import feast.core.StoreProto.Store.CassandraConfig; import feast.core.StoreProto.Store.RedisConfig; -import feast.core.StoreProto.Store.StoreType; import feast.core.StoreProto.Store.Subscription; import feast.serving.FeastProperties; -import feast.serving.FeastProperties.JobProperties; +import feast.serving.FeastProperties.StoreProperties; import feast.serving.service.BigQueryServingService; +import feast.serving.service.CassandraServingService; import feast.serving.service.JobService; import feast.serving.service.NoopJobService; import feast.serving.service.RedisServingService; import feast.serving.service.ServingService; import feast.serving.specs.CachedSpecService; import io.opentracing.Tracer; +import java.io.File; +import java.io.FileInputStream; +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; @Configuration public class ServingServiceConfig { private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfig.class); - @Bean(name = "JobStore") - public Store jobStoreDefinition(FeastProperties feastProperties) { - JobProperties jobProperties = feastProperties.getJobs(); - if (feastProperties.getJobs().getStoreType().equals("")) { - return Store.newBuilder().build(); - } - Map options = jobProperties.getStoreOptions(); - Builder storeDefinitionBuilder = - Store.newBuilder().setType(StoreType.valueOf(jobProperties.getStoreType())); - return setStoreConfig(storeDefinitionBuilder, options); - } - private Store setStoreConfig(Store.Builder builder, Map options) { switch (builder.getType()) { case REDIS: @@ -76,6 +73,13 @@ private Store setStoreConfig(Store.Builder builder, Map options) .build(); return builder.setBigqueryConfig(bqConfig).build(); case CASSANDRA: + CassandraConfig cassandraConfig = + CassandraConfig.newBuilder() + .setBootstrapHosts(options.get("host")) + .setPort(Integer.parseInt(options.get("port"))) + .setKeyspace(options.get("keyspace")) + .build(); + return builder.setCassandraConfig(cassandraConfig).build(); default: throw new IllegalArgumentException( String.format( @@ -89,23 +93,34 @@ public ServingService servingService( FeastProperties feastProperties, CachedSpecService specService, JobService jobService, - Tracer tracer) { + Tracer tracer, + StoreConfiguration storeConfiguration) { ServingService servingService = null; Store store = specService.getStore(); switch (store.getType()) { case REDIS: - RedisConfig redisConfig = store.getRedisConfig(); - JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(feastProperties.getStore().getRedisPoolMaxSize()); - poolConfig.setMaxIdle(feastProperties.getStore().getRedisPoolMaxIdle()); - JedisPool jedisPool = - new JedisPool(poolConfig, redisConfig.getHost(), redisConfig.getPort()); - servingService = new RedisServingService(jedisPool, specService, tracer); + servingService = + new RedisServingService( + storeConfiguration.getServingRedisConnection(), specService, tracer); break; case BIGQUERY: BigQueryConfig bqConfig = store.getBigqueryConfig(); - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + GoogleCredentials credentials; + File credentialsPath = + new File( + "/etc/gcloud/service-accounts/credentials.json"); // TODO: update to your key path. + try (FileInputStream serviceAccountStream = new FileInputStream(credentialsPath)) { + credentials = ServiceAccountCredentials.fromStream(serviceAccountStream); + } catch (Exception e) { + throw new IllegalStateException("No credentials file found", e); + } + BigQuery bigquery = + BigQueryOptions.newBuilder() + .setCredentials(credentials) + .setProjectId(feastProperties.getJobs().getStagingProject()) + .build() + .getService(); Storage storage = StorageOptions.getDefaultInstance().getService(); String jobStagingLocation = feastProperties.getJobs().getStagingLocation(); if (!jobStagingLocation.contains("://")) { @@ -132,9 +147,55 @@ public ServingService servingService( specService, jobService, jobStagingLocation, + feastProperties.getJobs().getBigqueryInitialRetryDelaySecs(), + feastProperties.getJobs().getBigqueryTotalTimeoutSecs(), storage); break; case CASSANDRA: + StoreProperties storeProperties = feastProperties.getStore(); + CassandraConfig cassandraConfig = store.getCassandraConfig(); + List contactPoints = + Arrays.stream(cassandraConfig.getBootstrapHosts().split(",")) + .map(h -> new InetSocketAddress(h, cassandraConfig.getPort())) + .collect(Collectors.toList()); + CqlSession cluster = + CqlSession.builder() + .addContactPoints(contactPoints) + .withLocalDatacenter(storeProperties.getCassandraDcName()) + .build(); + // Session in Cassandra is thread-safe and maintains connections to cluster nodes internally + // Recommended to use one session per keyspace instead of open and close connection for each + // request + log.info(String.format("Cluster name: %s", cluster.getName())); + log.info(String.format("Cluster keyspace: %s", cluster.getMetadata().getKeyspaces())); + log.info(String.format("Cluster nodes: %s", cluster.getMetadata().getNodes())); + log.info(String.format("Cluster tokenmap: %s", cluster.getMetadata().getTokenMap())); + log.info( + String.format( + "Cluster default profile: %s", + cluster.getContext().getConfig().getDefaultProfile().toString())); + log.info( + String.format( + "Cluster lb policies: %s", cluster.getContext().getLoadBalancingPolicies())); + ConsistencyLevel cl; + log.info(String.format("Consistency level: %s", cassandraConfig.getConsistency())); + if (cassandraConfig.getConsistency().equals("ONE")) { + log.info("Serving with read consistency ONE"); + cl = ConsistencyLevel.ONE; + } else { + log.info("Serving with read consistency TWO"); + cl = ConsistencyLevel.TWO; + } + servingService = + new CassandraServingService( + cluster, + cassandraConfig.getKeyspace(), + cassandraConfig.getTableName(), + cassandraConfig.getVersionless(), + cl, + specService, + tracer); + break; case UNRECOGNIZED: case INVALID: throw new IllegalArgumentException( diff --git a/serving/src/main/java/feast/serving/configuration/SpecServiceConfig.java b/serving/src/main/java/feast/serving/configuration/SpecServiceConfig.java index 3c91c2765aa..26ebfa956ca 100644 --- a/serving/src/main/java/feast/serving/configuration/SpecServiceConfig.java +++ b/serving/src/main/java/feast/serving/configuration/SpecServiceConfig.java @@ -35,7 +35,7 @@ public class SpecServiceConfig { private static final Logger log = org.slf4j.LoggerFactory.getLogger(SpecServiceConfig.class); private String feastCoreHost; private int feastCorePort; - private static final int CACHE_REFRESH_RATE_MINUTES = 1; + private static final int CACHE_REFRESH_RATE_SECONDS = 10; @Autowired public SpecServiceConfig(FeastProperties feastProperties) { @@ -51,15 +51,14 @@ public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( // reload all specs including new ones periodically scheduledExecutorService.scheduleAtFixedRate( cachedSpecStorage::scheduledPopulateCache, - CACHE_REFRESH_RATE_MINUTES, - CACHE_REFRESH_RATE_MINUTES, - TimeUnit.MINUTES); + CACHE_REFRESH_RATE_SECONDS, + CACHE_REFRESH_RATE_SECONDS, + TimeUnit.SECONDS); return scheduledExecutorService; } @Bean public CachedSpecService specService(FeastProperties feastProperties) { - CoreSpecService coreService = new CoreSpecService(feastCoreHost, feastCorePort); Path path = Paths.get(feastProperties.getStore().getConfigPath()); CachedSpecService cachedSpecStorage = new CachedSpecService(coreService, path); diff --git a/serving/src/main/java/feast/serving/configuration/StoreConfiguration.java b/serving/src/main/java/feast/serving/configuration/StoreConfiguration.java new file mode 100644 index 00000000000..84dc7b7f8d4 --- /dev/null +++ b/serving/src/main/java/feast/serving/configuration/StoreConfiguration.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.configuration; + +import io.lettuce.core.api.StatefulRedisConnection; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class StoreConfiguration { + + // We can define other store specific beans here + // These beans can be autowired or can be created in this class. + private final StatefulRedisConnection servingRedisConnection; + private final StatefulRedisConnection jobStoreRedisConnection; + + @Autowired + public StoreConfiguration( + ObjectProvider> servingRedisConnection, + ObjectProvider> jobStoreRedisConnection) { + this.servingRedisConnection = servingRedisConnection.getIfAvailable(); + this.jobStoreRedisConnection = jobStoreRedisConnection.getIfAvailable(); + } + + public StatefulRedisConnection getServingRedisConnection() { + return servingRedisConnection; + } + + public StatefulRedisConnection getJobStoreRedisConnection() { + return jobStoreRedisConnection; + } +} diff --git a/serving/src/main/java/feast/serving/configuration/redis/JobStoreRedisConfig.java b/serving/src/main/java/feast/serving/configuration/redis/JobStoreRedisConfig.java new file mode 100644 index 00000000000..77d9262bcb3 --- /dev/null +++ b/serving/src/main/java/feast/serving/configuration/redis/JobStoreRedisConfig.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.configuration.redis; + +import com.google.common.base.Enums; +import feast.core.StoreProto; +import feast.serving.FeastProperties; +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.codec.ByteArrayCodec; +import io.lettuce.core.resource.ClientResources; +import io.lettuce.core.resource.DefaultClientResources; +import java.util.Map; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JobStoreRedisConfig { + + @Bean(destroyMethod = "shutdown") + ClientResources jobStoreClientResources() { + return DefaultClientResources.create(); + } + + @Bean(destroyMethod = "shutdown") + RedisClient jobStoreRedisClient( + ClientResources jobStoreClientResources, FeastProperties feastProperties) { + StoreProto.Store.StoreType storeType = + Enums.getIfPresent( + StoreProto.Store.StoreType.class, feastProperties.getJobs().getStoreType()) + .orNull(); + if (storeType != StoreProto.Store.StoreType.REDIS) return null; + Map jobStoreConf = feastProperties.getJobs().getStoreOptions(); + // If job conf is empty throw StoreException + if (jobStoreConf == null + || jobStoreConf.get("host") == null + || jobStoreConf.get("host").isEmpty() + || jobStoreConf.get("port") == null + || jobStoreConf.get("port").isEmpty()) + throw new IllegalArgumentException("Store Configuration is not set"); + RedisURI uri = + RedisURI.create(jobStoreConf.get("host"), Integer.parseInt(jobStoreConf.get("port"))); + return RedisClient.create(jobStoreClientResources, uri); + } + + @Bean(destroyMethod = "close") + StatefulRedisConnection jobStoreRedisConnection( + ObjectProvider jobStoreRedisClient) { + if (jobStoreRedisClient.getIfAvailable() == null) return null; + return jobStoreRedisClient.getIfAvailable().connect(new ByteArrayCodec()); + } +} diff --git a/serving/src/main/java/feast/serving/configuration/redis/ServingStoreRedisConfig.java b/serving/src/main/java/feast/serving/configuration/redis/ServingStoreRedisConfig.java new file mode 100644 index 00000000000..17a50eef6d6 --- /dev/null +++ b/serving/src/main/java/feast/serving/configuration/redis/ServingStoreRedisConfig.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.configuration.redis; + +import feast.core.StoreProto; +import feast.serving.specs.CachedSpecService; +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.codec.ByteArrayCodec; +import io.lettuce.core.resource.ClientResources; +import io.lettuce.core.resource.DefaultClientResources; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.annotation.*; + +@Configuration +public class ServingStoreRedisConfig { + + @Bean + StoreProto.Store.RedisConfig servingStoreRedisConf(CachedSpecService specService) { + if (specService.getStore().getType() != StoreProto.Store.StoreType.REDIS) return null; + return specService.getStore().getRedisConfig(); + } + + @Bean(destroyMethod = "shutdown") + ClientResources servingClientResources() { + return DefaultClientResources.create(); + } + + @Bean(destroyMethod = "shutdown") + RedisClient servingRedisClient( + ClientResources servingClientResources, + ObjectProvider servingStoreRedisConf) { + if (servingStoreRedisConf.getIfAvailable() == null) return null; + RedisURI redisURI = + RedisURI.create( + servingStoreRedisConf.getIfAvailable().getHost(), + servingStoreRedisConf.getIfAvailable().getPort()); + return RedisClient.create(servingClientResources, redisURI); + } + + @Bean(destroyMethod = "close") + StatefulRedisConnection servingRedisConnection( + ObjectProvider servingRedisClient) { + if (servingRedisClient.getIfAvailable() == null) return null; + return servingRedisClient.getIfAvailable().connect(new ByteArrayCodec()); + } +} diff --git a/serving/src/main/java/feast/serving/controller/HealthServiceController.java b/serving/src/main/java/feast/serving/controller/HealthServiceController.java index 53728544656..3d34aea97b5 100644 --- a/serving/src/main/java/feast/serving/controller/HealthServiceController.java +++ b/serving/src/main/java/feast/serving/controller/HealthServiceController.java @@ -18,6 +18,7 @@ import feast.core.StoreProto.Store; import feast.serving.ServingAPIProto.GetFeastServingInfoRequest; +import feast.serving.interceptors.GrpcMonitoringInterceptor; import feast.serving.service.ServingService; import feast.serving.specs.CachedSpecService; import io.grpc.health.v1.HealthGrpc.HealthImplBase; @@ -30,7 +31,7 @@ // Reference: https://github.com/grpc/grpc/blob/master/doc/health-checking.md -@GRpcService +@GRpcService(interceptors = {GrpcMonitoringInterceptor.class}) public class HealthServiceController extends HealthImplBase { private CachedSpecService specService; private ServingService servingService; diff --git a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java index 0eb9d1e3450..cc1f856d728 100644 --- a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java +++ b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java @@ -26,6 +26,7 @@ import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest; import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.serving.ServingServiceGrpc.ServingServiceImplBase; +import feast.serving.interceptors.GrpcMonitoringInterceptor; import feast.serving.service.ServingService; import feast.serving.util.RequestHelper; import io.grpc.stub.StreamObserver; @@ -36,7 +37,7 @@ import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; -@GRpcService +@GRpcService(interceptors = {GrpcMonitoringInterceptor.class}) public class ServingServiceGRpcController extends ServingServiceImplBase { private static final Logger log = diff --git a/serving/src/main/java/feast/serving/encoding/FeatureRowDecoder.java b/serving/src/main/java/feast/serving/encoding/FeatureRowDecoder.java new file mode 100644 index 00000000000..e70695d8c64 --- /dev/null +++ b/serving/src/main/java/feast/serving/encoding/FeatureRowDecoder.java @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.encoding; + +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.core.FeatureSetProto.FeatureSpec; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class FeatureRowDecoder { + + private final String featureSetRef; + private final FeatureSetSpec spec; + + public FeatureRowDecoder(String featureSetRef, FeatureSetSpec spec) { + this.featureSetRef = featureSetRef; + this.spec = spec; + } + + /** + * A feature row is considered encoded if the feature set and field names are not set. This method + * is required for backward compatibility purposes, to allow Feast serving to continue serving non + * encoded Feature Row ingested by an older version of Feast. + * + * @param featureRow Feature row + * @return boolean + */ + public Boolean isEncoded(FeatureRow featureRow) { + return featureRow.getFeatureSet().isEmpty() + && featureRow.getFieldsList().stream().allMatch(field -> field.getName().isEmpty()); + } + + /** + * Validates if an encoded feature row can be decoded without exception. + * + * @param featureRow Feature row + * @return boolean + */ + public Boolean isEncodingValid(FeatureRow featureRow) { + return featureRow.getFieldsList().size() == spec.getFeaturesList().size(); + } + + /** + * Decoding feature row by repopulating the field names based on the corresponding feature set + * spec. + * + * @param encodedFeatureRow Feature row + * @return boolean + */ + public FeatureRow decode(FeatureRow encodedFeatureRow) { + final List fieldsWithoutName = encodedFeatureRow.getFieldsList(); + + List featureNames = + spec.getFeaturesList().stream() + .sorted(Comparator.comparing(FeatureSpec::getName)) + .map(FeatureSpec::getName) + .collect(Collectors.toList()); + List fields = + IntStream.range(0, featureNames.size()) + .mapToObj( + featureNameIndex -> { + String featureName = featureNames.get(featureNameIndex); + return fieldsWithoutName + .get(featureNameIndex) + .toBuilder() + .setName(featureName) + .build(); + }) + .collect(Collectors.toList()); + return encodedFeatureRow + .toBuilder() + .clearFields() + .setFeatureSet(featureSetRef) + .addAllFields(fields) + .build(); + } +} diff --git a/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java new file mode 100644 index 00000000000..bc7ed8997e3 --- /dev/null +++ b/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.interceptors; + +import feast.serving.util.Metrics; +import io.grpc.ForwardingServerCall.SimpleForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCall.Listener; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * GrpcMonitoringInterceptor intercepts GRPC calls to provide request latency histogram metrics in + * the Prometheus client. + */ +public class GrpcMonitoringInterceptor implements ServerInterceptor { + + @Override + public Listener interceptCall( + ServerCall call, Metadata headers, ServerCallHandler next) { + + long startCallMillis = System.currentTimeMillis(); + String fullMethodName = call.getMethodDescriptor().getFullMethodName(); + String methodName = fullMethodName.substring(fullMethodName.indexOf("/") + 1); + + return next.startCall( + new SimpleForwardingServerCall(call) { + @Override + public void close(Status status, Metadata trailers) { + Metrics.requestLatency + .labels(methodName) + .observe((System.currentTimeMillis() - startCallMillis) / 1000f); + Metrics.grpcRequestCount.labels(methodName, status.getCode().name()).inc(); + super.close(status, trailers); + } + }, + headers); + } +} diff --git a/serving/src/main/java/feast/serving/service/BigQueryServingService.java b/serving/src/main/java/feast/serving/service/BigQueryServingService.java index 53d071b57d1..8e3b7ae53e4 100644 --- a/serving/src/main/java/feast/serving/service/BigQueryServingService.java +++ b/serving/src/main/java/feast/serving/service/BigQueryServingService.java @@ -18,8 +18,8 @@ import static feast.serving.store.bigquery.QueryTemplater.createEntityTableUUIDQuery; import static feast.serving.store.bigquery.QueryTemplater.generateFullTableName; -import static feast.serving.util.Metrics.requestLatency; +import com.google.cloud.RetryOption; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.Field; @@ -57,12 +57,12 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; -import org.joda.time.Duration; import org.slf4j.Logger; +import org.threeten.bp.Duration; public class BigQueryServingService implements ServingService { - public static final long TEMP_TABLE_EXPIRY_DURATION_MS = Duration.standardDays(1).getMillis(); + public static final long TEMP_TABLE_EXPIRY_DURATION_MS = Duration.ofDays(1).toMillis(); private static final Logger log = org.slf4j.LoggerFactory.getLogger(BigQueryServingService.class); private final BigQuery bigquery; @@ -71,6 +71,8 @@ public class BigQueryServingService implements ServingService { private final CachedSpecService specService; private final JobService jobService; private final String jobStagingLocation; + private final int initialRetryDelaySecs; + private final int totalTimeoutSecs; private final Storage storage; public BigQueryServingService( @@ -80,6 +82,8 @@ public BigQueryServingService( CachedSpecService specService, JobService jobService, String jobStagingLocation, + int initialRetryDelaySecs, + int totalTimeoutSecs, Storage storage) { this.bigquery = bigquery; this.projectId = projectId; @@ -87,6 +91,8 @@ public BigQueryServingService( this.specService = specService; this.jobService = jobService; this.jobStagingLocation = jobStagingLocation; + this.initialRetryDelaySecs = initialRetryDelaySecs; + this.totalTimeoutSecs = totalTimeoutSecs; this.storage = storage; } @@ -109,7 +115,6 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getF /** {@inheritDoc} */ @Override public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { - long startTime = System.currentTimeMillis(); List featureSetRequests = specService.getFeatureSets(getFeaturesRequest.getFeaturesList()); @@ -156,12 +161,11 @@ public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeat .setEntityTableColumnNames(entityNames) .setFeatureSetInfos(featureSetInfos) .setJobStagingLocation(jobStagingLocation) + .setInitialRetryDelaySecs(initialRetryDelaySecs) + .setTotalTimeoutSecs(totalTimeoutSecs) .build()) .start(); - requestLatency - .labels("getBatchFeatures") - .observe((System.currentTimeMillis() - startTime) / 1000); return GetBatchFeaturesResponse.newBuilder().setJob(feastJob).build(); } @@ -199,7 +203,7 @@ private Table loadEntities(DatasetSource datasetSource) { loadJobConfiguration = loadJobConfiguration.toBuilder().setUseAvroLogicalTypes(true).build(); Job job = bigquery.create(JobInfo.of(loadJobConfiguration)); - job.waitFor(); + waitForJob(job); TableInfo expiry = bigquery @@ -239,7 +243,7 @@ private TableId generateUUIDs(Table loadedEntityTable) { .setDestinationTable(TableId.of(projectId, datasetId, createTempTableName())) .build(); Job queryJob = bigquery.create(JobInfo.of(queryJobConfig)); - queryJob.waitFor(); + Job completedJob = waitForJob(queryJob); TableInfo expiry = bigquery .getTable(queryJobConfig.getDestinationTable()) @@ -247,7 +251,7 @@ private TableId generateUUIDs(Table loadedEntityTable) { .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) .build(); bigquery.update(expiry); - queryJobConfig = queryJob.getConfiguration(); + queryJobConfig = completedJob.getConfiguration(); return queryJobConfig.getDestinationTable(); } catch (InterruptedException | BigQueryException e) { throw Status.INTERNAL @@ -257,6 +261,21 @@ private TableId generateUUIDs(Table loadedEntityTable) { } } + private Job waitForJob(Job queryJob) throws InterruptedException { + Job completedJob = + queryJob.waitFor( + RetryOption.initialRetryDelay(Duration.ofSeconds(initialRetryDelaySecs)), + RetryOption.totalTimeout(Duration.ofSeconds(totalTimeoutSecs))); + if (completedJob == null) { + throw Status.INTERNAL.withDescription("Job no longer exists").asRuntimeException(); + } else if (completedJob.getStatus().getError() != null) { + throw Status.INTERNAL + .withDescription("Job failed: " + completedJob.getStatus().getError()) + .asRuntimeException(); + } + return completedJob; + } + public static String createTempTableName() { return "_" + UUID.randomUUID().toString().replace("-", ""); } diff --git a/serving/src/main/java/feast/serving/service/CassandraBackedJobService.java b/serving/src/main/java/feast/serving/service/CassandraBackedJobService.java new file mode 100644 index 00000000000..833f3e5b302 --- /dev/null +++ b/serving/src/main/java/feast/serving/service/CassandraBackedJobService.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.bindMarker; +import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.selectFrom; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.*; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; +import com.datastax.oss.driver.api.querybuilder.insert.Insert; +import com.datastax.oss.driver.api.querybuilder.select.Select; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import feast.serving.ServingAPIProto.Job; +import feast.serving.ServingAPIProto.Job.Builder; +import java.time.LocalDate; +import java.util.Optional; +import org.joda.time.Duration; +import org.slf4j.Logger; + +// TODO: Do rate limiting, currently if clients call get() or upsert() +// and an exceedingly high rate e.g. they wrap job reload in a while loop with almost no wait +// Redis connection may break and need to restart Feast serving. Need to handle this. + +public class CassandraBackedJobService implements JobService { + + private static final Logger log = + org.slf4j.LoggerFactory.getLogger(CassandraBackedJobService.class); + private final CqlSession session; + private final int defaultExpirySeconds = (int) Duration.standardDays(1).getStandardSeconds(); + + public CassandraBackedJobService(CqlSession session) { + this.session = session; + } + + @Override + public Optional get(String id) { + Job job = null; + Job latestJob = Job.newBuilder().build(); + Select query = + selectFrom("admin", "jobs") + .column("job_uuid") + .column("job_data") + .column("timestamp") + .whereColumn("job_uuid") + .isEqualTo(bindMarker()); + PreparedStatement s = session.prepare(query.build()); + ResultSet res = session.execute(s.bind(id)); + LocalDate timestamp = LocalDate.EPOCH; + while (res.getAvailableWithoutFetching() > 0) { + Row r = res.one(); + ColumnDefinitions defs = r.getColumnDefinitions(); + LocalDate newTs = LocalDate.EPOCH; + for (int i = 0; i < defs.size(); i++) { + if (defs.get(i).equals("timestamp")) { + if (r.getLocalDate(i).compareTo(timestamp) > 0) { + newTs = r.getLocalDate(i); + } + } + if (defs.get(i).equals("job_data")) { + Builder builder = Job.newBuilder(); + try { + JsonFormat.parser().merge(r.getString(i), builder); + job = builder.build(); + } catch (InvalidProtocolBufferException e) { + throw new IllegalStateException("Could not build job from %s", e); + } + } + } + if (newTs.compareTo(timestamp) > 0) { + latestJob = job; + } + } + return Optional.ofNullable(latestJob); + } + + @Override + public void upsert(Job job) { + try { + Insert query = + QueryBuilder.insertInto("admin", "jobs") + .value("job_uuid", bindMarker()) + .value("timestamp", bindMarker()) + .value( + "job_data", + QueryBuilder.literal( + JsonFormat.printer().omittingInsignificantWhitespace().print(job))) + .usingTtl(defaultExpirySeconds); + PreparedStatement s = session.prepare(query.build()); + ResultSet res = + session.execute( + s.bind( + QueryBuilder.literal(job.getId()), + QueryBuilder.literal(System.currentTimeMillis()))); + } catch (Exception e) { + log.error(String.format("Failed to upsert job: %s", e.getMessage())); + } + } +} diff --git a/serving/src/main/java/feast/serving/service/CassandraServingService.java b/serving/src/main/java/feast/serving/service/CassandraServingService.java new file mode 100644 index 00000000000..eaf2acc96d5 --- /dev/null +++ b/serving/src/main/java/feast/serving/service/CassandraServingService.java @@ -0,0 +1,362 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import static feast.serving.util.Metrics.requestCount; +import static feast.serving.util.Metrics.requestLatency; +import static feast.serving.util.RefUtil.generateFeatureStringRef; + +import com.datastax.oss.driver.api.core.ConsistencyLevel; +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.*; +import com.google.common.collect.Maps; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Timestamp; +import feast.core.FeatureSetProto.EntitySpec; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.serving.ServingAPIProto.FeastServingType; +import feast.serving.ServingAPIProto.FeatureReference; +import feast.serving.ServingAPIProto.GetBatchFeaturesRequest; +import feast.serving.ServingAPIProto.GetBatchFeaturesResponse; +import feast.serving.ServingAPIProto.GetFeastServingInfoRequest; +import feast.serving.ServingAPIProto.GetFeastServingInfoResponse; +import feast.serving.ServingAPIProto.GetJobRequest; +import feast.serving.ServingAPIProto.GetJobResponse; +import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest; +import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; +import feast.serving.specs.CachedSpecService; +import feast.serving.specs.FeatureSetRequest; +import feast.serving.util.ValueUtil; +import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FieldProto.Field; +import feast.types.ValueProto.Value; +import io.grpc.Status; +import io.opentracing.Scope; +import io.opentracing.Tracer; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.slf4j.Logger; + +public class CassandraServingService implements ServingService { + + private static final Logger log = + org.slf4j.LoggerFactory.getLogger(CassandraServingService.class); + private final CqlSession session; + private final String keyspace; + private final String tableName; + private final Boolean versionless; + private final ConsistencyLevel consistency; + private final Tracer tracer; + private final PreparedStatement query; + private final CachedSpecService specService; + + public CassandraServingService( + CqlSession session, + String keyspace, + String tableName, + Boolean versionless, + ConsistencyLevel consistency, + CachedSpecService specService, + Tracer tracer) { + this.session = session; + this.keyspace = keyspace; + this.tableName = tableName; + this.tracer = tracer; + PreparedStatement query = + session.prepare( + String.format( + "SELECT entities, feature, value, WRITETIME(value) as writetime FROM %s.%s WHERE entities = ?", + keyspace, tableName)); + this.query = query; + this.specService = specService; + this.versionless = versionless; + this.consistency = consistency; + } + + /** {@inheritDoc} */ + @Override + public GetFeastServingInfoResponse getFeastServingInfo( + GetFeastServingInfoRequest getFeastServingInfoRequest) { + return GetFeastServingInfoResponse.newBuilder() + .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE) + .build(); + } + + /** {@inheritDoc} */ + @Override + public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { + try (Scope scope = tracer.buildSpan("Cassandra-getOnlineFeatures").startActive(true)) { + long startTime = System.currentTimeMillis(); + GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder = + GetOnlineFeaturesResponse.newBuilder(); + + List entityRows = request.getEntityRowsList(); + Map> featureValuesMap = + entityRows.stream() + .collect(Collectors.toMap(row -> row, row -> Maps.newHashMap(row.getFieldsMap()))); + List featureSetRequests = + specService.getFeatureSets(request.getFeaturesList()); + for (FeatureSetRequest featureSetRequest : featureSetRequests) { + + List featureSetEntityNames = + featureSetRequest.getSpec().getEntitiesList().stream() + .map(EntitySpec::getName) + .collect(Collectors.toList()); + + List cassandraKeys = + createLookupKeys(featureSetEntityNames, entityRows, featureSetRequest, versionless); + try { + getAndProcessAll(cassandraKeys, entityRows, featureValuesMap, featureSetRequest); + } catch (Exception e) { + log.error(e.getStackTrace().toString()); + throw Status.INTERNAL + .withDescription("Unable to parse cassandra response/ while retrieving feature") + .withCause(e) + .asRuntimeException(); + } + } + List fieldValues = + featureValuesMap.values().stream() + .map(valueMap -> FieldValues.newBuilder().putAllFields(valueMap).build()) + .collect(Collectors.toList()); + requestLatency + .labels("getOnlineFeatures") + .observe((System.currentTimeMillis() - startTime) / 1000); + return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build(); + } + } + + @Override + public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { + throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + } + + @Override + public GetJobResponse getJob(GetJobRequest getJobRequest) { + throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + } + + List createLookupKeys( + List featureSetEntityNames, + List entityRows, + FeatureSetRequest featureSetRequest, + Boolean versionless) { + try (Scope scope = tracer.buildSpan("Cassandra-makeCassandraKeys").startActive(true)) { + FeatureSetSpec fsSpec = featureSetRequest.getSpec(); + String featureSetId; + if (versionless) { + featureSetId = String.format("%s/%s", fsSpec.getProject(), fsSpec.getName()); + } else { + featureSetId = + String.format("%s/%s:%s", fsSpec.getProject(), fsSpec.getName(), fsSpec.getVersion()); + } + return entityRows.stream() + .map(row -> createCassandraKey(featureSetId, featureSetEntityNames, row)) + .collect(Collectors.toList()); + } + } + + /** + * Send a list of get request as an mget + * + * @param keys list of string keys + */ + protected void getAndProcessAll( + List keys, + List entityRows, + Map> featureValuesMap, + FeatureSetRequest featureSetRequest) { + FeatureSetSpec spec = featureSetRequest.getSpec(); + log.debug("Sending multi get: {}", keys); + List results = sendMultiGet(keys); + long startTime = System.currentTimeMillis(); + try (Scope scope = tracer.buildSpan("Cassandra-processResponse").startActive(true)) { + int foundResults = 0; + while (true) { + if (foundResults == results.size()) { + break; + } + for (int i = 0; i < results.size(); i++) { + EntityRow entityRow = entityRows.get(i); + Map featureValues = featureValuesMap.get(entityRow); + ResultSet queryRows = results.get(i); + Instant instant = Instant.now(); + List fields = new ArrayList<>(); + if (!queryRows.isFullyFetched()) { + continue; + } + List ee = queryRows.getExecutionInfos(); + foundResults += 1; + if (queryRows.getAvailableWithoutFetching() == 0) { + log.warn(String.format("Failed to find a row for the key %s", keys.get(i))); + log.warn( + String.format( + "Incoming Payload: %s", queryRows.getExecutionInfo().getIncomingPayload())); + log.warn(String.format("Errors: %s", queryRows.getExecutionInfo().getErrors())); + log.warn( + String.format("Coordinator: %s", queryRows.getExecutionInfo().getCoordinator())); + } + while (queryRows.getAvailableWithoutFetching() > 0) { + Row row = queryRows.one(); + ee = queryRows.getExecutionInfos(); + + long microSeconds = row.getLong("writetime"); + instant = + Instant.ofEpochSecond( + TimeUnit.MICROSECONDS.toSeconds(microSeconds), + TimeUnit.MICROSECONDS.toNanos( + Math.floorMod(microSeconds, TimeUnit.SECONDS.toMicros(1)))); + try { + fields.add( + Field.newBuilder() + .setName(row.getString("feature")) + .setValue( + Value.parseFrom(ByteBuffer.wrap(row.getBytesUnsafe("value").array()))) + .build()); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + } + } + FeatureRow featureRow = + FeatureRow.newBuilder() + .addAllFields(fields) + .setEventTimestamp( + Timestamp.newBuilder() + .setSeconds(instant.getEpochSecond()) + .setNanos(instant.getNano()) + .build()) + .build(); + featureSetRequest + .getFeatureReferences() + .parallelStream() + .forEach( + request -> + requestCount + .labels( + spec.getProject(), + String.format("%s:%d", request.getName(), request.getVersion())) + .inc()); + Map featureNames = + featureSetRequest.getFeatureReferences().stream() + .collect( + Collectors.toMap( + FeatureReference::getName, featureReference -> featureReference)); + featureRow.getFieldsList().stream() + .filter(field -> featureNames.keySet().contains(field.getName())) + .forEach( + field -> { + FeatureReference ref = featureNames.get(field.getName()); + String id = generateFeatureStringRef(ref); + featureValues.put(id, field.getValue()); + }); + } + } + } finally { + requestLatency + .labels("processResponse") + .observe((System.currentTimeMillis() - startTime) / 1000); + } + } + + FeatureRow parseResponse(ResultSet resultSet) { + List fields = new ArrayList<>(); + Instant instant = Instant.now(); + while (resultSet.getAvailableWithoutFetching() > 0) { + Row row = resultSet.one(); + long microSeconds = row.getLong("writetime"); + instant = + Instant.ofEpochSecond( + TimeUnit.MICROSECONDS.toSeconds(microSeconds), + TimeUnit.MICROSECONDS.toNanos( + Math.floorMod(microSeconds, TimeUnit.SECONDS.toMicros(1)))); + try { + fields.add( + Field.newBuilder() + .setName(row.getString("feature")) + .setValue(Value.parseFrom(ByteBuffer.wrap(row.getBytesUnsafe("value").array()))) + .build()); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + } + } + return FeatureRow.newBuilder() + .addAllFields(fields) + .setEventTimestamp( + Timestamp.newBuilder() + .setSeconds(instant.getEpochSecond()) + .setNanos(instant.getNano()) + .build()) + .build(); + } + + /** + * Create cassandra keys + * + * @param featureSet featureSet reference of the feature. E.g. feature_set_1:1 + * @param featureSetEntityNames entity names that belong to the featureSet + * @param entityRow entityRow to build the key from + * @return String + */ + private static String createCassandraKey( + String featureSet, List featureSetEntityNames, EntityRow entityRow) { + Map fieldsMap = entityRow.getFieldsMap(); + List res = new ArrayList<>(); + Collections.sort(featureSetEntityNames); + for (String entityName : featureSetEntityNames) { + res.add(entityName + "=" + ValueUtil.toString(fieldsMap.get(entityName))); + } + return featureSet + ":" + String.join("|", res); + } + + /** + * Send a list of get request as an cassandra execution + * + * @param keys list of cassandra keys + * @return list of {@link FeatureRow} in cassandra representation for each cassandra keys + */ + private List sendMultiGet(List keys) { + try (Scope scope = tracer.buildSpan("Cassandra-sendMultiGet").startActive(true)) { + List results = new ArrayList<>(); + long startTime = System.currentTimeMillis(); + try { + for (String key : keys) { + results.add( + session.execute( + query.bind(key).setTracing(false).setConsistencyLevel(this.consistency))); + } + return results; + } catch (Exception e) { + throw Status.NOT_FOUND + .withDescription("Unable to retrieve feature from Cassandra") + .withCause(e) + .asRuntimeException(); + } finally { + requestLatency + .labels("sendMultiGet") + .observe((System.currentTimeMillis() - startTime) / 1000d); + } + } + } +} diff --git a/serving/src/main/java/feast/serving/service/RedisBackedJobService.java b/serving/src/main/java/feast/serving/service/RedisBackedJobService.java index 7bfce552254..0bf53630379 100644 --- a/serving/src/main/java/feast/serving/service/RedisBackedJobService.java +++ b/serving/src/main/java/feast/serving/service/RedisBackedJobService.java @@ -19,10 +19,11 @@ import com.google.protobuf.util.JsonFormat; import feast.serving.ServingAPIProto.Job; import feast.serving.ServingAPIProto.Job.Builder; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.sync.RedisCommands; import java.util.Optional; import org.joda.time.Duration; import org.slf4j.Logger; -import redis.clients.jedis.Jedis; // TODO: Do rate limiting, currently if clients call get() or upsert() // and an exceedingly high rate e.g. they wrap job reload in a while loop with almost no wait @@ -31,38 +32,39 @@ public class RedisBackedJobService implements JobService { private static final Logger log = org.slf4j.LoggerFactory.getLogger(RedisBackedJobService.class); - private final Jedis jedis; + private final RedisCommands syncCommand; // Remove job state info after "defaultExpirySeconds" to prevent filling up Redis memory // and since users normally don't require info about relatively old jobs. private final int defaultExpirySeconds = (int) Duration.standardDays(1).getStandardSeconds(); - public RedisBackedJobService(Jedis jedis) { - this.jedis = jedis; + public RedisBackedJobService(StatefulRedisConnection connection) { + this.syncCommand = connection.sync(); } @Override public Optional get(String id) { - String json = jedis.get(id); - if (json == null) { - return Optional.empty(); - } Job job = null; - Builder builder = Job.newBuilder(); try { + String json = new String(syncCommand.get(id.getBytes())); + if (json.isEmpty()) { + return Optional.empty(); + } + Builder builder = Job.newBuilder(); JsonFormat.parser().merge(json, builder); job = builder.build(); } catch (Exception e) { log.error(String.format("Failed to parse JSON for Feast job: %s", e.getMessage())); } - return Optional.ofNullable(job); } @Override public void upsert(Job job) { try { - jedis.set(job.getId(), JsonFormat.printer().omittingInsignificantWhitespace().print(job)); - jedis.expire(job.getId(), defaultExpirySeconds); + syncCommand.set( + job.getId().getBytes(), + JsonFormat.printer().omittingInsignificantWhitespace().print(job).getBytes()); + syncCommand.expire(job.getId().getBytes(), defaultExpirySeconds); } catch (Exception e) { log.error(String.format("Failed to upsert job: %s", e.getMessage())); } diff --git a/serving/src/main/java/feast/serving/service/RedisServingService.java b/serving/src/main/java/feast/serving/service/RedisServingService.java index 48fc485214d..56ee1e80ec7 100644 --- a/serving/src/main/java/feast/serving/service/RedisServingService.java +++ b/serving/src/main/java/feast/serving/service/RedisServingService.java @@ -16,6 +16,7 @@ */ package feast.serving.service; +import static feast.serving.util.Metrics.invalidEncodingCount; import static feast.serving.util.Metrics.missingKeyCount; import static feast.serving.util.Metrics.requestCount; import static feast.serving.util.Metrics.requestLatency; @@ -41,6 +42,7 @@ import feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; +import feast.serving.encoding.FeatureRowDecoder; import feast.serving.specs.CachedSpecService; import feast.serving.specs.FeatureSetRequest; import feast.serving.util.RefUtil; @@ -49,24 +51,28 @@ import feast.types.FieldProto.Field; import feast.types.ValueProto.Value; import io.grpc.Status; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.sync.RedisCommands; import io.opentracing.Scope; import io.opentracing.Tracer; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import org.slf4j.Logger; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; public class RedisServingService implements ServingService { private static final Logger log = org.slf4j.LoggerFactory.getLogger(RedisServingService.class); - private final JedisPool jedisPool; private final CachedSpecService specService; private final Tracer tracer; + private final RedisCommands syncCommands; - public RedisServingService(JedisPool jedisPool, CachedSpecService specService, Tracer tracer) { - this.jedisPool = jedisPool; + public RedisServingService( + StatefulRedisConnection connection, + CachedSpecService specService, + Tracer tracer) { + this.syncCommands = connection.sync(); this.specService = specService; this.tracer = tracer; } @@ -84,7 +90,6 @@ public GetFeastServingInfoResponse getFeastServingInfo( @Override public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { try (Scope scope = tracer.buildSpan("Redis-getOnlineFeatures").startActive(true)) { - long startTime = System.currentTimeMillis(); GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder = GetOnlineFeaturesResponse.newBuilder(); @@ -106,7 +111,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ try { sendAndProcessMultiGet(redisKeys, entityRows, featureValuesMap, featureSetRequest); - } catch (InvalidProtocolBufferException e) { + } catch (InvalidProtocolBufferException | ExecutionException e) { throw Status.INTERNAL .withDescription("Unable to parse protobuf while retrieving feature") .withCause(e) @@ -117,9 +122,6 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ featureValuesMap.values().stream() .map(valueMap -> FieldValues.newBuilder().putAllFields(valueMap).build()) .collect(Collectors.toList()); - requestLatency - .labels("getOnlineFeatures") - .observe((System.currentTimeMillis() - startTime) / 1000); return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build(); } } @@ -192,9 +194,9 @@ private void sendAndProcessMultiGet( List entityRows, Map> featureValuesMap, FeatureSetRequest featureSetRequest) - throws InvalidProtocolBufferException { + throws InvalidProtocolBufferException, ExecutionException { - List jedisResps = sendMultiGet(redisKeys); + List values = sendMultiGet(redisKeys); long startTime = System.currentTimeMillis(); try (Scope scope = tracer.buildSpan("Redis-processResponse").startActive(true)) { FeatureSetSpec spec = featureSetRequest.getSpec(); @@ -206,12 +208,12 @@ private void sendAndProcessMultiGet( RefUtil::generateFeatureStringRef, featureReference -> Value.newBuilder().build())); - for (int i = 0; i < jedisResps.size(); i++) { + for (int i = 0; i < values.size(); i++) { EntityRow entityRow = entityRows.get(i); Map featureValues = featureValuesMap.get(entityRow); - byte[] jedisResponse = jedisResps.get(i); - if (jedisResponse == null) { + byte[] value = values.get(i); + if (value == null) { featureSetRequest .getFeatureReferences() .parallelStream() @@ -226,7 +228,28 @@ private void sendAndProcessMultiGet( continue; } - FeatureRow featureRow = FeatureRow.parseFrom(jedisResponse); + FeatureRow featureRow = FeatureRow.parseFrom(value); + String featureSetRef = redisKeys.get(i).getFeatureSet(); + FeatureRowDecoder decoder = + new FeatureRowDecoder(featureSetRef, specService.getFeatureSetSpec(featureSetRef)); + if (decoder.isEncoded(featureRow)) { + if (decoder.isEncodingValid(featureRow)) { + featureRow = decoder.decode(featureRow); + } else { + featureSetRequest + .getFeatureReferences() + .parallelStream() + .forEach( + request -> + invalidEncodingCount + .labels( + spec.getProject(), + String.format("%s:%d", request.getName(), request.getVersion())) + .inc()); + featureValues.putAll(nullValues); + continue; + } + } boolean stale = isStale(featureSetRequest, entityRow, featureRow); if (stale) { @@ -298,13 +321,15 @@ private boolean isStale( private List sendMultiGet(List keys) { try (Scope scope = tracer.buildSpan("Redis-sendMultiGet").startActive(true)) { long startTime = System.currentTimeMillis(); - try (Jedis jedis = jedisPool.getResource()) { + try { byte[][] binaryKeys = keys.stream() .map(AbstractMessageLite::toByteArray) .collect(Collectors.toList()) .toArray(new byte[0][0]); - return jedis.mget(binaryKeys); + return syncCommands.mget(binaryKeys).stream() + .map(io.lettuce.core.Value::getValue) + .collect(Collectors.toList()); } catch (Exception e) { throw Status.NOT_FOUND .withDescription("Unable to retrieve feature from Redis") @@ -313,7 +338,7 @@ private List sendMultiGet(List keys) { } finally { requestLatency .labels("sendMultiGet") - .observe((System.currentTimeMillis() - startTime) / 1000); + .observe((System.currentTimeMillis() - startTime) / 1000d); } } } diff --git a/serving/src/main/java/feast/serving/specs/CachedSpecService.java b/serving/src/main/java/feast/serving/specs/CachedSpecService.java index 040a870ffe1..085c456374a 100644 --- a/serving/src/main/java/feast/serving/specs/CachedSpecService.java +++ b/serving/src/main/java/feast/serving/specs/CachedSpecService.java @@ -49,7 +49,6 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; -import org.apache.commons.lang3.tuple.Triple; import org.slf4j.Logger; /** In-memory cache of specs. */ @@ -91,6 +90,7 @@ public CachedSpecService(CoreSpecService coreService, Path configPath) { featureSetCacheLoader = CacheLoader.from(featureSets::get); featureSetCache = CacheBuilder.newBuilder().maximumSize(MAX_SPEC_COUNT).build(featureSetCacheLoader); + featureSetCache.putAll(featureSets); } /** @@ -102,9 +102,14 @@ public Store getStore() { return this.store; } + public FeatureSetSpec getFeatureSetSpec(String featureSetRef) throws ExecutionException { + return featureSetCache.get(featureSetRef); + } + /** * Get FeatureSetSpecs for the given features. * + * @param featureReferences A reference to the corresponding feature set * @return FeatureSetRequest containing the specs, and their respective feature references */ public List getFeatureSets(List featureReferences) { @@ -194,13 +199,8 @@ private Map getFeatureSetMap() { private Map getFeatureToFeatureSetMapping( Map featureSets) { HashMap mapping = new HashMap<>(); - featureSets.values().stream() - .collect( - groupingBy( - featureSet -> - Triple.of( - featureSet.getProject(), featureSet.getName(), featureSet.getVersion()))) + .collect(groupingBy(featureSet -> Pair.of(featureSet.getProject(), featureSet.getName()))) .forEach( (group, groupedFeatureSets) -> { groupedFeatureSets = @@ -239,7 +239,6 @@ private Store readConfig(Path path) { try { List fileContents = Files.readAllLines(path); String yaml = fileContents.stream().reduce("", (l1, l2) -> l1 + "\n" + l2); - log.info("loaded store config at {}: \n{}", path.toString(), yaml); return yamlToStoreProto(yaml); } catch (IOException e) { throw new RuntimeException( diff --git a/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java b/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java index d437294dfc3..8589b432d1f 100644 --- a/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java +++ b/serving/src/main/java/feast/serving/store/bigquery/BatchRetrievalQueryRunnable.java @@ -21,6 +21,7 @@ import static feast.serving.store.bigquery.QueryTemplater.createTimestampLimitQuery; import com.google.auto.value.AutoValue; +import com.google.cloud.RetryOption; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.DatasetId; @@ -51,7 +52,29 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.threeten.bp.Duration; +/** + * BatchRetrievalQueryRunnable is a Runnable for running a BigQuery Feast batch retrieval job async. + * + *

It does the following, in sequence: + * + *

1. Retrieve the temporal bounds of the entity dataset provided. This will be used to filter + * the feature set tables when performing the feature retrieval. + * + *

2. For each of the feature sets requested, generate the subquery for doing a point-in-time + * correctness join of the features in the feature set to the entity table. + * + *

3. Run each of the subqueries in parallel and wait for them to complete. If any of the jobs + * are unsuccessful, the thread running the BatchRetrievalQueryRunnable catches the error and + * updates the job database. + * + *

4. When all the subquery jobs are complete, join the outputs of all the subqueries into a + * single table. + * + *

5. Extract the output of the join to a remote file, and write the location of the remote file + * to the job database, and mark the retrieval job as successful. + */ @AutoValue public abstract class BatchRetrievalQueryRunnable implements Runnable { @@ -75,6 +98,10 @@ public abstract class BatchRetrievalQueryRunnable implements Runnable { public abstract String jobStagingLocation(); + public abstract int initialRetryDelaySecs(); + + public abstract int totalTimeoutSecs(); + public abstract Storage storage(); public static Builder builder() { @@ -101,6 +128,10 @@ public abstract static class Builder { public abstract Builder setJobStagingLocation(String jobStagingLocation); + public abstract Builder setInitialRetryDelaySecs(int initialRetryDelaySecs); + + public abstract Builder setTotalTimeoutSecs(int totalTimeoutSecs); + public abstract Builder setStorage(Storage storage); public abstract BatchRetrievalQueryRunnable build(); @@ -109,24 +140,28 @@ public abstract static class Builder { @Override public void run() { + // 1. Retrieve the temporal bounds of the entity dataset provided FieldValueList timestampLimits = getTimestampLimits(entityTableName()); + // 2. Generate the subqueries List featureSetQueries = generateQueries(timestampLimits); QueryJobConfiguration queryConfig; try { + // 3 & 4. Run the subqueries in parallel then collect the outputs Job queryJob = runBatchQuery(featureSetQueries); queryConfig = queryJob.getConfiguration(); String exportTableDestinationUri = String.format("%s/%s/*.avro", jobStagingLocation(), feastJobId()); + // 5. Export the table // Hardcode the format to Avro for now ExtractJobConfiguration extractConfig = ExtractJobConfiguration.of( queryConfig.getDestinationTable(), exportTableDestinationUri, "Avro"); Job extractJob = bigquery().create(JobInfo.of(extractConfig)); - extractJob.waitFor(); + waitForJob(extractJob); } catch (BigQueryException | InterruptedException | IOException e) { jobService() .upsert( @@ -141,6 +176,7 @@ public void run() { List fileUris = parseOutputFileURIs(); + // 5. Update the job database jobService() .upsert( ServingAPIProto.Job.newBuilder() @@ -174,14 +210,16 @@ private List parseOutputFileURIs() { Job runBatchQuery(List featureSetQueries) throws BigQueryException, InterruptedException, IOException { - Job queryJob; ExecutorService executorService = Executors.newFixedThreadPool(featureSetQueries.size()); ExecutorCompletionService executorCompletionService = new ExecutorCompletionService<>(executorService); List featureSetInfos = new ArrayList<>(); + // For each of the feature sets requested, start an async job joining the features in that + // feature set to the provided entity table for (int i = 0; i < featureSetQueries.size(); i++) { + System.out.println(featureSetQueries.get(i)); QueryJobConfiguration queryJobConfig = QueryJobConfiguration.newBuilder(featureSetQueries.get(i)) .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) @@ -196,7 +234,10 @@ Job runBatchQuery(List featureSetQueries) } for (int i = 0; i < featureSetQueries.size(); i++) { + System.out.println(i); try { + // Try to retrieve the outputs of all the jobs. The timeout here is a formality; + // a stricter timeout is implemented in the actual SubqueryCallable. FeatureSetInfo featureSetInfo = executorCompletionService.take().get(SUBQUERY_TIMEOUT_SECS, TimeUnit.SECONDS); featureSetInfos.add(featureSetInfo); @@ -211,6 +252,7 @@ Job runBatchQuery(List featureSetQueries) .build()); executorService.shutdownNow(); + e.printStackTrace(); throw Status.INTERNAL .withDescription("Error running batch query") .withCause(e) @@ -218,6 +260,8 @@ Job runBatchQuery(List featureSetQueries) } } + // Generate and run a join query to collect the outputs of all the + // subqueries into a single table. String joinQuery = QueryTemplater.createJoinQuery( featureSetInfos, entityTableColumnNames(), entityTableName()); @@ -225,8 +269,8 @@ Job runBatchQuery(List featureSetQueries) QueryJobConfiguration.newBuilder(joinQuery) .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) .build(); - queryJob = bigquery().create(JobInfo.of(queryJobConfig)); - queryJob.waitFor(); + Job queryJob = bigquery().create(JobInfo.of(queryJobConfig)); + Job completedQueryJob = waitForJob(queryJob); TableInfo expiry = bigquery() @@ -236,7 +280,7 @@ Job runBatchQuery(List featureSetQueries) .build(); bigquery().update(expiry); - return queryJob; + return completedQueryJob; } private List generateQueries(FieldValueList timestampLimits) { @@ -270,7 +314,7 @@ private FieldValueList getTimestampLimits(String entityTableName) { .build(); try { Job job = bigquery().create(JobInfo.of(getTimestampLimitsQuery)); - TableResult getTimestampLimitsQueryResult = job.waitFor().getQueryResults(); + TableResult getTimestampLimitsQueryResult = waitForJob(job).getQueryResults(); TableInfo expiry = bigquery() .getTable(getTimestampLimitsQuery.getDestinationTable()) @@ -293,4 +337,19 @@ private FieldValueList getTimestampLimits(String entityTableName) { .asRuntimeException(); } } + + private Job waitForJob(Job queryJob) throws InterruptedException { + Job completedJob = + queryJob.waitFor( + RetryOption.initialRetryDelay(Duration.ofSeconds(initialRetryDelaySecs())), + RetryOption.totalTimeout(Duration.ofSeconds(totalTimeoutSecs()))); + if (completedJob == null) { + throw Status.INTERNAL.withDescription("Job no longer exists").asRuntimeException(); + } else if (completedJob.getStatus().getError() != null) { + throw Status.INTERNAL + .withDescription("Job failed: " + completedJob.getStatus().getError()) + .asRuntimeException(); + } + return completedJob; + } } diff --git a/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java b/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java index e0b8f457986..14026030b42 100644 --- a/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java +++ b/serving/src/main/java/feast/serving/store/bigquery/SubqueryCallable.java @@ -30,8 +30,8 @@ import java.util.concurrent.Callable; /** - * Waits for a bigquery job to complete; when complete, it updates the feature set info with the - * output table name, as well as increments the completed jobs counter in the query job listener. + * Waits for a point-in-time correctness join to complete. On completion, returns a featureSetInfo + * updated with the reference to the table containing the results of the query. */ @AutoValue public abstract class SubqueryCallable implements Callable { diff --git a/serving/src/main/java/feast/serving/util/Metrics.java b/serving/src/main/java/feast/serving/util/Metrics.java index 99f6353e742..fa66f79a804 100644 --- a/serving/src/main/java/feast/serving/util/Metrics.java +++ b/serving/src/main/java/feast/serving/util/Metrics.java @@ -24,9 +24,9 @@ public class Metrics { public static final Histogram requestLatency = Histogram.build() .buckets(0.001, 0.002, 0.004, 0.006, 0.008, 0.01, 0.015, 0.02, 0.025, 0.03, 0.035, 0.05) - .name("request_latency_ms") + .name("request_latency_seconds") .subsystem("feast_serving") - .help("Request latency in seconds.") + .help("Request latency in seconds") .labelNames("method") .register(); @@ -46,6 +46,14 @@ public class Metrics { .labelNames("project", "feature_name") .register(); + public static final Counter invalidEncodingCount = + Counter.build() + .name("invalid_encoding_feature_count") + .subsystem("feast_serving") + .help("number requested feature rows that were stored with the wrong encoding") + .labelNames("project", "feature_name") + .register(); + public static final Counter staleKeyCount = Counter.build() .name("stale_feature_count") @@ -53,4 +61,12 @@ public class Metrics { .help("number requested feature rows that were stale") .labelNames("project", "feature_name") .register(); + + public static final Counter grpcRequestCount = + Counter.build() + .name("grpc_request_count") + .subsystem("feast_serving") + .help("number of grpc requests served") + .labelNames("method", "status_code") + .register(); } diff --git a/serving/src/main/java/feast/serving/util/ValueUtil.java b/serving/src/main/java/feast/serving/util/ValueUtil.java new file mode 100644 index 00000000000..e3ede6af984 --- /dev/null +++ b/serving/src/main/java/feast/serving/util/ValueUtil.java @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.util; + +import feast.types.ValueProto.Value; + +public class ValueUtil { + + public static String toString(Value value) { + String strValue; + switch (value.getValCase()) { + case BYTES_VAL: + strValue = value.getBytesVal().toString(); + break; + case STRING_VAL: + strValue = value.getStringVal(); + break; + case INT32_VAL: + strValue = String.valueOf(value.getInt32Val()); + break; + case INT64_VAL: + strValue = String.valueOf(value.getInt64Val()); + break; + case DOUBLE_VAL: + strValue = String.valueOf(value.getDoubleVal()); + break; + case FLOAT_VAL: + strValue = String.valueOf(value.getFloatVal()); + break; + case BOOL_VAL: + strValue = String.valueOf(value.getBoolVal()); + break; + default: + throw new IllegalArgumentException( + String.format("toString method not supported for type %s", value.getValCase())); + } + return strValue; + } +} diff --git a/serving/src/main/proto/feast b/serving/src/main/proto/feast deleted file mode 120000 index d520da9126b..00000000000 --- a/serving/src/main/proto/feast +++ /dev/null @@ -1 +0,0 @@ -../../../../protos/feast \ No newline at end of file diff --git a/serving/src/main/proto/third_party b/serving/src/main/proto/third_party deleted file mode 120000 index 363d20598e6..00000000000 --- a/serving/src/main/proto/third_party +++ /dev/null @@ -1 +0,0 @@ -../../../../protos/third_party \ No newline at end of file diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index 2daa83fbfb2..8c06b70ad79 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -1,7 +1,7 @@ feast: # This value is retrieved from project.version properties in pom.xml # https://docs.spring.io/spring-boot/docs/current/reference/html/ - version: @project.version@ +# version: @project.version@ # GRPC service address for Feast Core # Feast Serving requires connection to Feast Core to retrieve and reload Feast metadata (e.g. FeatureSpecs, Store information) core-host: ${FEAST_CORE_HOST:localhost} @@ -24,15 +24,38 @@ feast: redis-pool-max-size: ${FEAST_REDIS_POOL_MAX_SIZE:128} # If serving redis, the redis pool max idle conns redis-pool-max-idle: ${FEAST_REDIS_POOL_MAX_IDLE:16} + # If serving cassandra, minimum connection for local host (one in same data center) + cassandra-pool-core-local-connections: ${FEAST_CASSANDRA_CORE_LOCAL_CONNECTIONS:1} + # If serving cassandra, maximum connection for local host (one in same data center) + cassandra-pool-max-local-connections: ${FEAST_CASSANDRA_MAX_LOCAL_CONNECTIONS:1} + # If serving cassandra, minimum connection for remote host (one in remote data center) + cassandra-pool-core-remote-connections: ${FEAST_CASSANDRA_CORE_REMOTE_CONNECTIONS:1} + # If serving cassandra, maximum connection for remote host (one in same data center) + cassandra-pool-max-remote-connections: ${FEAST_CASSANDRA_MAX_REMOTE_CONNECTIONS:1} + # If serving cassandra, maximum number of concurrent requests per local connection (one in same data center) + cassandra-pool-max-requests-local-connection: ${FEAST_CASSANDRA_MAX_REQUESTS_LOCAL_CONNECTION:32768} + # If serving cassandra, maximum number of concurrent requests per remote connection (one in remote data center) + cassandra-pool-max-requests-remote-connection: ${FEAST_CASSANDRA_MAX_REQUESTS_REMOTE_CONNECTION:2048} + # If serving cassandra, number of requests which trigger opening of new local connection (if it is available) + cassandra-pool-new-local-connection-threshold: ${FEAST_CASSANDRA_NEW_LOCAL_CONNECTION_THRESHOLD:30000} + # If serving cassandra, number of requests which trigger opening of new remote connection (if it is available) + cassandra-pool-new-remote-connection-threshold: ${FEAST_CASSANDRA_NEW_REMOTE_CONNECTION_THRESHOLD:400} + # If serving cassandra, number of milliseconds to wait to acquire connection (after that go to next available host in query plan) + cassandra-pool-timeout-millis: ${FEAST_CASSANDRA_POOL_TIMEOUT_MILLIS:0} jobs: - # job-staging-location specifies the URI to store intermediate files for batch serving. + # staging-location specifies the URI to store intermediate files for batch serving. # Feast Serving client is expected to have read access to this staging location # to download the batch features. # # For example: gs://mybucket/myprefix # Please omit the trailing slash in the URI. staging-location: ${FEAST_JOB_STAGING_LOCATION:} + # + # Retry options for BigQuery jobs: + bigquery-initial-retry-delay-secs: 1 + bigquery-total-timeout-secs: 21600 + # # Type of store to store job metadata. This only needs to be set if the # serving store type is Bigquery. store-type: ${FEAST_JOB_STORE_TYPE:} @@ -42,6 +65,10 @@ feast: # store-options: # host: localhost # port: 6379 + # Optionally, you can configure the connection pool with the following items: + # max-conn: 8 + # max-idle: 8 + # max-wait-millis: 50 store-options: {} grpc: @@ -57,4 +84,4 @@ server: # The port number on which the Tomcat webserver that serves REST API endpoints should listen # It is set by default to 8081 so it does not conflict with Tomcat webserver on Feast Core # if both Feast Core and Serving are running on the same machine - port: ${SERVER_PORT:8081} \ No newline at end of file + port: ${SERVER_PORT:8081} diff --git a/serving/src/main/resources/log4j2.xml b/serving/src/main/resources/log4j2.xml index 02520cb36c2..1948c302be3 100644 --- a/serving/src/main/resources/log4j2.xml +++ b/serving/src/main/resources/log4j2.xml @@ -21,19 +21,23 @@ %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex + ${env:LOG_TYPE:-Console} + ${env:LOG_LEVEL:-info} + + + - - + + - - - + + \ No newline at end of file diff --git a/serving/src/main/resources/templates/join_featuresets.sql b/serving/src/main/resources/templates/join_featuresets.sql index e57b0c10314..60b7c7d7a12 100644 --- a/serving/src/main/resources/templates/join_featuresets.sql +++ b/serving/src/main/resources/templates/join_featuresets.sql @@ -1,3 +1,6 @@ +/* + Joins the outputs of multiple point-in-time-correctness joins to a single table. + */ WITH joined as ( SELECT * FROM `{{ leftTableName }}` {% for featureSet in featureSets %} diff --git a/serving/src/main/resources/templates/single_featureset_pit_join.sql b/serving/src/main/resources/templates/single_featureset_pit_join.sql index f6678421851..f3f20828ff1 100644 --- a/serving/src/main/resources/templates/single_featureset_pit_join.sql +++ b/serving/src/main/resources/templates/single_featureset_pit_join.sql @@ -1,9 +1,24 @@ -WITH union_features AS (SELECT +/* + This query template performs the point-in-time correctness join for a single feature set table + to the provided entity table. + + 1. Concatenate the timestamp and entities from the feature set table with the entity dataset. + Feature values are joined to this table later for improved efficiency. + featureset_timestamp is equal to null in rows from the entity dataset. + */ +WITH union_features AS ( +SELECT + -- uuid is a unique identifier for each row in the entity dataset. Generated by `QueryTemplater.createEntityTableUUIDQuery` uuid, + -- event_timestamp contains the timestamps to join onto event_timestamp, + -- the feature_timestamp, i.e. the latest occurrence of the requested feature relative to the entity_dataset timestamp NULL as {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, + -- created timestamp of the feature at the corresponding feature_timestamp NULL as created_timestamp, + -- select only entities belonging to this feature set {{ featureSet.entities | join(', ')}}, + -- boolean for filtering the dataset later true AS is_entity_table FROM `{{leftTableName}}` UNION ALL @@ -14,14 +29,26 @@ SELECT created_timestamp, {{ featureSet.entities | join(', ')}}, false AS is_entity_table -FROM `{{projectId}}.{{datasetId}}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` WHERE event_timestamp <= '{{maxTimestamp}}' AND event_timestamp >= Timestamp_sub(TIMESTAMP '{{ minTimestamp }}', interval {{ featureSet.maxAge }} second) -), joined AS ( +FROM `{{projectId}}.{{datasetId}}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` WHERE event_timestamp <= '{{maxTimestamp}}' +{% if featureSet.maxAge == 0 %}{% else %}AND event_timestamp >= Timestamp_sub(TIMESTAMP '{{ minTimestamp }}', interval {{ featureSet.maxAge }} second){% endif %} +), +/* + 2. Window the data in the unioned dataset, partitioning by entity and ordering by event_timestamp, as + well as is_entity_table. + Within each window, back-fill the feature_timestamp - as a result of this, the null feature_timestamps + in the rows from the entity table should now contain the latest timestamps relative to the row's + event_timestamp. + + For rows where event_timestamp(provided datetime) - feature_timestamp > max age, set the + feature_timestamp to null. + */ +joined AS ( SELECT uuid, event_timestamp, {{ featureSet.entities | join(', ')}}, {% for featureName in featureSet.features %} - IF(event_timestamp >= {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp AND Timestamp_sub(event_timestamp, interval {{ featureSet.maxAge }} second) < {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}, NULL) as {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}{% if loop.last %}{% else %}, {% endif %} + IF(event_timestamp >= {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp {% if featureSet.maxAge == 0 %}{% else %}AND Timestamp_sub(event_timestamp, interval {{ featureSet.maxAge }} second) < {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp{% endif %}, {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}, NULL) as {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}{% if loop.last %}{% else %}, {% endif %} {% endfor %} FROM ( SELECT @@ -34,6 +61,10 @@ SELECT FROM union_features WINDOW w AS (PARTITION BY {{ featureSet.entities | join(', ') }} ORDER BY event_timestamp DESC, is_entity_table DESC, created_timestamp DESC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) ) +/* + 3. Select only the rows from the entity table, and join the features from the original feature set table + to the dataset using the entity values, feature_timestamp, and created_timestamps. + */ LEFT JOIN ( SELECT event_timestamp as {{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, @@ -42,10 +73,14 @@ SELECT {% for featureName in featureSet.features %} {{ featureName }} as {{ featureSet.project }}_{{ featureName }}_v{{ featureSet.version }}{% if loop.last %}{% else %}, {% endif %} {% endfor %} -FROM `{{projectId}}.{{datasetId}}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` WHERE event_timestamp <= '{{maxTimestamp}}' AND event_timestamp >= Timestamp_sub(TIMESTAMP '{{ minTimestamp }}', interval {{ featureSet.maxAge }} second) +FROM `{{projectId}}.{{datasetId}}.{{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}` WHERE event_timestamp <= '{{maxTimestamp}}' +{% if featureSet.maxAge == 0 %}{% else %}AND event_timestamp >= Timestamp_sub(TIMESTAMP '{{ minTimestamp }}', interval {{ featureSet.maxAge }} second){% endif %} ) USING ({{ featureSet.project }}_{{ featureSet.name }}_v{{ featureSet.version }}_feature_timestamp, created_timestamp, {{ featureSet.entities | join(', ')}}) WHERE is_entity_table ) +/* + 4. Finally, deduplicate the rows by selecting the first occurrence of each entity table row UUID. + */ SELECT k.* FROM ( diff --git a/serving/src/test/java/feast/serving/encoding/FeatureRowDecoderTest.java b/serving/src/test/java/feast/serving/encoding/FeatureRowDecoderTest.java new file mode 100644 index 00000000000..8f6c79ad66c --- /dev/null +++ b/serving/src/test/java/feast/serving/encoding/FeatureRowDecoderTest.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.encoding; + +import static org.junit.Assert.*; + +import com.google.protobuf.Timestamp; +import feast.core.FeatureSetProto; +import feast.core.FeatureSetProto.FeatureSetSpec; +import feast.types.FeatureRowProto; +import feast.types.FieldProto.Field; +import feast.types.ValueProto.Value; +import feast.types.ValueProto.ValueType; +import java.util.Collections; +import org.junit.Test; + +public class FeatureRowDecoderTest { + + private FeatureSetProto.EntitySpec entity = + FeatureSetProto.EntitySpec.newBuilder().setName("entity1").build(); + + private FeatureSetSpec spec = + FeatureSetSpec.newBuilder() + .addAllEntities(Collections.singletonList(entity)) + .addFeatures( + FeatureSetProto.FeatureSpec.newBuilder() + .setName("feature1") + .setValueType(ValueType.Enum.FLOAT)) + .addFeatures( + FeatureSetProto.FeatureSpec.newBuilder() + .setName("feature2") + .setValueType(ValueType.Enum.INT32)) + .setName("feature_set_name") + .build(); + + @Test + public void featureRowWithFieldNamesIsNotConsideredAsEncoded() { + + FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); + FeatureRowProto.FeatureRow nonEncodedFeatureRow = + FeatureRowProto.FeatureRow.newBuilder() + .setFeatureSet("feature_set_ref") + .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) + .addFields( + Field.newBuilder().setName("feature1").setValue(Value.newBuilder().setInt32Val(2))) + .addFields( + Field.newBuilder() + .setName("feature2") + .setValue(Value.newBuilder().setFloatVal(1.0f))) + .build(); + assertFalse(decoder.isEncoded(nonEncodedFeatureRow)); + } + + @Test + public void encodingIsInvalidIfNumberOfFeaturesInSpecDiffersFromFeatureRow() { + + FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); + + FeatureRowProto.FeatureRow encodedFeatureRow = + FeatureRowProto.FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) + .addFields(Field.newBuilder().setValue(Value.newBuilder().setInt32Val(2))) + .build(); + + assertFalse(decoder.isEncodingValid(encodedFeatureRow)); + } + + @Test + public void shouldDecodeValidEncodedFeatureRow() { + + FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); + + FeatureRowProto.FeatureRow encodedFeatureRow = + FeatureRowProto.FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) + .addFields(Field.newBuilder().setValue(Value.newBuilder().setInt32Val(2))) + .addFields(Field.newBuilder().setValue(Value.newBuilder().setFloatVal(1.0f))) + .build(); + + FeatureRowProto.FeatureRow expectedFeatureRow = + FeatureRowProto.FeatureRow.newBuilder() + .setFeatureSet("feature_set_ref") + .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) + .addFields( + Field.newBuilder().setName("feature1").setValue(Value.newBuilder().setInt32Val(2))) + .addFields( + Field.newBuilder() + .setName("feature2") + .setValue(Value.newBuilder().setFloatVal(1.0f))) + .build(); + + assertTrue(decoder.isEncoded(encodedFeatureRow)); + assertTrue(decoder.isEncodingValid(encodedFeatureRow)); + assertEquals(expectedFeatureRow, decoder.decode(encodedFeatureRow)); + } +} diff --git a/serving/src/test/java/feast/serving/service/RedisBackedJobServiceTest.java b/serving/src/test/java/feast/serving/service/RedisBackedJobServiceTest.java new file mode 100644 index 00000000000..34bc31d2c26 --- /dev/null +++ b/serving/src/test/java/feast/serving/service/RedisBackedJobServiceTest.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.service; + +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.codec.ByteArrayCodec; +import java.io.IOException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import redis.embedded.RedisServer; + +public class RedisBackedJobServiceTest { + private static Integer REDIS_PORT = 51235; + private RedisServer redis; + + @Before + public void setUp() throws IOException { + redis = new RedisServer(REDIS_PORT); + redis.start(); + } + + @After + public void teardown() { + redis.stop(); + } + + @Test + public void shouldRecoverIfRedisConnectionIsLost() throws IOException { + RedisClient client = RedisClient.create(RedisURI.create("localhost", REDIS_PORT)); + RedisBackedJobService jobService = + new RedisBackedJobService(client.connect(new ByteArrayCodec())); + jobService.get("does not exist"); + redis.stop(); + try { + jobService.get("does not exist"); + } catch (Exception e) { + // pass, this should fail, and return a broken connection to the pool + } + redis.start(); + jobService.get("does not exist"); + client.shutdown(); + } +} diff --git a/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java b/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java index 042107e1177..d8778747946 100644 --- a/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/RedisServingServiceTest.java @@ -16,6 +16,9 @@ */ package feast.serving.service; +import static feast.serving.test.TestUtil.intValue; +import static feast.serving.test.TestUtil.responseToMapList; +import static feast.serving.test.TestUtil.strValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.mockito.Mockito.when; @@ -38,38 +41,37 @@ import feast.types.FeatureRowProto.FeatureRow; import feast.types.FieldProto.Field; import feast.types.ValueProto.Value; +import io.lettuce.core.KeyValue; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.sync.RedisCommands; import io.opentracing.Tracer; import io.opentracing.Tracer.SpanBuilder; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Mockito; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; public class RedisServingServiceTest { - @Mock JedisPool jedisPool; - - @Mock Jedis jedis; - @Mock CachedSpecService specService; @Mock Tracer tracer; + @Mock StatefulRedisConnection connection; + + @Mock RedisCommands syncCommands; + private RedisServingService redisServingService; private byte[][] redisKeyList; @Before public void setUp() { initMocks(this); - - redisServingService = new RedisServingService(jedisPool, specService, tracer); + when(connection.sync()).thenReturn(syncCommands); + redisServingService = new RedisServingService(connection, specService, tracer); redisKeyList = Lists.newArrayList( RedisKey.newBuilder() @@ -149,12 +151,14 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { .setSpec(getFeatureSetSpec()) .build(); - List featureRowBytes = - featureRows.stream().map(AbstractMessageLite::toByteArray).collect(Collectors.toList()); + List> featureRowBytes = + featureRows.stream() + .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) + .collect(Collectors.toList()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(jedisPool.getResource()).thenReturn(jedis); - when(jedis.mget(redisKeyList)).thenReturn(featureRowBytes); + when(connection.sync()).thenReturn(syncCommands); + when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -234,12 +238,14 @@ public void shouldReturnResponseWithValuesWhenFeatureSetSpecHasUnspecifiedMaxAge .setSpec(getFeatureSetSpecWithNoMaxAge()) .build(); - List featureRowBytes = - featureRows.stream().map(AbstractMessageLite::toByteArray).collect(Collectors.toList()); + List> featureRowBytes = + featureRows.stream() + .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) + .collect(Collectors.toList()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(jedisPool.getResource()).thenReturn(jedis); - when(jedis.mget(redisKeyList)).thenReturn(featureRowBytes); + when(connection.sync()).thenReturn(syncCommands); + when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -315,12 +321,14 @@ public void shouldReturnKeysWithoutVersionifNotProvided() { .setSpec(getFeatureSetSpec()) .build(); - List featureRowBytes = - featureRows.stream().map(AbstractMessageLite::toByteArray).collect(Collectors.toList()); + List> featureRowBytes = + featureRows.stream() + .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) + .collect(Collectors.toList()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(jedisPool.getResource()).thenReturn(jedis); - when(jedis.mget(redisKeyList)).thenReturn(featureRowBytes); + when(connection.sync()).thenReturn(syncCommands); + when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -401,11 +409,14 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { .setSpec(getFeatureSetSpec()) .build(); - List featureRowBytes = Lists.newArrayList(featureRows.get(0).toByteArray(), null); + List> featureRowBytes = + featureRows.stream() + .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) + .collect(Collectors.toList()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(jedisPool.getResource()).thenReturn(jedis); - when(jedis.mget(redisKeyList)).thenReturn(featureRowBytes); + when(connection.sync()).thenReturn(syncCommands); + when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -489,12 +500,14 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { .setSpec(spec) .build(); - List featureRowBytes = - featureRows.stream().map(AbstractMessageLite::toByteArray).collect(Collectors.toList()); + List> featureRowBytes = + featureRows.stream() + .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) + .collect(Collectors.toList()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(jedisPool.getResource()).thenReturn(jedis); - when(jedis.mget(redisKeyList)).thenReturn(featureRowBytes); + when(connection.sync()).thenReturn(syncCommands); + when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -569,12 +582,14 @@ public void shouldFilterOutUndesiredRows() { .setSpec(getFeatureSetSpec()) .build(); - List featureRowBytes = - featureRows.stream().map(AbstractMessageLite::toByteArray).collect(Collectors.toList()); + List> featureRowBytes = + featureRows.stream() + .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) + .collect(Collectors.toList()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(jedisPool.getResource()).thenReturn(jedis); - when(jedis.mget(redisKeyList)).thenReturn(featureRowBytes); + when(connection.sync()).thenReturn(syncCommands); + when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -595,20 +610,6 @@ public void shouldFilterOutUndesiredRows() { responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); } - private List> responseToMapList(GetOnlineFeaturesResponse response) { - return response.getFieldValuesList().stream() - .map(FieldValues::getFieldsMap) - .collect(Collectors.toList()); - } - - private Value intValue(int val) { - return Value.newBuilder().setInt64Val(val).build(); - } - - private Value strValue(String val) { - return Value.newBuilder().setStringVal(val).build(); - } - private FeatureSetSpec getFeatureSetSpec() { return FeatureSetSpec.newBuilder() .setProject("project") diff --git a/serving/src/test/java/feast/serving/test/TestUtil.java b/serving/src/test/java/feast/serving/test/TestUtil.java new file mode 100644 index 00000000000..b5eb371ab25 --- /dev/null +++ b/serving/src/test/java/feast/serving/test/TestUtil.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.test; + +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; +import feast.types.ValueProto.Value; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@SuppressWarnings("WeakerAccess") +public class TestUtil { + + public static List> responseToMapList(GetOnlineFeaturesResponse response) { + return response.getFieldValuesList().stream() + .map(FieldValues::getFieldsMap) + .collect(Collectors.toList()); + } + + public static Value intValue(int val) { + return Value.newBuilder().setInt64Val(val).build(); + } + + public static Value strValue(String val) { + return Value.newBuilder().setStringVal(val).build(); + } +} diff --git a/serving/src/test/resources/embedded-store/LoadCassandra.cql b/serving/src/test/resources/embedded-store/LoadCassandra.cql new file mode 100644 index 00000000000..c80da294b71 --- /dev/null +++ b/serving/src/test/resources/embedded-store/LoadCassandra.cql @@ -0,0 +1,8 @@ +CREATE KEYSPACE test with replication = {'class':'SimpleStrategy','replication_factor':1}; + +CREATE TABLE test.feature_store( + entities text, + feature text, + value blob, + PRIMARY KEY (entities, feature) +) WITH CLUSTERING ORDER BY (feature DESC); \ No newline at end of file diff --git a/tests/e2e/bq-batch-retrieval.py b/tests/e2e/bq-batch-retrieval.py index 8616dd37a92..0cf05e77e1d 100644 --- a/tests/e2e/bq-batch-retrieval.py +++ b/tests/e2e/bq-batch-retrieval.py @@ -118,6 +118,14 @@ def test_apply_all_featuresets(client): client.apply(fs1) client.apply(fs2) + no_max_age_fs = FeatureSet( + "no_max_age", + features=[Feature("feature_value8", ValueType.INT64)], + entities=[Entity("entity_id", ValueType.INT64)], + max_age=Duration(seconds=0), + ) + client.apply(no_max_age_fs) + def test_get_batch_features_with_file(client): file_fs1 = client.get_feature_set(name="file_feature_set", version=1) @@ -327,3 +335,28 @@ def test_multiple_featureset_joins(client): assert output["entity_id"].to_list() == [int(i) for i in output["feature_value6"].to_list()] assert output["other_entity_id"].to_list() == output["other_feature_value7"].to_list() + + +def test_no_max_age(client): + no_max_age_fs = client.get_feature_set(name="no_max_age", version=1) + + time_offset = datetime.utcnow().replace(tzinfo=pytz.utc) + N_ROWS = 10 + features_8_df = pd.DataFrame( + { + "datetime": [time_offset] * N_ROWS, + "entity_id": [i for i in range(N_ROWS)], + "feature_value8": [i for i in range(N_ROWS)], + } + ) + client.ingest(no_max_age_fs, features_8_df) + + time.sleep(15) + feature_retrieval_job = client.get_batch_features( + entity_rows=features_8_df[["datetime", "entity_id"]], feature_refs=[f"{PROJECT_NAME}/feature_value8:1"] + ) + + output = feature_retrieval_job.to_dataframe() + print(output.head()) + + assert output["entity_id"].to_list() == output["feature_value8"].to_list() \ No newline at end of file