squashing last 61 commits.

Merge remote-tracking branch 'fork/feature/adding-remote-onlinestore-rebase' into feature/adding-remote-onlinestore-rebase Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com>
feast-dev · Jun 13, 2024 · f2079fb · f2079fb
1 parent 1f17b9b
commit f2079fb
Show file tree

Hide file tree

Showing 132 changed files with 5,018 additions and 2,327 deletions.
diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml
@@ -3,64 +3,6 @@ name: fork-pr-integration-tests-aws
 on: [pull_request]
 
 jobs:
-  build-docker-image:
-    if: github.repository == 'your github repo' # swap here with your project id
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          # pull_request_target runs the workflow in the context of the base repo
-          # as such actions/checkout needs to be explicit configured to retrieve
-          # code from the PR.
-          ref: refs/pull/${{ github.event.pull_request.number }}/merge
-          submodules: recursive
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          install: true
-      - name: Set up AWS SDK
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-west-2
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
-      - name: Set ECR image tag
-        id: image-tag
-        run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
-      - name: Cache Public ECR Image
-        id: lambda_python_3_11
-        uses: actions/cache@v2
-        with:
-          path: ~/cache
-          key: lambda_python_3_11
-      - name: Handle Cache Miss (pull public ECR image & save it to tar file)
-        if: steps.cache-primes.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p ~/cache
-          docker pull public.ecr.aws/lambda/python:3.11
-          docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
-      - name: Handle Cache Hit (load docker image from tar file)
-        if: steps.cache-primes.outputs.cache-hit == 'true'
-        run: |
-          docker load -i ~/cache/lambda_python_3_11.tar
-      - name: Build and push
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: feast-python-server
-        run: |
-          docker build \
-            --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
-            --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
-            --load \
-            .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
-    outputs:
-      DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
   integration-test-python:
     if: github.repository == 'your github repo' # swap here with your project id
     runs-on: ${{ matrix.os }}
@@ -138,8 +80,6 @@ jobs:
           docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
       - name: Test python
         if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
-        env:
-          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
         run: |
           pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread -k "aws and not Snowflake and not BigQuery and not minio_registry"
           pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread -k "File and not Snowflake and not BigQuery and not minio_registry"

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -116,7 +116,7 @@ jobs:
     needs: get-version
     strategy:
       matrix:
-        component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server]
+        component: [feature-server, feature-server-java, feature-transformation-server]
     env:
       REGISTRY: feastdev
     steps:

diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml
@@ -6,61 +6,8 @@ on:
       - master
 
 jobs:
-  build-lambda-docker-image:
-    if: github.repository == 'feast-dev/feast'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          install: true
-      - name: Set up AWS SDK
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-west-2
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
-      - name: Set ECR image tag
-        id: image-tag
-        run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
-      - name: Cache Public ECR Image
-        id: lambda_python_3_11
-        uses: actions/cache@v2
-        with:
-          path: ~/cache
-          key: lambda_python_3_11
-      - name: Handle Cache Miss (pull public ECR image & save it to tar file)
-        if: steps.cache-primes.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p ~/cache
-          docker pull public.ecr.aws/lambda/python:3.11
-          docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
-      - name: Handle Cache Hit (load docker image from tar file)
-        if: steps.cache-primes.outputs.cache-hit == 'true'
-        run: |
-          docker load -i ~/cache/lambda_python_3_11.tar
-      - name: Build and push
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: feast-python-server
-        run: |
-          docker build \
-            --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
-            --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
-            --load \
-            .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
-    outputs:
-      DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
   integration-test-python:
     if: github.repository == 'feast-dev/feast'
-    needs: build-lambda-docker-image
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -125,7 +72,6 @@ jobs:
           docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
       - name: Test python and go
         env:
-          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
           SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
           SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
           SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
@@ -134,7 +80,6 @@ jobs:
         run: make test-python-integration
       - name: Benchmark python
         env:
-          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
           SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
           SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
           SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}

diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
@@ -61,65 +61,9 @@ jobs:
         run: gcloud info
       - name: Run DynamoDB / Bigtable cleanup script
         run: python infra/scripts/cleanup_ci.py
-  build-docker-image:
-    if: github.repository == 'feast-dev/feast'
-    needs: [check_date]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: master
-          submodules: recursive
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          install: true
-      - name: Set up AWS SDK
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-west-2
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
-      - name: Set ECR image tag
-        id: image-tag
-        run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
-      - name: Cache Public ECR Image
-        id: lambda_python_3_11
-        uses: actions/cache@v4
-        with:
-          path: ~/cache
-          key: lambda_python_3_11
-      - name: Handle Cache Miss (pull public ECR image & save it to tar file)
-        if: steps.lambda_python_3_11.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p ~/cache
-          docker pull public.ecr.aws/lambda/python:3.11
-          docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
-      - name: Handle Cache Hit (load docker image from tar file)
-        if: steps.lambda_python_3_11.outputs.cache-hit == 'true'
-        run: |
-          docker load -i ~/cache/lambda_python_3_11.tar
-      - name: Build and push
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: feast-python-server
-        run: |
-          docker build \
-            --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
-            --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
-            --load \
-            .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
-    outputs:
-      DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
   integration-test-python:
     if: github.repository == 'feast-dev/feast'
-    needs: [check_date, build-docker-image, cleanup_dynamo_tables]
+    needs: [check_date, cleanup_dynamo_tables]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -205,7 +149,6 @@ jobs:
       - name: Test python
         if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
         env:
-          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
           SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
           SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
           SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}

diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -13,75 +13,12 @@ on:
 #  cancel-in-progress: true
 
 jobs:
-  build-docker-image:
-    # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
-    if:
-      ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
-      (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
-      github.repository == 'feast-dev/feast'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          # pull_request_target runs the workflow in the context of the base repo
-          # as such actions/checkout needs to be explicit configured to retrieve
-          # code from the PR.
-          ref: refs/pull/${{ github.event.pull_request.number }}/merge
-          submodules: recursive
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          install: true
-      - name: Set up AWS SDK
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-west-2
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
-      - name: Set ECR image tag
-        id: image-tag
-        run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`"
-      - name: Cache Public ECR Image
-        id: lambda_python_3_11
-        uses: actions/cache@v2
-        with:
-          path: ~/cache
-          key: lambda_python_3_11
-      - name: Handle Cache Miss (pull public ECR image & save it to tar file)
-        if: steps.cache-primes.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p ~/cache
-          docker pull public.ecr.aws/lambda/python:3.11
-          docker save public.ecr.aws/lambda/python:3.11 -o ~/cache/lambda_python_3_11.tar
-      - name: Handle Cache Hit (load docker image from tar file)
-        if: steps.cache-primes.outputs.cache-hit == 'true'
-        run: |
-          docker load -i ~/cache/lambda_python_3_11.tar
-      - name: Build and push
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: feast-python-server
-        run: |
-          docker build \
-            --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
-            --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
-            --load \
-            .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
-    outputs:
-      DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
   integration-test-python:
     # when using pull_request_target, all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
     if:
       ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
       (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
       github.repository == 'feast-dev/feast'
-    needs: build-docker-image
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -49,7 +49,7 @@ jobs:
     needs: [get-version, publish-python-sdk]
     strategy:
       matrix:
-        component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server, feast-operator]
+        component: [feature-server, feature-server-java, feature-transformation-server, feast-operator]
     env:
       MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
       REGISTRY: feastdev

diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile
@@ -0,0 +1,8 @@
+FROM gitpod/workspace-base
+RUN sudo apt-get update && sudo apt-get install -y python3-dev python3-setuptools python3-pip python-is-python3 && sudo rm -rf /var/lib/apt/lists/*
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+RUN curl -fsSL https://pixi.sh/install.sh | bash
+ENV PATH=$PATH:/home/gitpod/.cargo/bin
+RUN curl -s "https://get.sdkman.io" | bash
+SHELL ["/bin/bash", "-c"]
+RUN source "/home/gitpod/.sdkman/bin/sdkman-init.sh" && sdk install java 14.0.2-zulu
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -1,42 +1,20 @@
 #  https://www.gitpod.io/docs/config-gitpod-file
+image:
+    file: .gitpod.Dockerfile
+
 tasks:
   - init: |
-      python -m venv venv
-      source venv/bin/activate
-
-      pip install pre-commit
+      uv venv
+      uv pip install pre-commit
       pre-commit install --hook-type pre-commit --hook-type pre-push
-      pip install '.[dev]'
-      make compile-protos-python
-      make compile-protos-go
-      make compile-go-lib
-    env:
-      PYTHONUSERBASE: "/workspace/.pip-modules"
-    command: |
-      source venv/bin/activate
-
-      git config --global alias.ci 'commit -s'
-      git config --global alias.sw switch
-      git config --global alias.st status
-      git config --global alias.co checkout
-      git config --global alias.br branch
-      git config --global alias.df diff
-github:
-  prebuilds:
-    # enable for the default branch (defaults to true)
-    master: true
-    # enable for all branches in this repo (defaults to false)
-    branches: false
-    # enable for pull requests coming from this repo (defaults to true)
-    pullRequests: true
-    # enable for pull requests coming from forks (defaults to false)
-    pullRequestsFromForks: false
-    # add a check to pull requests (defaults to true)
-    addCheck: true
-    # add a "Review in Gitpod" button as a comment to pull requests (defaults to false)
-    addComment: false
-    # add a "Review in Gitpod" button to the pull request's description (defaults to false)
-    addBadge: false
+      source .venv/bin/activate
+      export PYTHON=3.10 && make install-python-ci-dependencies-uv-venv
+    #   git config --global alias.ci 'commit -s'
+    #   git config --global alias.sw switch
+    #   git config --global alias.st status
+    #   git config --global alias.co checkout
+    #   git config --global alias.br branch
+    #   git config --global alias.df diff
 
 vscode:
   extensions: