Migrate sparkop e2e tests to kf-feast cluster (#62)

* Migrate sparkop e2e tests to kf-feast cluster Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Run tests within pod Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Add resource to test spark job Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Update to spark 3.0.2 Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Add java options to enable netty reflection Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> Co-authored-by: Khor Shu Heng <khor.heng@gojek.com>
feast-dev · Apr 28, 2021 · e543bfb · e543bfb
1 parent b8f0313
commit e543bfb
Show file tree

Hide file tree

Showing 5 changed files with 136 additions and 51 deletions.
diff --git a/.prow.yaml b/.prow.yaml
@@ -66,9 +66,11 @@ presubmits:
   always_run: true
   max_concurrency: 1
   spec:
+    metadata:
+      namespace: sparkop-e2e
     containers:
       - image: gcr.io/kf-feast/feast-ci:latest
-        command: [ "infra/scripts/aws-runner.sh", "--location-from-prow", "--project-name", "feast-ci-sparkop-project" ]
+        command: [ "infra/scripts/test-end-to-end-sparkop.sh"]
         resources:
           requests:
             cpu: "2"
@@ -78,18 +80,6 @@ presubmits:
             value: /etc/gcloud/service-account.json
           - name: DOCKER_REPOSITORY
             value: gcr.io/kf-feast
-          - name: AWS_ACCESS_KEY_ID
-            valueFrom:
-              secretKeyRef:
-                name: feast-aws-creds
-                key: AWS_ACCESS_KEY_ID
-          - name: AWS_SECRET_ACCESS_KEY
-            valueFrom:
-              secretKeyRef:
-                name: feast-aws-creds
-                key: AWS_SECRET_ACCESS_KEY
-          - name: AWS_DEFAULT_REGION
-            value: us-west-2
         volumeMounts:
           - mountPath: /etc/gcloud/service-account.json
             name: service-account
@@ -235,9 +225,11 @@ postsubmits:
   branches:
     - ^master$
   spec:
+    metadata:
+      namespace: sparkop-e2e
     containers:
       - image: gcr.io/kf-feast/feast-ci:latest
-        command: [ "infra/scripts/aws-runner.sh", "--location-from-prow", "--project-name", "feast-ci-sparkop-project" ]
+        command: [ "infra/scripts/test-end-to-end-sparkop.sh"]
         resources:
           requests:
             cpu: "2"
@@ -247,18 +239,6 @@ postsubmits:
             value: /etc/gcloud/service-account.json
           - name: DOCKER_REPOSITORY
             value: gcr.io/kf-feast
-          - name: AWS_ACCESS_KEY_ID
-            valueFrom:
-              secretKeyRef:
-                name: feast-aws-creds
-                key: AWS_ACCESS_KEY_ID
-          - name: AWS_SECRET_ACCESS_KEY
-            valueFrom:
-              secretKeyRef:
-                name: feast-aws-creds
-                key: AWS_SECRET_ACCESS_KEY
-          - name: AWS_DEFAULT_REGION
-            value: us-west-2
         volumeMounts:
           - mountPath: /etc/gcloud/service-account.json
             name: service-account

diff --git a/infra/scripts/helm/k8s-jobservice.tpl.yaml b/infra/scripts/helm/k8s-jobservice.tpl.yaml
@@ -0,0 +1,54 @@
+feast-jobservice:
+  envOverrides:
+    FEAST_CORE_URL: feast-release-feast-core:6565
+    FEAST_SPARK_LAUNCHER: k8s
+    FEAST_SPARK_K8S_NAMESPACE: sparkop-e2e
+    FEAST_SPARK_K8S_USE_INCLUSTER_CONFIG: True
+    FEAST_TELEMETRY: False
+    FEAST_SPARK_STAGING_LOCATION: gs://feast-templocation-kf-feast
+    FEAST_REDIS_HOST: feast-release-redis-master
+    FEAST_REDIS_PORT: 6379
+    FEAST_JOB_SERVICE_ENABLE_CONTROL_LOOP: False
+    FEAST_SPARK_INGESTION_JAR: local:///opt/spark/jars/feast-ingestion-spark-${IMAGE_TAG}.jar
+
+  sparkOperator:
+    enabled: true
+    jobTemplate:
+      apiVersion: "sparkoperator.k8s.io/v1beta2"
+      kind: SparkApplication
+      spec:
+        type: Scala
+        mode: cluster
+        image: "gcr.io/kf-feast/feast-spark:${IMAGE_TAG}"
+        hadoopConf:
+          "fs.gs.project.id": "kf-feast"
+          "google.cloud.auth.service.account.enable": "true"
+          "google.cloud.auth.service.account.json.keyfile": "/mnt/secrets/credentials.json"
+        sparkVersion: "3.0.2"
+        timeToLiveSeconds: 3600
+        pythonVersion: "3"
+        restartPolicy:
+          type: Never
+        driver:
+          cores: 1
+          coreLimit: "1200m"
+          memory: "600m"
+          labels:
+            version: 3.0.2
+          javaOptions: "-Dio.netty.tryReflectionSetAccessible=true"
+          secrets:
+            - name: feast-gcp-service-account
+              path: /mnt/secrets
+              secretType: GCPServiceAccount
+        executor:
+          cores: 1
+          instances: 1
+          memory: "800m"
+          labels:
+            version: 3.0.2
+          javaOptions: "-Dio.netty.tryReflectionSetAccessible=true"
+          secrets:
+            - name: feast-gcp-service-account
+              path: /mnt/secrets
+              secretType: GCPServiceAccount
+
diff --git a/infra/scripts/k8s-common-functions.sh b/infra/scripts/k8s-common-functions.sh
@@ -91,13 +91,18 @@ function helm_install {
           exit 1
     fi
 
+    if [ -z ${JOBSERVICE_HELM_VALUES:-} ]; then
+      export HELM_ARGS="--set feast-jobservice.image.repository=${DOCKER_REPOSITORY}/feast-jobservice \
+        --set feast-jobservice.image.tag=${JOBSERVICE_GIT_TAG:-$GIT_TAG} \
+        --set prometheus-statsd-exporter.enabled=false $@";
+    else
+      export HELM_ARGS="--values ${JOBSERVICE_HELM_VALUES}";
+    fi
+
     if ! time helm install --wait "$RELEASE" "${HELM_CHART_LOCATION:-./infra/charts/feast-spark}" \
         --timeout 5m \
-        --set "feast-jobservice.image.repository=${DOCKER_REPOSITORY}/feast-jobservice" \
-        --set "feast-jobservice.image.tag=${JOBSERVICE_GIT_TAG:-$GIT_TAG}" \
-        --set "prometheus-statsd-exporter.enabled=false" \
         --namespace "$NAMESPACE" \
-        "$@" ; then
+        ${HELM_ARGS}; then
 
         echo "Error during helm install. "
         kubectl -n "$NAMESPACE" get pods
@@ -142,4 +147,4 @@ subjects:
   - kind: ServiceAccount
     name: default
 EOF
-}
+}
diff --git a/infra/scripts/test-end-to-end-sparkop.sh b/infra/scripts/test-end-to-end-sparkop.sh
@@ -1,21 +1,69 @@
 #!/usr/bin/env bash
 
-set -euo pipefail
+set -e
 
-pip install "s3fs" "aiobotocore==1.2.2" "boto3" "urllib3>=1.25.4"
+export GIT_TAG=${PULL_PULL_SHA:-${PULL_BASE_SHA}}
+export GIT_REMOTE_URL=https://github.com/feast-dev/feast-spark.git
+export DOCKER_REPOSITORY=gcr.io/kf-feast
+export JOBSERVICE_HELM_VALUES=infra/scripts/helm/k8s-jobservice.yaml
 
-export DISABLE_FEAST_SERVICE_FIXTURES=1
-export DISABLE_SERVICE_FIXTURES=1
-export FEAST_TELEMETRY="False"
+test -z ${GCLOUD_PROJECT} && GCLOUD_PROJECT="kf-feast"
+test -z ${GCLOUD_REGION} && GCLOUD_REGION="us-central1"
+test -z ${GCLOUD_NETWORK} && GCLOUD_NETWORK="default"
+test -z ${GCLOUD_SUBNET} && GCLOUD_SUBNET="default"
+test -z ${KUBE_CLUSTER} && KUBE_CLUSTER="feast-e2e-dataflow"
 
-export FEAST_SPARK_K8S_NAMESPACE=sparkop
+gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS}
+gcloud -q auth configure-docker
 
-PYTHONPATH=sdk/python pytest tests/e2e/ \
-      --feast-version develop \
-      --core-url feast-release-feast-core:6565 \
-      --serving-url feast-release-feast-online-serving:6566 \
-      --env k8s \
-      --staging-path $STAGING_PATH \
-      --redis-url feast-release-redis-master.sparkop.svc.cluster.local:6379 \
-      --kafka-brokers feast-release-kafka.sparkop.svc.cluster.local:9092 \
-      -m "not bq"
+gcloud config set project ${GCLOUD_PROJECT}
+gcloud config set compute/region ${GCLOUD_REGION}
+gcloud config list
+
+gcloud container clusters get-credentials ${KUBE_CLUSTER} --region ${GCLOUD_REGION} --project ${GCLOUD_PROJECT}
+
+source infra/scripts/k8s-common-functions.sh
+
+NAMESPACE="sparkop-e2e"
+
+k8s_cleanup "feast-release" "$NAMESPACE"
+k8s_cleanup "js" "$NAMESPACE"
+
+kubectl delete sparkapplication --all -n $NAMESPACE
+kubectl delete scheduledsparkapplication --all -n $NAMESPACE
+
+wait_for_image "${DOCKER_REPOSITORY}" feast-jobservice "${GIT_TAG}"
+wait_for_image "${DOCKER_REPOSITORY}" feast-spark "${GIT_TAG}"
+
+sed s/\$\{IMAGE_TAG\}/${JOBSERVICE_GIT_TAG:-$GIT_TAG}/g infra/scripts/helm/k8s-jobservice.tpl.yaml > $JOBSERVICE_HELM_VALUES
+
+helm_install "js" "${DOCKER_REPOSITORY}" "${GIT_TAG}" "$NAMESPACE" \
+  --set 'feast-online-serving.application-override\.yaml.feast.stores[0].type=REDIS' \
+  --set 'feast-online-serving.application-override\.yaml.feast.stores[0].name=online' \
+  --set 'feast-online-serving.application-override\.yaml.feast.stores[0].config.host=feast-release-redis-master' \
+  --set 'feast-online-serving.application-override\.yaml.feast.stores[0].config.port=6379' \
+
+CMD=$(printf '%s' \
+  "mkdir src && cd src && git clone --recursive ${GIT_REMOTE_URL} && cd feast-spark && " \
+  "git config remote.origin.fetch '+refs/pull/*:refs/remotes/origin/pull/*' && " \
+  "git fetch -q && git checkout ${GIT_TAG} && " \
+  "make install-python && " \
+  "python -m pip install -qr tests/requirements.txt && " \
+  "pytest -v tests/e2e/ --env k8s " \
+  "--staging-path gs://feast-templocation-kf-feast/ " \
+  "--core-url feast-release-feast-core:6565 " \
+  "--serving-url feast-release-feast-online-serving:6566 " \
+  "--job-service-url js-feast-jobservice:6568 " \
+  "--kafka-brokers feast-release-kafka-headless:9092 --bq-project kf-feast --feast-version dev")
+
+# Delete old test running pod if it exists
+kubectl delete pod -n "$NAMESPACE" ci-test-runner 2>/dev/null || true
+
+kubectl run -n "$NAMESPACE" -i ci-test-runner  \
+    --pod-running-timeout=5m \
+    --restart=Never \
+    --image="${DOCKER_REPOSITORY}/feast-ci:latest" \
+    --env="FEAST_TELEMETRY=false" \
+    --env="DISABLE_FEAST_SERVICE_FIXTURES=1" \
+    --env="DISABLE_SERVICE_FIXTURES=1" \
+    -- bash -c "$CMD"
diff --git a/tests/e2e/fixtures/client.py b/tests/e2e/fixtures/client.py
@@ -95,15 +95,12 @@ def feast_client(
         return Client(
             core_url=f"{feast_core[0]}:{feast_core[1]}",
             serving_url=f"{feast_serving[0]}:{feast_serving[1]}",
-            spark_launcher="k8s",
-            spark_staging_location=os.path.join(local_staging_path, "k8s"),
-            spark_ingestion_jar=ingestion_job_jar,
-            redis_host=pytestconfig.getoption("redis_url").split(":")[0],
-            redis_port=pytestconfig.getoption("redis_url").split(":")[1],
             historical_feature_output_location=os.path.join(
                 local_staging_path, "historical_output"
             ),
+            spark_staging_location=os.path.join(local_staging_path, "k8s"),
             enable_auth=pytestconfig.getoption("enable_auth"),
+            **job_service_env,
         )
     else:
         raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}")
@@ -185,6 +182,7 @@ def tfrecord_feast_client(
             historical_feature_output_location=os.path.join(
                 local_staging_path, "historical_output"
             ),
+            **job_service_env,
         )
     else:
         raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}")