From 7a78959bbee1e0cc7a04e025efc92de1ed7ba898 Mon Sep 17 00:00:00 2001 From: Rintaro Okamura Date: Wed, 20 Jan 2021 14:56:57 +0900 Subject: [PATCH] E2E deploy test: rewrite in go tests (#814) * :sparkles: add tests/e2e package Signed-off-by: Rintaro Okamura * :sparkles: use stream apis Signed-off-by: Rintaro Okamura * :sparkles: add GetObject & SearchByID Signed-off-by: Rintaro Okamura * :recycle: improve validation Signed-off-by: Rintaro Okamura * :green_heart: update e2e-deploy Signed-off-by: Rintaro Okamura * :green_heart: add libhdf5 for e2e-deploy test Signed-off-by: Rintaro Okamura * :green_heart: fix vald client Signed-off-by: Rintaro Okamura * :green_heart: fix NewPortforward interface Signed-off-by: Rintaro Okamura * :fire: commented out logging search results Signed-off-by: Rintaro Okamura * :white_check_mark: improve e2e test Signed-off-by: Rintaro Okamura * :bento: update values-ci manifest Signed-off-by: Rintaro Okamura * :green_heart: simplify Signed-off-by: Rintaro Okamura * :wrench: add config Signed-off-by: Rintaro Okamura * :wrench: inc replicas Signed-off-by: Rintaro Okamura * :green_heart: fix loop Signed-off-by: Rintaro Okamura * :loud_sound: add log Signed-off-by: Rintaro Okamura * :mag: improve logs Signed-off-by: Rintaro Okamura * :wrench: add keepalive Signed-off-by: Rintaro Okamura * :green_heart: add Cassandra E2E test Signed-off-by: Rintaro Okamura * :wrench: revise config Signed-off-by: Rintaro Okamura * :wrench: revise config Signed-off-by: Rintaro Okamura * :wrench: cassandra consistency = one Signed-off-by: Rintaro Okamura * :green_heart: trigger e2e deploy test using chatops Signed-off-by: Rintaro Okamura * :white_check_mark: use correct version of containers Signed-off-by: Rintaro Okamura * :loud_sound: print HELM_EXTRA_OPTIONS Signed-off-by: Rintaro Okamura * :green_heart: remove redundant condition checks Signed-off-by: Rintaro Okamura * :arrow_up: upgrade setup-k3d Signed-off-by: Rintaro Okamura * :white_check_mark: just a workaround for strange mysql table definition Signed-off-by: Rintaro Okamura * :truck: move values into .github/helm/values Signed-off-by: Rintaro Okamura * :white_check_mark: add sidecar test Signed-off-by: Rintaro Okamura * :wrench: fix Signed-off-by: Rintaro Okamura * :wrench: fix config for agent-sidecar Signed-off-by: Rintaro Okamura * :wrench: reduce number of data Signed-off-by: Rintaro Okamura * :wrench: add createindex phase Signed-off-by: Rintaro Okamura * :wrench: remove useless config Signed-off-by: Rintaro Okamura * :green_heart: remove useless lines Signed-off-by: Rintaro Okamura * :wrench: fix sidecar config Signed-off-by: Rintaro Okamura * :white_check_mark: add tests for mysql deleteMeta Signed-off-by: Rintaro Okamura * :wrench: Add options for which data is used for test Signed-off-by: Rintaro Okamura * :art: format yamls Signed-off-by: Rintaro Okamura * :page_facing_up: Update license header Signed-off-by: Rintaro Okamura * :arrow_up: Upgrade to v1 APIs Signed-off-by: Rintaro Okamura * :wrench: Fix Helm values for E2E test Signed-off-by: Rintaro Okamura * :green_heart: Fix method call Signed-off-by: Rintaro Okamura * :wrench: Reduce number of data Signed-off-by: Rintaro Okamura * :green_heart: Fix to use v1 vald API Signed-off-by: Rintaro Okamura * :wrench: Reduce number of Scylla cluster member Signed-off-by: Rintaro Okamura * :children_crossing: Add a variable for scylla deploy task Signed-off-by: Rintaro Okamura * :wrench: Add minio deploy task Signed-off-by: Rintaro Okamura * :recycle: Use kubectl wait instead for sh loop Signed-off-by: Rintaro Okamura :pencil2: Fix typo Signed-off-by: Rintaro Okamura * :heavy_plus_sign: Add cli-runtime to go.mod.default / Remove duplicated client-go entry Signed-off-by: Rintaro Okamura * :recycle: use assoc list for checking pr-xxx image existences Signed-off-by: Rintaro Okamura * :arrow_up: Update go.sum Signed-off-by: Rintaro Okamura * :green_heart: Fix build Signed-off-by: Rintaro Okamura * :wrench: tuning: add sleep Signed-off-by: Rintaro Okamura * :wrench: Tuning number of requests Signed-off-by: Rintaro Okamura * :green_heart: Fix e2e tests code Signed-off-by: Rintaro Okamura * :white_check_mark: Update e2e tests Signed-off-by: Rintaro Okamura --- .github/chatops_commands.md | 1 + .github/helm/values/values-agent-sidecar.yaml | 89 ++ .../helm/values/values-redis-mysql.yaml | 39 +- .github/helm/values/values-scylla.yaml | 152 ++++ .github/workflows/e2e-deploy.yml | 406 ++++++++-- Makefile | 4 + Makefile.d/k8s.mk | 19 +- charts/vald/values-scylla.yaml | 8 - cmd/manager/backup/cassandra/sample.yaml | 2 - cmd/meta/cassandra/sample.yaml | 2 - docs/tutorial/get-started.md | 2 - example/helm/values-scylla.yaml | 8 - go.mod | 2 + go.sum | 33 +- hack/go.mod.default | 1 + internal/db/rdb/mysql/mysql.go | 24 +- internal/db/rdb/mysql/mysql_test.go | 266 +++++- k8s/external/minio/deployment.yaml | 58 ++ k8s/external/minio/mb-job.yaml | 41 + k8s/external/minio/svc.yaml | 25 + k8s/external/scylla/scyllacluster.yaml | 2 +- tests/e2e/crud_test.go | 758 ++++++++++++++++++ tests/e2e/portforward/portforward.go | 130 +++ tests/e2e/sidecar_test.go | 455 +++++++++++ 24 files changed, 2407 insertions(+), 120 deletions(-) create mode 100644 .github/helm/values/values-agent-sidecar.yaml rename charts/vald/values-ci.yaml => .github/helm/values/values-redis-mysql.yaml (75%) create mode 100644 .github/helm/values/values-scylla.yaml create mode 100644 k8s/external/minio/deployment.yaml create mode 100644 k8s/external/minio/mb-job.yaml create mode 100644 k8s/external/minio/svc.yaml create mode 100644 tests/e2e/crud_test.go create mode 100644 tests/e2e/portforward/portforward.go create mode 100644 tests/e2e/sidecar_test.go diff --git a/.github/chatops_commands.md b/.github/chatops_commands.md index d9f0408676..08db1aee1f 100644 --- a/.github/chatops_commands.md +++ b/.github/chatops_commands.md @@ -4,3 +4,4 @@ - :white_check_mark: `/gen-test` - generate test codes - :label: `/label` - add labels - :rewind: `/rebase` - rebase master +- :end: :two: :end: `/label actions/e2e-deploy` - run E2E deploy & integration test diff --git a/.github/helm/values/values-agent-sidecar.yaml b/.github/helm/values/values-agent-sidecar.yaml new file mode 100644 index 0000000000..060b73a205 --- /dev/null +++ b/.github/helm/values/values-agent-sidecar.yaml @@ -0,0 +1,89 @@ +# +# Copyright (C) 2019-2021 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +defaults: + logging: + level: info + +agent: + minReplicas: 1 + podManagementPolicy: Parallel + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + volumes: + - name: ngt-index + emptyDir: {} + volumeMounts: + - name: ngt-index + mountPath: /var/ngt + ngt: + auto_index_duration_limit: 3m + auto_index_check_duration: 1m + auto_index_length: 1000 + dimension: 784 + index_path: /var/ngt/index + enable_in_memory_mode: false + sidecar: + enabled: true + initContainerEnabled: true + env: + - name: AWS_ACCESS_KEY + value: ACCESSKEY + - name: AWS_SECRET_ACCESS_KEY + value: SECRETKEY + resources: + requests: + cpu: 100m + memory: 100Mi + config: + filename: vald-agent-ngt-index + blob_storage: + storage_type: "s3" + bucket: "vald-minio" + s3: + endpoint: "http://minio.default.svc.cluster.local:9000" + region: "us-east-1" + force_path_style: true + +gateway: + vald: + enabled: false + lb: + enabled: false + backup: + enabled: false + meta: + enabled: false + +discoverer: + enabled: false + +manager: + compressor: + enabled: false + + backup: + enabled: false + + index: + enabled: false + +meta: + enabled: false diff --git a/charts/vald/values-ci.yaml b/.github/helm/values/values-redis-mysql.yaml similarity index 75% rename from charts/vald/values-ci.yaml rename to .github/helm/values/values-redis-mysql.yaml index 21f6103670..6304e7cfec 100644 --- a/charts/vald/values-ci.yaml +++ b/.github/helm/values/values-redis-mysql.yaml @@ -14,8 +14,15 @@ # limitations under the License. # +defaults: + logging: + level: info + gateway: vald: + enabled: false + lb: + enabled: true minReplicas: 1 hpa: enabled: false @@ -25,6 +32,24 @@ gateway: memory: 50Mi gateway_config: index_replica: 3 + backup: + enabled: true + minReplicas: 1 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + meta: + enabled: true + minReplicas: 1 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi agent: minReplicas: 3 @@ -37,10 +62,10 @@ agent: cpu: 100m memory: 50Mi ngt: - auto_index_duration_limit: 60s - auto_index_check_duration: 5s - auto_index_length: 10 - dimension: 6 + auto_index_duration_limit: 3m + auto_index_check_duration: 1m + auto_index_length: 1000 + dimension: 784 discoverer: minReplicas: 1 @@ -53,7 +78,7 @@ discoverer: manager: compressor: - minReplicas: 1 + minReplicas: 3 hpa: enabled: false resources: @@ -64,7 +89,7 @@ manager: compress_algorithm: gob backup: - minReplicas: 1 + minReplicas: 3 hpa: enabled: false resources: @@ -80,7 +105,7 @@ manager: memory: 30Mi meta: - minReplicas: 1 + minReplicas: 3 hpa: enabled: false resources: diff --git a/.github/helm/values/values-scylla.yaml b/.github/helm/values/values-scylla.yaml new file mode 100644 index 0000000000..311c1fa228 --- /dev/null +++ b/.github/helm/values/values-scylla.yaml @@ -0,0 +1,152 @@ +# +# Copyright (C) 2019-2021 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +defaults: + logging: + level: info + +gateway: + vald: + enabled: false + lb: + enabled: true + minReplicas: 1 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + gateway_config: + index_replica: 3 + backup: + enabled: true + minReplicas: 1 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + meta: + enabled: true + minReplicas: 1 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + +agent: + minReplicas: 3 + maxReplicas: 10 + podManagementPolicy: Parallel + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + ngt: + auto_index_duration_limit: 3m + auto_index_check_duration: 1m + auto_index_length: 1000 + dimension: 784 + +discoverer: + minReplicas: 1 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + +manager: + compressor: + minReplicas: 3 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + compress: + compress_algorithm: gob + + backup: + minReplicas: 3 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 30Mi + image: + repository: vdaas/vald-manager-backup-cassandra + initContainers: + - type: wait-for-cassandra + name: wait-for-scylla + image: cassandra:latest + cassandra: + hosts: + - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local + sleepDuration: 2 + env: [] + mysql: + enabled: false + cassandra: + enabled: true + config: + hosts: + - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local + consistency: one + + index: + replicas: 1 + resources: + requests: + cpu: 100m + memory: 30Mi + +meta: + minReplicas: 3 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 30Mi + image: + repository: vdaas/vald-meta-cassandra + initContainers: + - type: wait-for-cassandra + name: wait-for-scylla + image: cassandra:latest + cassandra: + hosts: + - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local + sleepDuration: 2 + env: [] + redis: + enabled: false + cassandra: + enabled: true + config: + hosts: + - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local + consistency: one diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml index ec7ea0a9a6..2c752a3928 100644 --- a/.github/workflows/e2e-deploy.yml +++ b/.github/workflows/e2e-deploy.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Run e2e deploy test" +name: "Run E2E deploy and integration test" on: push: tags: @@ -21,15 +21,20 @@ on: - "v*.*.*" - "*.*.*-*" - "v*.*.*-*" + pull_request: + types: + - "labeled" jobs: - e2e-deploy: - name: e2e deploy test + e2e-deploy-redis-mysql: + name: "E2E deploy test (Redis, MySQL)" runs-on: ubuntu-latest timeout-minutes: 45 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' steps: - uses: actions/checkout@v2 - name: wait for dockers + if: startsWith( github.ref, 'refs/tags/') run: | tag=$(cat versions/VALD_VERSION) for image in \ @@ -38,113 +43,378 @@ jobs: vdaas/vald-manager-compressor \ vdaas/vald-meta-redis \ vdaas/vald-manager-backup-mysql \ - vdaas/vald-gateway \ + vdaas/vald-backup-gateway \ + vdaas/vald-lb-gateway \ + vdaas/vald-meta-gateway \ vdaas/vald-manager-index do echo "searching ${image}:${tag}" - until curl -s "https://registry.hub.docker.com/v2/repositories/${image}/tags" | jq '.results[].name' | grep "${tag}"; do + until curl -s "https://registry.hub.docker.com/v2/repositories/${image}/tags/${tag}" | jq '.name' | grep -v "null"; do echo "waiting for ${image}:${tag} to be uploaded..." sleep 2 done done - - name: Fetch golang version + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + + declare -A images=( + ["vdaas/vald-agent-ngt"]="agent.image.tag" + ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag" + ["vdaas/vald-manager-compressor"]="manager.compressor.image.tag" + ["vdaas/vald-meta-redis"]="meta.image.tag" + ["vdaas/vald-manager-backup-mysql"]="manager.backup.image.tag" + ["vdaas/vald-backup-gateway"]="gateway.backup.image.tag" + ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag" + ["vdaas/vald-meta-gateway"]="gateway.meta.image.tag" + ["vdaas/vald-manager-index"]="manager.index.image.tag" + ) + + for image in "${!images[@]}" + do + echo "check for ${image}" + + if curl -s "https://registry.hub.docker.com/v2/repositories/${image}/tags/pr-${pr_num}" | jq '.name' | grep -v "null"; then + echo "${image}:pr-${pr_num} exists. adding a helm option '--set ${images[${image}]}=pr-${pr_num}'." + export HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS} --set ${images[${image}]}=pr-${pr_num}" + fi + done + + echo "HELM_EXTRA_OPTIONS=${HELM_EXTRA_OPTIONS}" + echo "HELM_EXTRA_OPTIONS=${HELM_EXTRA_OPTIONS}" >> $GITHUB_ENV + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libhdf5-dev + - name: Fetch Helm version run: | - KIND_VERSION=`make version/kind` HELM_VERSION=`make version/helm` - VALDCLI_VERSION=`make version/valdcli` - echo "::set-output name=kind::${KIND_VERSION}" echo "::set-output name=helm::${HELM_VERSION}" - echo "::set-output name=valdcli::${VALDCLI_VERSION}" id: version - - uses: engineerd/setup-kind@v0.2.0 + - uses: rinx/setup-k3d@v0.0.2 with: - version: ${{ steps.version.outputs.kind }} - skipClusterCreation: true + version: latest + name: vald + agents: 3 + - name: check k3d + run: | + kubectl cluster-info - uses: azure/setup-helm@v1 with: version: ${{ steps.version.outputs.helm }} - name: Helm version run: | helm version - - name: start kind - run: | - kind create cluster --name vald - kubectl cluster-info - - name: install valdcli - run: | - curl -OL "https://github.com/vdaas/vald-client-clj/releases/download/${VALDCLI_VERSION}/valdcli-linux-static.zip" - unzip valdcli-linux-static.zip - chmod a+x valdcli - sudo mv valdcli /usr/local/bin/valdcli - env: - VALDCLI_VERSION: ${{ steps.version.outputs.valdcli }} - name: deploy vald run: | sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + make k8s/external/mysql/deploy make k8s/external/redis/deploy - jq_query='.items[] | select( ([ .status.conditions[] | select( .type == "Ready" and .status == "True" ) ] | length ) == 1) | .metadata.namespace + "/" + .metadata.name' - until [ $(kubectl get pod -o json | jq -r "$jq_query" | wc -l) -ge 2 ] + + helm repo add vald https://vald.vdaas.org/charts + tag=$(cat versions/VALD_VERSION) + helm install \ + --values .github/helm/values/values-redis-mysql.yaml \ + --set defaults.image.tag=${tag} \ + ${HELM_EXTRA_OPTIONS} \ + --generate-name charts/vald + + sleep 3 + + kubectl wait --for=condition=ready pod -l app=vald-meta-gateway --timeout=600s + + kubectl get pods + - name: run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${DATASET} + podname=`kubectl get pods --selector=app=vald-meta-gateway | tail -1 | awk '{print $1}'` + go test \ + -v tests/e2e/crud_test.go \ + -tags "e2e" \ + -timeout 15m \ + -host=localhost \ + -port=8081 \ + -dataset=`pwd`/hack/benchmark/assets/dataset/${DATASET} \ + -insert-num=100 \ + -search-num=100 \ + -search-by-id-num=10 \ + -get-object-num=5 \ + -update-num=2 \ + -remove-num=2 \ + -wait-after-insert=2m \ + -portforward \ + -portforward-ns=default \ + -portforward-pod-name=${podname} \ + -portforward-pod-port=8081 \ + -kubeconfig=${KUBECONFIG} + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + e2e-deploy-cassandra: + name: "E2E deploy test (Cassandra)" + runs-on: ubuntu-latest + timeout-minutes: 45 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + steps: + - uses: actions/checkout@v2 + - name: wait for dockers + if: startsWith( github.ref, 'refs/tags/') + run: | + tag=$(cat versions/VALD_VERSION) + for image in \ + vdaas/vald-agent-ngt \ + vdaas/vald-discoverer-k8s \ + vdaas/vald-manager-compressor \ + vdaas/vald-meta-cassandra \ + vdaas/vald-manager-backup-cassandra \ + vdaas/vald-backup-gateway \ + vdaas/vald-lb-gateway \ + vdaas/vald-meta-gateway \ + vdaas/vald-manager-index do - echo "waiting for databases to be ready..." - kubectl get pods - sleep 2 + echo "searching ${image}:${tag}" + until curl -s "https://registry.hub.docker.com/v2/repositories/${image}/tags/${tag}" | jq '.name' | grep -v "null"; do + echo "waiting for ${image}:${tag} to be uploaded..." + sleep 2 + done + done + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + + declare -A images=( + ["vdaas/vald-agent-ngt"]="agent.image.tag" + ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag" + ["vdaas/vald-manager-compressor"]="manager.compressor.image.tag" + ["vdaas/vald-meta-cassandra"]="meta.image.tag" + ["vdaas/vald-manager-backup-cassandra"]="manager.backup.image.tag" + ["vdaas/vald-backup-gateway"]="gateway.backup.image.tag" + ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag" + ["vdaas/vald-meta-gateway"]="gateway.meta.image.tag" + ["vdaas/vald-manager-index"]="manager.index.image.tag" + ) + + for image in "${!images[@]}" + do + echo "check for ${image}" + + if curl -s "https://registry.hub.docker.com/v2/repositories/${image}/tags/pr-${pr_num}" | jq '.name' | grep -v "null"; then + echo "${image}:pr-${pr_num} exists. adding a helm option '--set ${images[${image}]}=pr-${pr_num}'." + export HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS} --set ${images[${image}]}=pr-${pr_num}" + fi done + + echo "HELM_EXTRA_OPTIONS=${HELM_EXTRA_OPTIONS}" + echo "HELM_EXTRA_OPTIONS=${HELM_EXTRA_OPTIONS}" >> $GITHUB_ENV + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libhdf5-dev + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "::set-output name=helm::${HELM_VERSION}" + id: version + - uses: rinx/setup-k3d@v0.0.2 + with: + version: latest + name: vald + agents: 3 + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v1 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy vald + run: | + make k8s/external/scylla/deploy + helm repo add vald https://vald.vdaas.org/charts tag=$(cat versions/VALD_VERSION) helm install \ - --values charts/vald/values-ci.yaml \ + --values .github/helm/values/values-scylla.yaml \ --set defaults.image.tag=${tag} \ - --generate-name vald/vald - until [ $(kubectl get pod -o json | jq -r "$jq_query" | wc -l) -ge 11 ] + ${HELM_EXTRA_OPTIONS} \ + --generate-name charts/vald + + sleep 3 + + kubectl wait --for=condition=ready pod -l app=vald-meta-gateway --timeout=600s + + kubectl get pods + - name: run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${DATASET} + podname=`kubectl get pods --selector=app=vald-meta-gateway | tail -1 | awk '{print $1}'` + go test \ + -v tests/e2e/crud_test.go \ + -tags "e2e" \ + -timeout 15m \ + -host=localhost \ + -port=8081 \ + -dataset=`pwd`/hack/benchmark/assets/dataset/${DATASET} \ + -insert-num=1000 \ + -search-num=1000 \ + -search-by-id-num=10 \ + -get-object-num=10 \ + -update-num=3 \ + -remove-num=2 \ + -wait-after-insert=2m \ + -portforward \ + -portforward-ns=default \ + -portforward-pod-name=${podname} \ + -portforward-pod-port=8081 \ + -kubeconfig=${KUBECONFIG} + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + e2e-deploy-sidecar: + name: "E2E deploy test (Agent & Sidecar)" + runs-on: ubuntu-latest + timeout-minutes: 45 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + steps: + - uses: actions/checkout@v2 + - name: wait for dockers + if: startsWith( github.ref, 'refs/tags/') + run: | + tag=$(cat versions/VALD_VERSION) + for image in \ + vdaas/vald-agent-ngt \ + vdaas/vald-agent-sidecar + do + echo "searching ${image}:${tag}" + until curl -s "https://registry.hub.docker.com/v2/repositories/${image}/tags/${tag}" | jq '.name' | grep -v "null"; do + echo "waiting for ${image}:${tag} to be uploaded..." + sleep 2 + done + done + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + + declare -A images=( + ["vdaas/vald-agent-ngt"]="agent.image.tag" + ["vdaas/vald-agent-sidecar"]="agent.sidecar.image.tag" + ) + + for image in "${!images[@]}" + do + echo "check for ${image}" + + if curl -s "https://registry.hub.docker.com/v2/repositories/${image}/tags/pr-${pr_num}" | jq '.name' | grep -v "null"; then + echo "${image}:pr-${pr_num} exists. adding a helm option '--set ${images[${image}]}=pr-${pr_num}'." + export HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS} --set ${images[${image}]}=pr-${pr_num}" + fi + done + + echo "HELM_EXTRA_OPTIONS=${HELM_EXTRA_OPTIONS}" + echo "HELM_EXTRA_OPTIONS=${HELM_EXTRA_OPTIONS}" >> $GITHUB_ENV + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libhdf5-dev + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "::set-output name=helm::${HELM_VERSION}" + id: version + - uses: rinx/setup-k3d@v0.0.2 + with: + version: latest + name: vald + agents: 3 + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v1 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy vald + run: | + make k8s/external/minio/deploy + + helm repo add vald https://vald.vdaas.org/charts + tag=$(cat versions/VALD_VERSION) + helm install \ + --values .github/helm/values/values-agent-sidecar.yaml \ + --set defaults.image.tag=${tag} \ + ${HELM_EXTRA_OPTIONS} \ + --generate-name charts/vald + + sleep 3 + + kubectl wait --for=condition=ready pod -l app=vald-agent-ngt --timeout=600s + + kubectl get pods + - name: run E2E Agent & Sidecar + run: | + make hack/benchmark/assets/dataset/${DATASET} + podname=`kubectl get pods --selector=app=vald-agent-ngt | tail -1 | awk '{print $1}'` + go test \ + -v tests/e2e/sidecar_test.go \ + -run "TestE2EInsert|TestE2ECreateIndex|TestE2ESearch" \ + -tags "e2e" \ + -host=localhost \ + -port=8081 \ + -dataset=`pwd`/hack/benchmark/assets/dataset/${DATASET} \ + -insert-num=1000 \ + -search-num=1000 \ + -portforward \ + -portforward-ns=default \ + -portforward-pod-name=${podname} \ + -portforward-pod-port=8081 \ + -kubeconfig=${KUBECONFIG} + echo "killing agent pod" + kubectl delete pod vald-agent-ngt-0 + jq_query='.items[] | select( ([ .status.conditions[] | select( .type == "Ready" and .status == "True" ) ] | length ) == 1) | .metadata.namespace + "/" + .metadata.name' + until [ $(kubectl get pod --selector=app=vald-agent-ngt -o json | jq -r "$jq_query" | wc -l) -ge 1 ] do echo "waiting for Vald to be ready..." kubectl get pods sleep 2 done kubectl get pods - - name: insert - run: | - kubectl port-forward deployment/vald-gateway 8081:8081 & - pid=$! - sleep 10 - valdcli rand-vec -d 6 | valdcli -p 8081 insert --elapsed-time abc - valdcli -p 8081 stream-insert --elapsed-time << EOF - [{:vector [0.4554944575653239 0.17698450824379797 0.14510892025549904 0.45742806648293266 0.8255640513082158 0.6104319034657276], :id "d923e43c-7bdc-40fd-9a95-26e54edc54a5"} - {:vector [0.9659463766247516 0.7129174248792229 0.3345289671984051 0.5325195679844225 0.5589141699199695 0.20497376669300038], :id "75666cc5-c5e8-4ba6-a653-7f4f216e2710"}] - EOF - sleep 1 - for i in `seq 1 30` - do - valdcli rand-vec -d 6 | valdcli -p 8081 insert --elapsed-time "v${i}" - sleep 1 - done - kill $pid - - name: search - run: | - kubectl port-forward deployment/vald-gateway 8081:8081 & - pid=$! - sleep 10 - valdcli -p 8081 exists --elapsed-time abc - valdcli -p 8081 get-object --elapsed-time abc - valdcli -p 8081 stream-get-object --elapsed-time '["abc" "d923e43c-7bdc-40fd-9a95-26e54edc54a5" "75666cc5-c5e8-4ba6-a653-7f4f216e2710"]' - valdcli -p 8081 search --elapsed-time '[0.3 0.3 0.3 0.3 0.3 0.3]' - valdcli -p 8081 stream-search --elapsed-time '[[0.3 0.1 0.7 0.3 0.5 0.5] [0.3 0.3 0.4 0.3 0.4 0.4] [0.6 0.1 0.5 0.3 0.4 0.4]]' - valdcli -p 8081 search-by-id --elapsed-time abc - valdcli -p 8081 stream-search-by-id --elapsed-time '["abc" "d923e43c-7bdc-40fd-9a95-26e54edc54a5" "75666cc5-c5e8-4ba6-a653-7f4f216e2710"]' - kill $pid - slack: - name: Slack notification - needs: e2e-deploy + go test \ + -v tests/e2e/sidecar_test.go \ + -run "TestE2ESearch|TestE2EIndexInfo" \ + -tags "e2e" \ + -host=localhost \ + -port=8081 \ + -dataset=`pwd`/hack/benchmark/assets/dataset/${DATASET} \ + -insert-num=1000 \ + -search-num=1000 \ + -portforward \ + -portforward-ns=default \ + -portforward-pod-name=${podname} \ + -portforward-pod-port=8081 \ + -kubeconfig=${KUBECONFIG} + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + slack-notification: + name: "Slack notification" + needs: + - e2e-deploy-redis-mysql + - e2e-deploy-cassandra + - e2e-deploy-sidecar runs-on: ubuntu-latest - if: always() + if: startsWith( github.ref, 'refs/tags/') steps: - uses: technote-space/workflow-conclusion-action@v1 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: 8398a7/action-slack@v2 with: - author_name: e2e-deploy test + author_name: "E2E deploy test" status: ${{ env.WORKFLOW_CONCLUSION }} only_mention_fail: channel env: diff --git a/Makefile b/Makefile index 455335f4a4..daa054038d 100644 --- a/Makefile +++ b/Makefile @@ -177,6 +177,7 @@ GO_SOURCES = $(eval GO_SOURCES := $(shell find \ -not -path './hack/license/*' \ -not -path './hack/swagger/*' \ -not -path './hack/tools/*' \ + -not -path './tests/*' \ -type f \ -name '*.go' \ -not -regex '.*options?\.go' \ @@ -198,6 +199,7 @@ GO_OPTION_SOURCES = $(eval GO_OPTION_SOURCES := $(shell find \ -not -path './hack/license/*' \ -not -path './hack/swagger/*' \ -not -path './hack/tools/*' \ + -not -path './tests/*' \ -type f \ -regex '.*options?\.go' \ -not -name '*_test.go' \ @@ -220,6 +222,8 @@ DISTROLESS_IMAGE ?= gcr.io/distroless/static DISTROLESS_IMAGE_TAG ?= nonroot UPX_OPTIONS ?= -9 +K8S_EXTERNAL_SCYLLA_MANIFEST ?= k8s/external/scylla/scyllacluster.yaml + COMMA := , SHELL = bash diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index 6e9e511da7..64bc323fa3 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -203,6 +203,7 @@ k8s/vald/delete/scylla: \ k8s/external/mysql/deploy: kubectl apply -f k8s/jobs/db/initialize/mysql/configmap.yaml kubectl apply -f k8s/external/mysql + kubectl wait --for=condition=ready pod -l app=mysql --timeout=600s .PHONY: k8s/external/mysql/delete ## delete mysql from k8s @@ -221,6 +222,7 @@ k8s/external/mysql/initialize: ## deploy redis to k8s k8s/external/redis/deploy: kubectl apply -f k8s/external/redis + kubectl wait --for=condition=ready pod -l app=redis --timeout=600s .PHONY: k8s/external/redis/delete ## delete redis from k8s @@ -260,12 +262,10 @@ k8s/external/scylla/deploy: \ kubectl apply -f https://raw.githubusercontent.com/scylladb/scylla-operator/master/examples/common/operator.yaml kubectl wait -n scylla-operator-system --for=condition=ready pod -l statefulset.kubernetes.io/pod-name=scylla-operator-controller-manager-0 --timeout=600s kubectl -n scylla-operator-system get pod - kubectl apply -f k8s/external/scylla/scyllacluster.yaml + kubectl apply -f $(K8S_EXTERNAL_SCYLLA_MANIFEST) kubectl -n scylla get ScyllaCluster kubectl -n scylla get pods kubectl wait -n scylla --for=condition=ready pod -l statefulset.kubernetes.io/pod-name=vald-scylla-cluster-dc0-rack0-0 --timeout=600s - kubectl wait -n scylla --for=condition=ready pod -l statefulset.kubernetes.io/pod-name=vald-scylla-cluster-dc0-rack0-1 --timeout=600s - kubectl wait -n scylla --for=condition=ready pod -l statefulset.kubernetes.io/pod-name=vald-scylla-cluster-dc0-rack0-2 --timeout=600s kubectl -n scylla get ScyllaCluster kubectl -n scylla get pods kubectl apply -f k8s/jobs/db/initialize/scylla @@ -276,7 +276,7 @@ k8s/external/scylla/deploy: \ k8s/external/scylla/delete: \ k8s/external/cert-manager/delete kubectl delete -f k8s/jobs/db/initialize/scylla - kubectl delete -f k8s/external/scylla/scyllacluster.yaml + kubectl delete -f $(K8S_EXTERNAL_SCYLLA_MANIFEST) kubectl delete -f https://raw.githubusercontent.com/scylladb/scylla-operator/master/examples/common/operator.yaml .PHONY: k8s/external/cert-manager/deploy @@ -294,6 +294,17 @@ k8s/external/cert-manager/deploy: k8s/external/cert-manager/delete: kubectl delete -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml +.PHONY: k8s/external/minio/deploy +## deploy minio +k8s/external/minio/deploy: + kubectl apply -f k8s/external/minio + kubectl wait --for=condition=ready pod -l app=minio --timeout=600s + +.PHONY: k8s/external/minio/delete +## delete minio +k8s/external/minio/delete: + kubectl delete -f k8s/external/minio + .PHONY: k8s/metrics/metrics-server/deploy ## deploy metrics-serrver k8s/metrics/metrics-server/deploy: diff --git a/charts/vald/values-scylla.yaml b/charts/vald/values-scylla.yaml index 928c0b2ed0..44b77ecf74 100644 --- a/charts/vald/values-scylla.yaml +++ b/charts/vald/values-scylla.yaml @@ -33,8 +33,6 @@ manager: cassandra: hosts: - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-1.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-2.scylla.svc.cluster.local sleepDuration: 2 env: [] env: [] @@ -45,8 +43,6 @@ manager: config: hosts: - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-1.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-2.scylla.svc.cluster.local meta: image: @@ -58,8 +54,6 @@ meta: cassandra: hosts: - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-1.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-2.scylla.svc.cluster.local sleepDuration: 2 env: [] env: [] @@ -70,5 +64,3 @@ meta: config: hosts: - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-1.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-2.scylla.svc.cluster.local diff --git a/cmd/manager/backup/cassandra/sample.yaml b/cmd/manager/backup/cassandra/sample.yaml index 8ed2e006b5..69bfe31352 100644 --- a/cmd/manager/backup/cassandra/sample.yaml +++ b/cmd/manager/backup/cassandra/sample.yaml @@ -171,8 +171,6 @@ cassandra_config: white_list: [] hosts: - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-1.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-2.scylla.svc.cluster.local ignore_peer_addr: false keyspace: vald max_prepared_stmts: 1000 diff --git a/cmd/meta/cassandra/sample.yaml b/cmd/meta/cassandra/sample.yaml index 4efef1e151..c9e837706b 100644 --- a/cmd/meta/cassandra/sample.yaml +++ b/cmd/meta/cassandra/sample.yaml @@ -173,8 +173,6 @@ cassandra_config: white_list: [] hosts: - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-1.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-2.scylla.svc.cluster.local ignore_peer_addr: false keyspace: vald max_prepared_stmts: 1000 diff --git a/docs/tutorial/get-started.md b/docs/tutorial/get-started.md index 1815306888..1de17009d0 100644 --- a/docs/tutorial/get-started.md +++ b/docs/tutorial/get-started.md @@ -83,8 +83,6 @@ If you want to learn about Scylla, please refer to [the official website](https: ```bash kubectl apply -f k8s/external/scylla/scyllacluster.yaml kubectl wait -n scylla --for=condition=ready pod -l statefulset.kubernetes.io/pod-name=vald-scylla-cluster-dc0-rack0-0 --timeout=600s - kubectl wait -n scylla --for=condition=ready pod -l statefulset.kubernetes.io/pod-name=vald-scylla-cluster-dc0-rack0-1 --timeout=600s - kubectl wait -n scylla --for=condition=ready pod -l statefulset.kubernetes.io/pod-name=vald-scylla-cluster-dc0-rack0-2 --timeout=600s kubectl -n scylla get pods ``` diff --git a/example/helm/values-scylla.yaml b/example/helm/values-scylla.yaml index d088bfdbdb..3764a395e5 100644 --- a/example/helm/values-scylla.yaml +++ b/example/helm/values-scylla.yaml @@ -76,8 +76,6 @@ backupManager: cassandra: hosts: - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-1.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-2.scylla.svc.cluster.local sleepDuration: 2 env: [] resources: @@ -91,8 +89,6 @@ backupManager: config: hosts: - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-1.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-2.scylla.svc.cluster.local consistency: one meta: @@ -105,8 +101,6 @@ meta: cassandra: hosts: - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-1.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-2.scylla.svc.cluster.local sleepDuration: 2 env: [] resources: @@ -120,6 +114,4 @@ meta: config: hosts: - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-1.scylla.svc.cluster.local - - vald-scylla-cluster-dc0-rack0-2.scylla.svc.cluster.local consistency: one diff --git a/go.mod b/go.mod index 7f1c6d22af..24b0731860 100755 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ replace ( k8s.io/api => k8s.io/api v0.20.2 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.2 k8s.io/apimachinery => k8s.io/apimachinery v0.20.2 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.2 k8s.io/client-go => k8s.io/client-go v0.20.2 k8s.io/metrics => k8s.io/metrics v0.20.2 sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.7.0 @@ -82,6 +83,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.20.2 k8s.io/apimachinery v0.20.2 + k8s.io/cli-runtime v0.0.0-00010101000000-000000000000 k8s.io/client-go v0.20.2 k8s.io/metrics v0.0.0-00010101000000-000000000000 sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 diff --git a/go.sum b/go.sum index 12e4fca063..6dc7579752 100644 --- a/go.sum +++ b/go.sum @@ -44,7 +44,9 @@ github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1M github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af h1:wVe6/Ea46ZMeNkQjjBW6xcqyQA/j5e0D6GytH95g0gQ= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= @@ -104,12 +106,15 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.4.1 h1:7dLaJvASGRD7X49jSCSXXHwKPm0ZN9r9kJD+p+vS7dM= @@ -128,6 +133,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-fonts/dejavu v0.1.0 h1:JSajPXURYqpr+Cu8U9bt8K+XcACIHWqWrvWCKyeFmVQ= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= @@ -147,11 +153,15 @@ github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4= github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= @@ -199,6 +209,7 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -224,6 +235,7 @@ github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -260,6 +272,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -313,12 +326,15 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -378,6 +394,7 @@ github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuK github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phpdave11/gofpdi v1.0.7/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/cmdflag v0.0.2/go.mod h1:a3zKGZ3cdQUfxjd0RGMLZr8xI3nvpJOB+m6o/1X5BmU= @@ -450,6 +467,7 @@ github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY52 github.com/spf13/afero v1.3.4/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -833,40 +851,29 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= -k8s.io/apiextensions-apiserver v0.20.2 h1:rfrMWQ87lhd8EzQWRnbQ4gXrniL/yTRBgYH1x1+BLlo= k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= -k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= -k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= +k8s.io/cli-runtime v0.20.2/go.mod h1:FjH6uIZZZP3XmwrXWeeYCbgxcrD6YXxoAykBaWH0VdM= k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= -k8s.io/component-base v0.20.2 h1:LMmu5I0pLtwjpp5009KLuMGFqSc2S2isGw8t1hpYKLE= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/metrics v0.20.2 h1:o32EchiH4ukpUg86VLLAgkE9a9Ke0lijkzYxE+wSSRk= k8s.io/metrics v0.20.2/go.mod h1:yTck5nl5wt/lIeLcU6g0b8/AKJf2girwe0PQiaM4Mwk= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.7.0 h1:bU20IBBEPccWz5+zXpLnpVsgBYxqclaHu1pVDl/gEt8= sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/hack/go.mod.default b/hack/go.mod.default index 5ccb43d6bd..99241ee64e 100755 --- a/hack/go.mod.default +++ b/hack/go.mod.default @@ -30,6 +30,7 @@ replace ( k8s.io/client-go => k8s.io/client-go v0.20.2 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.2 k8s.io/apimachinery => k8s.io/apimachinery v0.20.2 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.2 k8s.io/client-go => k8s.io/client-go v0.20.2 k8s.io/metrics => k8s.io/metrics v0.20.2 sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.7.0 diff --git a/internal/db/rdb/mysql/mysql.go b/internal/db/rdb/mysql/mysql.go index 30c764d92e..09618860cd 100644 --- a/internal/db/rdb/mysql/mysql.go +++ b/internal/db/rdb/mysql/mysql.go @@ -349,7 +349,7 @@ func (m *mySQLClient) SetVectors(ctx context.Context, vecs ...Vector) error { return tx.Commit() } -func (m *mySQLClient) deleteVector(ctx context.Context, val interface{}) error { +func (m *mySQLClient) deleteVector(ctx context.Context, val string) error { if !m.connected.Load().(bool) { return errors.ErrMySQLConnectionClosed } @@ -363,12 +363,21 @@ func (m *mySQLClient) deleteVector(ctx context.Context, val interface{}) error { } defer tx.RollbackUnlessCommitted() + var id int64 + _, err = tx.Select(idColumnName).From(vectorTableName).Where(m.dbr.Eq(uuidColumnName, val)).Limit(1).LoadContext(ctx, &id) + if err != nil { + return err + } + if id == 0 { + return errors.ErrRequiredElementNotFoundByUUID(val) + } + _, err = tx.DeleteFrom(vectorTableName).Where(m.dbr.Eq(uuidColumnName, val)).ExecContext(ctx) if err != nil { return err } - _, err = tx.DeleteFrom(podIPTableName).Where(m.dbr.Eq(uuidColumnName, val)).ExecContext(ctx) + _, err = tx.DeleteFrom(podIPTableName).Where(m.dbr.Eq(idColumnName, id)).ExecContext(ctx) if err != nil { return err } @@ -381,8 +390,15 @@ func (m *mySQLClient) DeleteVector(ctx context.Context, uuid string) error { } // DeleteVectors is the same as DeleteVector() but it deletes multiple records. -func (m *mySQLClient) DeleteVectors(ctx context.Context, uuids ...string) error { - return m.deleteVector(ctx, uuids) +func (m *mySQLClient) DeleteVectors(ctx context.Context, uuids ...string) (err error) { + for _, uuid := range uuids { + err = m.deleteVector(ctx, uuid) + if err != nil { + return err + } + } + + return nil } // SetIPs insert the vector's uuid and the podIPs into database. diff --git a/internal/db/rdb/mysql/mysql_test.go b/internal/db/rdb/mysql/mysql_test.go index e5b415b397..15c1f9ba90 100644 --- a/internal/db/rdb/mysql/mysql_test.go +++ b/internal/db/rdb/mysql/mysql_test.go @@ -2656,7 +2656,7 @@ func Test_mySQLClient_SetVectors(t *testing.T) { func Test_mySQLClient_deleteVector(t *testing.T) { type args struct { ctx context.Context - val interface{} + val string } type fields struct { session dbr.Session @@ -2749,6 +2749,109 @@ func Test_mySQLClient_deleteVector(t *testing.T) { }, } }(), + func() test { + err := errors.New("error returned from Select.From.Where.Limit.LoadContext") + return test{ + name: "return error when Select(idColumnName) returns error", + args: args{ + ctx: context.Background(), + val: "vald-01", + }, + fields: fields{ + session: &dbr.MockSession{ + BeginFunc: func() (dbr.Tx, error) { + return &dbr.MockTx{ + CommitFunc: func() error { + return nil + }, + RollbackUnlessCommittedFunc: func() {}, + SelectFunc: func(column ...string) dbr.SelectStmt { + s := new(dbr.MockSelect) + s.FromFunc = func(table interface{}) dbr.SelectStmt { + return s + } + s.WhereFunc = func(query interface{}, value ...interface{}) dbr.SelectStmt { + return s + } + s.LimitFunc = func(n uint64) dbr.SelectStmt { + return s + } + s.LoadContextFunc = func(ctx context.Context, value interface{}) (int, error) { + return 0, err + } + + return s + }, + }, nil + }, + }, + connected: func() (v atomic.Value) { + v.Store(true) + return + }(), + dbr: &dbr.MockDBR{ + EqFunc: func(col string, val interface{}) dbr.Builder { + return dbr.New().Eq(col, val) + }, + }, + }, + want: want{ + err: err, + }, + } + }(), + func() test { + uuid := "uuid" + err := errors.ErrRequiredElementNotFoundByUUID(uuid) + return test{ + name: "return error when returned id = 0 from Select statement", + args: args{ + ctx: context.Background(), + val: uuid, + }, + fields: fields{ + session: &dbr.MockSession{ + BeginFunc: func() (dbr.Tx, error) { + return &dbr.MockTx{ + CommitFunc: func() error { + return nil + }, + RollbackUnlessCommittedFunc: func() {}, + SelectFunc: func(column ...string) dbr.SelectStmt { + s := new(dbr.MockSelect) + s.FromFunc = func(table interface{}) dbr.SelectStmt { + return s + } + s.WhereFunc = func(query interface{}, value ...interface{}) dbr.SelectStmt { + return s + } + s.LimitFunc = func(n uint64) dbr.SelectStmt { + return s + } + s.LoadContextFunc = func(ctx context.Context, value interface{}) (int, error) { + return 0, nil + } + + return s + }, + }, nil + }, + }, + connected: func() (v atomic.Value) { + v.Store(true) + return + }(), + dbr: &dbr.MockDBR{ + EqFunc: func(col string, val interface{}) dbr.Builder { + return dbr.New().Eq(col, val) + }, + }, + }, + want: want{ + err: err, + }, + } + }(), func() test { err := errors.New("vectorTableName error") return test{ @@ -2765,6 +2868,29 @@ func Test_mySQLClient_deleteVector(t *testing.T) { return nil }, RollbackUnlessCommittedFunc: func() {}, + SelectFunc: func(column ...string) dbr.SelectStmt { + s := new(dbr.MockSelect) + s.FromFunc = func(table interface{}) dbr.SelectStmt { + return s + } + s.WhereFunc = func(query interface{}, value ...interface{}) dbr.SelectStmt { + return s + } + s.LimitFunc = func(n uint64) dbr.SelectStmt { + return s + } + s.LoadContextFunc = func(ctx context.Context, value interface{}) (int, error) { + var id int64 + if reflect.TypeOf(value) == reflect.TypeOf(&id) { + id := int64(1) + reflect.ValueOf(value).Elem().Set(reflect.ValueOf(id)) + return 1, nil + } + return 0, nil + } + + return s + }, DeleteFromFunc: func(table string) dbr.DeleteStmt { s := new(dbr.MockDelete) s.ExecContextFunc = func(ctx context.Context) (sql.Result, error) { @@ -2812,6 +2938,29 @@ func Test_mySQLClient_deleteVector(t *testing.T) { return nil }, RollbackUnlessCommittedFunc: func() {}, + SelectFunc: func(column ...string) dbr.SelectStmt { + s := new(dbr.MockSelect) + s.FromFunc = func(table interface{}) dbr.SelectStmt { + return s + } + s.WhereFunc = func(query interface{}, value ...interface{}) dbr.SelectStmt { + return s + } + s.LimitFunc = func(n uint64) dbr.SelectStmt { + return s + } + s.LoadContextFunc = func(ctx context.Context, value interface{}) (int, error) { + var id int64 + if reflect.TypeOf(value) == reflect.TypeOf(&id) { + id := int64(1) + reflect.ValueOf(value).Elem().Set(reflect.ValueOf(id)) + return 1, nil + } + return 0, nil + } + + return s + }, DeleteFromFunc: func(table string) dbr.DeleteStmt { s := new(dbr.MockDelete) s.ExecContextFunc = func(ctx context.Context) (sql.Result, error) { @@ -2858,6 +3007,29 @@ func Test_mySQLClient_deleteVector(t *testing.T) { return nil }, RollbackUnlessCommittedFunc: func() {}, + SelectFunc: func(column ...string) dbr.SelectStmt { + s := new(dbr.MockSelect) + s.FromFunc = func(table interface{}) dbr.SelectStmt { + return s + } + s.WhereFunc = func(query interface{}, value ...interface{}) dbr.SelectStmt { + return s + } + s.LimitFunc = func(n uint64) dbr.SelectStmt { + return s + } + s.LoadContextFunc = func(ctx context.Context, value interface{}) (int, error) { + var id int64 + if reflect.TypeOf(value) == reflect.TypeOf(&id) { + id := int64(1) + reflect.ValueOf(value).Elem().Set(reflect.ValueOf(id)) + return 1, nil + } + return 0, nil + } + + return s + }, DeleteFromFunc: func(table string) dbr.DeleteStmt { s := new(dbr.MockDelete) s.ExecContextFunc = func(ctx context.Context) (sql.Result, error) { @@ -2956,6 +3128,29 @@ func Test_mySQLClient_DeleteVector(t *testing.T) { return nil }, RollbackUnlessCommittedFunc: func() {}, + SelectFunc: func(column ...string) dbr.SelectStmt { + s := new(dbr.MockSelect) + s.FromFunc = func(table interface{}) dbr.SelectStmt { + return s + } + s.WhereFunc = func(query interface{}, value ...interface{}) dbr.SelectStmt { + return s + } + s.LimitFunc = func(n uint64) dbr.SelectStmt { + return s + } + s.LoadContextFunc = func(ctx context.Context, value interface{}) (int, error) { + var id int64 + if reflect.TypeOf(value) == reflect.TypeOf(&id) { + id := int64(1) + reflect.ValueOf(value).Elem().Set(reflect.ValueOf(id)) + return 1, nil + } + return 0, nil + } + + return s + }, DeleteFromFunc: func(table string) dbr.DeleteStmt { s := new(dbr.MockDelete) s.ExecContextFunc = func(ctx context.Context) (sql.Result, error) { @@ -2997,6 +3192,29 @@ func Test_mySQLClient_DeleteVector(t *testing.T) { return nil }, RollbackUnlessCommittedFunc: func() {}, + SelectFunc: func(column ...string) dbr.SelectStmt { + s := new(dbr.MockSelect) + s.FromFunc = func(table interface{}) dbr.SelectStmt { + return s + } + s.WhereFunc = func(query interface{}, value ...interface{}) dbr.SelectStmt { + return s + } + s.LimitFunc = func(n uint64) dbr.SelectStmt { + return s + } + s.LoadContextFunc = func(ctx context.Context, value interface{}) (int, error) { + var id int64 + if reflect.TypeOf(value) == reflect.TypeOf(&id) { + id := int64(1) + reflect.ValueOf(value).Elem().Set(reflect.ValueOf(id)) + return 1, nil + } + return 0, nil + } + + return s + }, DeleteFromFunc: func(table string) dbr.DeleteStmt { s := new(dbr.MockDelete) s.ExecContextFunc = func(ctx context.Context) (sql.Result, error) { @@ -3095,6 +3313,29 @@ func Test_mySQLClient_DeleteVectors(t *testing.T) { return nil }, RollbackUnlessCommittedFunc: func() {}, + SelectFunc: func(column ...string) dbr.SelectStmt { + s := new(dbr.MockSelect) + s.FromFunc = func(table interface{}) dbr.SelectStmt { + return s + } + s.WhereFunc = func(query interface{}, value ...interface{}) dbr.SelectStmt { + return s + } + s.LimitFunc = func(n uint64) dbr.SelectStmt { + return s + } + s.LoadContextFunc = func(ctx context.Context, value interface{}) (int, error) { + var id int64 + if reflect.TypeOf(value) == reflect.TypeOf(&id) { + id := int64(1) + reflect.ValueOf(value).Elem().Set(reflect.ValueOf(id)) + return 1, nil + } + return 0, nil + } + + return s + }, DeleteFromFunc: func(table string) dbr.DeleteStmt { s := new(dbr.MockDelete) s.ExecContextFunc = func(ctx context.Context) (sql.Result, error) { @@ -3139,6 +3380,29 @@ func Test_mySQLClient_DeleteVectors(t *testing.T) { return nil }, RollbackUnlessCommittedFunc: func() {}, + SelectFunc: func(column ...string) dbr.SelectStmt { + s := new(dbr.MockSelect) + s.FromFunc = func(table interface{}) dbr.SelectStmt { + return s + } + s.WhereFunc = func(query interface{}, value ...interface{}) dbr.SelectStmt { + return s + } + s.LimitFunc = func(n uint64) dbr.SelectStmt { + return s + } + s.LoadContextFunc = func(ctx context.Context, value interface{}) (int, error) { + var id int64 + if reflect.TypeOf(value) == reflect.TypeOf(&id) { + id := int64(1) + reflect.ValueOf(value).Elem().Set(reflect.ValueOf(id)) + return 1, nil + } + return 0, nil + } + + return s + }, DeleteFromFunc: func(table string) dbr.DeleteStmt { s := new(dbr.MockDelete) s.ExecContextFunc = func(ctx context.Context) (sql.Result, error) { diff --git a/k8s/external/minio/deployment.yaml b/k8s/external/minio/deployment.yaml new file mode 100644 index 0000000000..8f52ebe611 --- /dev/null +++ b/k8s/external/minio/deployment.yaml @@ -0,0 +1,58 @@ +# +# Copyright (C) 2019-2021 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: minio + name: minio +spec: + progressDeadlineSeconds: 600 + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio + imagePullPolicy: Always + command: + - "/usr/bin/docker-entrypoint.sh" + - "server" + - "/data" + env: + - name: MINIO_ACCESS_KEY + value: ACCESSKEY + - name: MINIO_SECRET_KEY + value: SECRETKEY + ports: + - containerPort: 9000 + name: minio + protocol: TCP + resources: + requests: + cpu: 100m + memory: 100Mi + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 diff --git a/k8s/external/minio/mb-job.yaml b/k8s/external/minio/mb-job.yaml new file mode 100644 index 0000000000..43ada9d3fb --- /dev/null +++ b/k8s/external/minio/mb-job.yaml @@ -0,0 +1,41 @@ +--- +# +# Copyright (C) 2019-2021 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-make-bucket +spec: + template: + spec: + containers: + - name: mc + image: minio/mc + imagePullPolicy: Always + command: + - /bin/sh + - -c + - | + mc alias set minio ${ENDPOINT} ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} --api S3v4 + mc mb minio/vald-minio + env: + - name: ENDPOINT + value: http://minio:9000 + - name: MINIO_ACCESS_KEY + value: ACCESSKEY + - name: MINIO_SECRET_KEY + value: SECRETKEY + restartPolicy: Never diff --git a/k8s/external/minio/svc.yaml b/k8s/external/minio/svc.yaml new file mode 100644 index 0000000000..1d714880c3 --- /dev/null +++ b/k8s/external/minio/svc.yaml @@ -0,0 +1,25 @@ +# +# Copyright (C) 2019-2021 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + ports: + - port: 9000 + selector: + app: minio + clusterIP: None diff --git a/k8s/external/scylla/scyllacluster.yaml b/k8s/external/scylla/scyllacluster.yaml index 5b1d466207..cebf733387 100644 --- a/k8s/external/scylla/scyllacluster.yaml +++ b/k8s/external/scylla/scyllacluster.yaml @@ -95,7 +95,7 @@ spec: name: dc0 racks: - agentResources: {} - members: 3 + members: 1 name: rack0 resources: limits: diff --git a/tests/e2e/crud_test.go b/tests/e2e/crud_test.go new file mode 100644 index 0000000000..70e4f277d2 --- /dev/null +++ b/tests/e2e/crud_test.go @@ -0,0 +1,758 @@ +// +build e2e + +// +// Copyright (C) 2019-2021 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// package e2e provides e2e tests using ann-benchmarks datasets +package e2e + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strconv" + "sync" + "testing" + "time" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/tests/e2e/portforward" + + "gonum.org/v1/hdf5" + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) + +var ( + host string + port int + ds *dataset + + insertNum int + searchNum int + searchByIDNum int + getObjectNum int + updateNum int + removeNum int + + insertFrom int + searchFrom int + searchByIDFrom int + getObjectFrom int + updateFrom int + removeFrom int + + waitAfterInsertDuration time.Duration + + forwarder *portforward.Portforward +) + +func init() { + testing.Init() + + flag.StringVar(&host, "host", "localhost", "hostname") + flag.IntVar(&port, "port", 8081, "gRPC port") + + flag.IntVar(&insertNum, "insert-num", 10000, "number of id-vector pairs used for insert") + flag.IntVar(&searchNum, "search-num", 10000, "number of id-vector pairs used for search") + flag.IntVar(&searchByIDNum, "search-by-id-num", 100, "number of id-vector pairs used for search-by-id") + flag.IntVar(&getObjectNum, "get-object-num", 100, "number of id-vector pairs used for get-object") + flag.IntVar(&updateNum, "update-num", 10000, "number of id-vector pairs used for update") + flag.IntVar(&removeNum, "remove-num", 10000, "number of id-vector pairs used for remove") + + flag.IntVar(&insertFrom, "insert-from", 0, "first index of id-vector pairs used for insert") + flag.IntVar(&searchFrom, "search-from", 0, "first index of id-vector pairs used for search") + flag.IntVar(&searchByIDFrom, "search-by-id-from", 0, "first index of id-vector pairs used for search-by-id") + flag.IntVar(&getObjectFrom, "get-object-from", 0, "first index of id-vector pairs used for get-object") + flag.IntVar(&updateFrom, "update-from", 0, "first index of id-vector pairs used for update") + flag.IntVar(&removeFrom, "remove-from", 0, "first index of id-vector pairs used for remove") + + datasetName := flag.String("dataset", "fashion-mnist-784-euclidean.hdf5", "dataset") + waitAfterInsert := flag.String("wait-after-insert", "3m", "wait duration after inserting vectors") + + pf := flag.Bool("portforward", false, "enable port forwarding") + pfNamespace := flag.String("portforward-ns", "default", "namespace (only for port forward)") + pfPodName := flag.String("portforward-pod-name", "vald-gateway-0", "pod name (only for port forward)") + pfPodPort := flag.Int("portforward-pod-port", port, "pod gRPC port (only for port forward)") + kubeConfig := flag.String("kubeconfig", filepath.Join(os.Getenv("HOME"), ".kube", "config"), "kubeconfig path (only for port forward)") + + flag.Parse() + + var err error + if *pf { + forwarder, err = portforward.NewPortforward(*kubeConfig, *pfNamespace, *pfPodName, port, *pfPodPort) + if err != nil { + panic(err) + } + + err = forwarder.Start() + if err != nil { + panic(err) + } + } + + fmt.Printf("loading dataset: %s", *datasetName) + ds, err = hdf5ToDataset(*datasetName) + if err != nil { + panic(err) + } + fmt.Println("loading finished") + + waitAfterInsertDuration, err = time.ParseDuration(*waitAfterInsert) + if err != nil { + panic(err) + } +} + +func teardown() { + if forwarder != nil { + forwarder.Close() + } +} + +func TestMain(m *testing.M) { + ret := m.Run() + teardown() + os.Exit(ret) +} + +type dataset struct { + train map[string][]float32 + test map[string][]float32 + neighbors map[string][]string +} + +func hdf5ToDataset(name string) (*dataset, error) { + file, err := hdf5.OpenFile(name, hdf5.F_ACC_RDONLY) + if err != nil { + return nil, err + } + + defer file.Close() + + train, err := readDatasetF32(file, "train") + if err != nil { + return nil, err + } + + test, err := readDatasetF32(file, "test") + if err != nil { + return nil, err + } + + nbors, err := readDatasetI32(file, "neighbors") + if err != nil { + return nil, err + } + neighbors := make(map[string][]string, len(nbors)) + for k, vs := range nbors { + vss := make([]string, len(vs)) + for i, v := range vs { + vss[i] = strconv.Itoa(int(v)) + } + neighbors[k] = vss + } + + return &dataset{ + train: train, + test: test, + neighbors: neighbors, + }, nil +} + +func readDatasetF32(file *hdf5.File, name string) (map[string][]float32, error) { + data, err := file.OpenDataset(name) + if err != nil { + return nil, err + } + defer data.Close() + + dataspace := data.Space() + defer dataspace.Close() + + dims, _, err := dataspace.SimpleExtentDims() + if err != nil { + return nil, err + } + height, width := int(dims[0]), int(dims[1]) + + rawFloats := make([]float32, dataspace.SimpleExtentNPoints()) + if err := data.Read(&rawFloats); err != nil { + return nil, err + } + + vecs := make(map[string][]float32, height) + for i := 0; i < height; i++ { + vecs[strconv.Itoa(i)] = rawFloats[i*width : i*width+width] + } + + return vecs, nil +} + +func readDatasetI32(file *hdf5.File, name string) (map[string][]int32, error) { + data, err := file.OpenDataset(name) + if err != nil { + return nil, err + } + defer data.Close() + + dataspace := data.Space() + defer dataspace.Close() + + dims, _, err := dataspace.SimpleExtentDims() + if err != nil { + return nil, err + } + height, width := int(dims[0]), int(dims[1]) + + rawFloats := make([]int32, dataspace.SimpleExtentNPoints()) + if err := data.Read(&rawFloats); err != nil { + return nil, err + } + + vecs := make(map[string][]int32, height) + for i := 0; i < height; i++ { + vecs[strconv.Itoa(i)] = rawFloats[i*width : i*width+width] + } + + return vecs, nil +} + +func getClient(ctx context.Context) (vald.Client, error) { + conn, err := grpc.DialContext( + ctx, + host+":"+strconv.Itoa(port), + grpc.WithInsecure(), + grpc.WithKeepaliveParams( + keepalive.ClientParameters{ + Time: time.Second, + Timeout: 5 * time.Second, + PermitWithoutStream: true, + }, + ), + ) + if err != nil { + return nil, err + } + + return vald.NewValdClient(conn), nil +} + +func sleep(t *testing.T, dur time.Duration) { + t.Logf("sleep for %s", dur) + time.Sleep(dur) + t.Log("sleep finished.") +} + +func TestE2EInsert(t *testing.T) { + ctx := context.Background() + + client, err := getClient(ctx) + if err != nil { + t.Fatal(err) + } + + sc, err := client.StreamInsert(ctx) + if err != nil { + t.Fatal(err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + + count := 0 + for { + res, err := sc.Recv() + if err == io.EOF { + t.Logf("%d items inserted.", count) + return + } else if err != nil { + t.Fatal(err) + } + + loc := res.GetLocation() + if loc == nil { + err := res.GetStatus() + if err != nil { + t.Errorf("an error returned: %s", err.GetMessage()) + } + } else { + t.Logf("returned: %s", loc) + } + + count++ + + if count%1000 == 0 { + t.Logf("inserted: %d", count) + } + } + }() + + t.Log("insert start") + for i := insertFrom; i < len(ds.train); i++ { + id := strconv.Itoa(i) + err := sc.Send(&payload.Insert_Request{ + Vector: &payload.Object_Vector{ + Id: id, + Vector: ds.train[id], + }, + Config: &payload.Insert_Config{ + SkipStrictExistCheck: false, + }, + }) + if err != nil { + t.Fatalf("send failed at %d: %s", i+1, err) + } + + if (i+1)%1000 == 0 { + t.Logf("sent: %d", i+1) + } + + if i+1 >= insertFrom+insertNum { + t.Logf("%d items sent.", i+1) + break + } + } + + sc.CloseSend() + + wg.Wait() + + t.Log("insert finished.") + + sleep(t, waitAfterInsertDuration) +} + +func TestE2ESearch(t *testing.T) { + ctx := context.Background() + + client, err := getClient(ctx) + if err != nil { + t.Fatal(err) + } + + sc, err := client.StreamSearch(ctx) + if err != nil { + t.Fatal(err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + + k := 0 + for { + res, err := sc.Recv() + if err == io.EOF { + t.Logf("%d items searched.", k) + return + } else if err != nil { + t.Fatal(err) + } + + resp := res.GetResponse() + if resp == nil { + err := res.GetStatus() + if err != nil { + t.Errorf("an error returned: %s", err.GetMessage()) + } + } else { + topKIDs := make([]string, len(resp.GetResults())) + for i, d := range resp.GetResults() { + topKIDs[i] = d.Id + } + + if len(topKIDs) == 0 { + t.Errorf("empty result is returned for ID %d: %#v", k, topKIDs) + } + + // TODO: validation + // calculate recall? + // t.Logf("result: %#v", topKIDs) + // t.Logf("expected: %#v", ds.neighbors[strconv.Itoa(k)][:len(topKIDs)]) + } + + k++ + + if k%1000 == 0 { + t.Logf("searched: %d", k) + } + } + }() + + t.Log("search start") + for i := searchFrom; i < len(ds.test); i++ { + id := strconv.Itoa(i) + err := sc.Send(&payload.Search_Request{ + Vector: ds.test[id], + Config: &payload.Search_Config{ + Num: 100, + Radius: -1.0, + Epsilon: 0.01, + Timeout: 3000000000, + }, + }) + if err != nil { + t.Fatal(err) + } + + if (i+1)%1000 == 0 { + t.Logf("sent: %d", i+1) + } + + if i+1 >= searchFrom+searchNum { + t.Logf("%d items sent.", i+1) + break + } + } + + sc.CloseSend() + + wg.Wait() + + t.Log("search finished.") +} + +func TestE2ESearchByID(t *testing.T) { + ctx := context.Background() + + client, err := getClient(ctx) + if err != nil { + t.Fatal(err) + } + + sc, err := client.StreamSearchByID(ctx) + if err != nil { + t.Fatal(err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + + count := 0 + for { + res, err := sc.Recv() + if err == io.EOF { + t.Logf("%d items searched.", count) + return + } else if err != nil { + t.Fatal(err) + } + + resp := res.GetResponse() + if resp == nil { + err := res.GetStatus() + if err != nil { + t.Errorf("an error returned: %s", err.GetMessage()) + } + } else { + topKIDs := make([]string, len(resp.GetResults())) + for i, d := range resp.GetResults() { + topKIDs[i] = d.Id + } + + if len(topKIDs) == 0 { + t.Errorf("empty result is returned: %#v", topKIDs) + } + } + + count++ + + if count%1000 == 0 { + t.Logf("searched: %d", count) + } + } + }() + + t.Log("search-by-id start") + for i := searchByIDFrom; i < len(ds.train); i++ { + id := strconv.Itoa(i) + err := sc.Send(&payload.Search_IDRequest{ + Id: id, + Config: &payload.Search_Config{ + Num: 100, + Radius: -1.0, + Epsilon: 0.01, + Timeout: 3000000000, + }, + }) + if err != nil { + t.Fatal(err) + } + + if (i+1)%1000 == 0 { + t.Logf("sent: %d", i+1) + } + + if i+1 >= searchByIDFrom+searchByIDNum { + t.Logf("%d items sent.", i+1) + break + } + } + + sc.CloseSend() + + wg.Wait() + + t.Log("search-by-id finished.") +} + +func TestE2EGetObject(t *testing.T) { + ctx := context.Background() + + client, err := getClient(ctx) + if err != nil { + t.Fatal(err) + } + + sc, err := client.StreamGetObject(ctx) + if err != nil { + t.Fatal(err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + + count := 0 + for { + res, err := sc.Recv() + if err == io.EOF { + t.Logf("%d items got.", count) + return + } else if err != nil { + t.Fatal(err) + } + + resp := res.GetVector() + if resp == nil { + err := res.GetStatus() + if err != nil { + t.Errorf("an error returned: %s", err.GetMessage()) + } + } else { + if !reflect.DeepEqual(res.GetVector(), ds.train[resp.GetId()]) { + t.Errorf( + "result: %#v, expected: %#v", + res.GetVector(), + ds.train[resp.GetId()], + ) + } + } + + count++ + + if count%1000 == 0 { + t.Logf("get object: %d", count) + } + } + }() + + t.Log("get object start") + for i := getObjectFrom; i < len(ds.train); i++ { + id := strconv.Itoa(i) + err := sc.Send(&payload.Object_ID{ + Id: id, + }) + if err != nil { + t.Fatal(err) + } + + if (i+1)%1000 == 0 { + t.Logf("sent: %d", i+1) + } + + if i+1 >= getObjectFrom+getObjectNum { + t.Logf("%d items sent.", i+1) + break + } + } + + sc.CloseSend() + + wg.Wait() + + t.Log("get object finished.") +} + +func TestE2EUpdate(t *testing.T) { + ctx := context.Background() + + client, err := getClient(ctx) + if err != nil { + t.Fatal(err) + } + + sc, err := client.StreamUpdate(ctx) + if err != nil { + t.Fatal(err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + + count := 0 + for { + res, err := sc.Recv() + if err == io.EOF { + t.Logf("%d items updated.", count) + return + } else if err != nil { + t.Fatal(err) + } + + loc := res.GetLocation() + if loc == nil { + err := res.GetStatus() + if err != nil { + t.Errorf("an error returned: %s", err.GetMessage()) + } + } else { + t.Logf("returned: %s", loc) + } + + count++ + + if count%1000 == 0 { + t.Logf("updated: %d", count) + } + } + }() + + t.Log("update start") + for i := updateFrom; i < len(ds.train); i++ { + id := strconv.Itoa(i) + v := ds.train[id] + err := sc.Send(&payload.Update_Request{ + Vector: &payload.Object_Vector{ + Id: id, + Vector: append(v[1:], v[0]), // shift + }, + Config: &payload.Update_Config{ + SkipStrictExistCheck: false, + }, + }) + if err != nil { + t.Fatal(err) + } + + if (i+1)%1000 == 0 { + t.Logf("sent: %d", i+1) + } + + if i+1 >= updateFrom+updateNum { + t.Logf("%d items sent.", i+1) + break + } + } + + sc.CloseSend() + + wg.Wait() + + t.Log("update finished.") +} + +func TestE2ERemove(t *testing.T) { + ctx := context.Background() + + client, err := getClient(ctx) + if err != nil { + t.Fatal(err) + } + + sc, err := client.StreamRemove(ctx) + if err != nil { + t.Fatal(err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + + count := 0 + for { + res, err := sc.Recv() + if err == io.EOF { + t.Logf("%d items removed.", count) + return + } else if err != nil { + t.Fatal(err) + } + + loc := res.GetLocation() + if loc == nil { + err := res.GetStatus() + if err != nil { + t.Errorf("an error returned: %s", err.GetMessage()) + } + } else { + t.Logf("returned: %s", loc) + } + + count++ + + if count%1000 == 0 { + t.Logf("removed: %d", count) + } + } + }() + + t.Log("remove start") + for i := removeFrom; i < len(ds.train); i++ { + id := strconv.Itoa(i) + err := sc.Send(&payload.Remove_Request{ + Id: &payload.Object_ID{ + Id: id, + }, + Config: &payload.Remove_Config{ + SkipStrictExistCheck: false, + }, + }) + if err != nil { + t.Fatal(err) + } + + if (i+1)%1000 == 0 { + t.Logf("sent: %d", i+1) + } + + if i+1 >= removeFrom+removeNum { + t.Logf("%d items sent.", i+1) + break + } + } + + sc.CloseSend() + + wg.Wait() + + t.Log("remove finished.") +} diff --git a/tests/e2e/portforward/portforward.go b/tests/e2e/portforward/portforward.go new file mode 100644 index 0000000000..91fa2332c0 --- /dev/null +++ b/tests/e2e/portforward/portforward.go @@ -0,0 +1,130 @@ +// +build e2e + +// +// Copyright (C) 2019-2021 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// package portforward provides port-forward functionality for e2e tests +package portforward + +import ( + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" +) + +// portforwarder provides a port-forwarding functionality of kubectl +// reference: https://github.com/gianarb/kube-port-forward +type Portforward struct { + namespace string + podName string + + localPort int + podPort int + + restConfig *rest.Config + readyCh chan struct{} + stopCh chan struct{} +} + +func NewPortforward(kubeConfig, namespace, podName string, localPort, podPort int) (*Portforward, error) { + if kubeConfig == "" { + if home := os.Getenv("HOME"); home != "" { + kubeConfig = filepath.Join(home, ".kube", "config") + } else { + kubeConfig = os.Getenv("KUBECONFIG") + } + } + + config, err := clientcmd.BuildConfigFromFlags("", kubeConfig) + if err != nil { + return nil, err + } + + return &Portforward{ + namespace: namespace, + podName: podName, + localPort: localPort, + podPort: podPort, + restConfig: config, + readyCh: make(chan struct{}), + stopCh: make(chan struct{}, 1), + }, nil +} + +func (p *Portforward) Start() error { + stream := genericclioptions.IOStreams{ + In: os.Stdin, + Out: os.Stdout, + ErrOut: os.Stderr, + } + + path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", + p.namespace, p.podName) + + hostIP := strings.TrimLeft(p.restConfig.Host, "https:/") + + transport, upgrader, err := spdy.RoundTripperFor(p.restConfig) + if err != nil { + return err + } + + ech := make(chan error, 1) + go func() { + fw, err := portforward.New( + spdy.NewDialer( + upgrader, + &http.Client{Transport: transport}, + http.MethodPost, + &url.URL{Scheme: "https", Path: path, Host: hostIP}, + ), + []string{fmt.Sprintf("%d:%d", p.localPort, p.podPort)}, + p.stopCh, + p.readyCh, + stream.Out, + stream.ErrOut, + ) + if err != nil { + ech <- err + } + + err = fw.ForwardPorts() + if err != nil { + ech <- err + } + }() + + select { + case <-p.readyCh: + return nil + case err = <-ech: + return err + } +} + +func (p *Portforward) Close() error { + close(p.stopCh) + + return nil +} diff --git a/tests/e2e/sidecar_test.go b/tests/e2e/sidecar_test.go new file mode 100644 index 0000000000..0cc5331f23 --- /dev/null +++ b/tests/e2e/sidecar_test.go @@ -0,0 +1,455 @@ +// +build e2e + +// +// Copyright (C) 2019-2021 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// package e2e provides e2e tests using ann-benchmarks datasets +package e2e + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + "time" + + "github.com/vdaas/vald/apis/grpc/v1/agent/core" + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/tests/e2e/portforward" + + "gonum.org/v1/hdf5" + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) + +var ( + host string + port int + ds *dataset + + insertNum int + searchNum int + searchByIDNum int + getObjectNum int + updateNum int + removeNum int + + forwarder *portforward.Portforward +) + +func init() { + testing.Init() + + flag.StringVar(&host, "host", "localhost", "hostname") + flag.IntVar(&port, "port", 8081, "gRPC port") + + flag.IntVar(&insertNum, "insert-num", 10000, "number of id-vector pairs used for insert") + flag.IntVar(&searchNum, "search-num", 10000, "number of id-vector pairs used for search") + + datasetName := flag.String("dataset", "fashion-mnist-784-euclidean.hdf5", "dataset") + + pf := flag.Bool("portforward", false, "enable port forwarding") + pfNamespace := flag.String("portforward-ns", "default", "namespace (only for port forward)") + pfPodName := flag.String("portforward-pod-name", "vald-gateway-0", "pod name (only for port forward)") + pfPodPort := flag.Int("portforward-pod-port", port, "pod gRPC port (only for port forward)") + kubeConfig := flag.String("kubeconfig", filepath.Join(os.Getenv("HOME"), ".kube", "config"), "kubeconfig path (only for port forward)") + + flag.Parse() + + var err error + if *pf { + forwarder, err = portforward.NewPortforward(*kubeConfig, *pfNamespace, *pfPodName, port, *pfPodPort) + if err != nil { + panic(err) + } + + err = forwarder.Start() + if err != nil { + panic(err) + } + } + + fmt.Printf("loading dataset: %s", *datasetName) + ds, err = hdf5ToDataset(*datasetName) + if err != nil { + panic(err) + } + fmt.Println("loading finished") +} + +func teardown() { + if forwarder != nil { + forwarder.Close() + } +} + +func TestMain(m *testing.M) { + ret := m.Run() + teardown() + os.Exit(ret) +} + +type dataset struct { + train map[string][]float32 + test map[string][]float32 + neighbors map[string][]string +} + +func hdf5ToDataset(name string) (*dataset, error) { + file, err := hdf5.OpenFile(name, hdf5.F_ACC_RDONLY) + if err != nil { + return nil, err + } + + defer file.Close() + + train, err := readDatasetF32(file, "train") + if err != nil { + return nil, err + } + + test, err := readDatasetF32(file, "test") + if err != nil { + return nil, err + } + + nbors, err := readDatasetI32(file, "neighbors") + if err != nil { + return nil, err + } + neighbors := make(map[string][]string, len(nbors)) + for k, vs := range nbors { + vss := make([]string, len(vs)) + for i, v := range vs { + vss[i] = strconv.Itoa(int(v)) + } + neighbors[k] = vss + } + + return &dataset{ + train: train, + test: test, + neighbors: neighbors, + }, nil +} + +func readDatasetF32(file *hdf5.File, name string) (map[string][]float32, error) { + data, err := file.OpenDataset(name) + if err != nil { + return nil, err + } + defer data.Close() + + dataspace := data.Space() + defer dataspace.Close() + + dims, _, err := dataspace.SimpleExtentDims() + if err != nil { + return nil, err + } + height, width := int(dims[0]), int(dims[1]) + + rawFloats := make([]float32, dataspace.SimpleExtentNPoints()) + if err := data.Read(&rawFloats); err != nil { + return nil, err + } + + vecs := make(map[string][]float32, height) + for i := 0; i < height; i++ { + vecs[strconv.Itoa(i)] = rawFloats[i*width : i*width+width] + } + + return vecs, nil +} + +func readDatasetI32(file *hdf5.File, name string) (map[string][]int32, error) { + data, err := file.OpenDataset(name) + if err != nil { + return nil, err + } + defer data.Close() + + dataspace := data.Space() + defer dataspace.Close() + + dims, _, err := dataspace.SimpleExtentDims() + if err != nil { + return nil, err + } + height, width := int(dims[0]), int(dims[1]) + + rawFloats := make([]int32, dataspace.SimpleExtentNPoints()) + if err := data.Read(&rawFloats); err != nil { + return nil, err + } + + vecs := make(map[string][]int32, height) + for i := 0; i < height; i++ { + vecs[strconv.Itoa(i)] = rawFloats[i*width : i*width+width] + } + + return vecs, nil +} + +func getAgentClient(ctx context.Context) (core.AgentClient, error) { + conn, err := grpc.DialContext( + ctx, + host+":"+strconv.Itoa(port), + grpc.WithInsecure(), + grpc.WithKeepaliveParams( + keepalive.ClientParameters{ + Time: time.Second, + Timeout: 5 * time.Second, + PermitWithoutStream: true, + }, + ), + ) + if err != nil { + return nil, err + } + + return core.NewAgentClient(conn), nil +} + +func getClient(ctx context.Context) (vald.Client, error) { + conn, err := grpc.DialContext( + ctx, + host+":"+strconv.Itoa(port), + grpc.WithInsecure(), + grpc.WithKeepaliveParams( + keepalive.ClientParameters{ + Time: time.Second, + Timeout: 5 * time.Second, + PermitWithoutStream: true, + }, + ), + ) + if err != nil { + return nil, err + } + + return vald.NewValdClient(conn), nil +} + +func TestE2EInsert(t *testing.T) { + ctx := context.Background() + + client, err := getClient(ctx) + if err != nil { + t.Fatal(err) + } + + sc, err := client.StreamInsert(ctx) + if err != nil { + t.Fatal(err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + + count := 0 + for { + res, err := sc.Recv() + if err == io.EOF { + t.Logf("%d items inserted.", count) + return + } else if err != nil { + t.Fatal(err) + } + + loc := res.GetLocation() + if loc == nil { + err := res.GetStatus() + if err != nil { + t.Errorf("an error returned: %s", err.GetMessage()) + } + } else { + t.Logf("returned: %s", loc) + } + + count++ + + if count%1000 == 0 { + t.Logf("inserted: %d", count) + } + } + }() + + t.Log("insert start") + for i := 0; i < len(ds.train); i++ { + id := strconv.Itoa(i) + err := sc.Send(&payload.Insert_Request{ + Vector: &payload.Object_Vector{ + Id: id, + Vector: ds.train[id], + }, + Config: &payload.Insert_Config{ + SkipStrictExistCheck: false, + }, + }) + if err != nil { + t.Fatal(err) + } + + if (i+1)%1000 == 0 { + t.Logf("sent: %d", i+1) + } + + if i+1 >= insertNum { + t.Logf("%d items sent.", i+1) + break + } + } + + sc.CloseSend() + + wg.Wait() + + t.Log("insert finished.") +} + +func TestE2ECreateIndex(t *testing.T) { + ctx := context.Background() + + client, err := getAgentClient(ctx) + if err != nil { + t.Fatal(err) + } + + _, err = client.CreateAndSaveIndex(ctx, &payload.Control_CreateIndexRequest{ + PoolSize: 10000, + }) + if err != nil { + t.Fatal(err) + } +} + +func TestE2ESearch(t *testing.T) { + ctx := context.Background() + + client, err := getClient(ctx) + if err != nil { + t.Fatal(err) + } + + sc, err := client.StreamSearch(ctx) + if err != nil { + t.Fatal(err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + + k := 0 + for { + res, err := sc.Recv() + if err == io.EOF { + t.Logf("%d items searched.", k) + return + } else if err != nil { + t.Fatal(err) + } + + resp := res.GetResponse() + if resp == nil { + err := res.GetStatus() + if err != nil { + t.Errorf("an error returned: %s", err.GetMessage()) + } + } else { + topKIDs := make([]string, len(resp.GetResults())) + for i, d := range resp.GetResults() { + topKIDs[i] = d.Id + } + + if len(topKIDs) == 0 { + t.Errorf("empty result is returned for ID %d: %#v", k, topKIDs) + } + + // TODO: validation + // calculate recall? + // t.Logf("result: %#v", topKIDs) + // t.Logf("expected: %#v", ds.neighbors[strconv.Itoa(k)][:len(topKIDs)]) + } + + k++ + + if k%1000 == 0 { + t.Logf("searched: %d", k) + } + } + }() + + t.Log("search start") + for i := 0; i < len(ds.test); i++ { + id := strconv.Itoa(i) + err := sc.Send(&payload.Search_Request{ + Vector: ds.test[id], + Config: &payload.Search_Config{ + Num: 100, + Radius: -1.0, + Epsilon: 0.01, + Timeout: 3000000000, + }, + }) + if err != nil { + t.Fatal(err) + } + + if (i+1)%1000 == 0 { + t.Logf("sent: %d", i+1) + } + + if i+1 >= searchNum { + t.Logf("%d items sent.", i+1) + break + } + } + + sc.CloseSend() + + wg.Wait() + + t.Log("search finished.") +} + +func TestE2EIndexInfo(t *testing.T) { + ctx := context.Background() + + client, err := getAgentClient(ctx) + if err != nil { + t.Fatal(err) + } + + res, err := client.IndexInfo(ctx, &payload.Empty{}) + if err != nil { + t.Fatal(err) + } + + if res.GetStored() != uint32(insertNum) { + t.Errorf("stored index number: %d, not equals to expected: %d", res.GetStored(), insertNum) + } +}