From b682495559ee2ac0aa98d8ed046f7082e38326c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nick=20M=C3=BCller?= Date: Tue, 8 Nov 2022 16:54:17 +0100 Subject: [PATCH] Updating of artifacts and artifact data (#83) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fixed repo interface mocks not being generated Signed-off-by: Nick Müller * Implemented updating of artifacts Currently only supports replacement of associated ArtifactData Uses unreleased flyteidl update for new datacatalog endpoint Signed-off-by: Nick Müller * Updated project to Go 1.18 Updated Dockerfile to be in line with flyteadmin Updated GitHub automation to use Go 1.18 Signed-off-by: Nick Müller * Fixed artifact data upsert in Artifact.Update Signed-off-by: Nick Müller * Removed golang_dockerfile from boilerplate update Signed-off-by: Nick Müller * UpdateArtifact returns ID of artifact updated Signed-off-by: Nick Müller * Updated to latest version of flytepropeller Signed-off-by: Nick Müller * Updated to latest released version of flyteidl Signed-off-by: Nick Müller * Updated to latest released flytestdlib version Signed-off-by: Nick Müller * Missing artifact data is now removed after DB models have been updated Prevents DB entries without underlying blob storage data in case of a partial update failure Signed-off-by: Nick Müller Signed-off-by: Nick Müller --- Dockerfile | 24 +- boilerplate/update.cfg | 1 - go.mod | 42 ++- go.sum | 90 +++-- pkg/manager/impl/artifact_data_store.go | 10 + pkg/manager/impl/artifact_manager.go | 199 ++++++++-- pkg/manager/impl/artifact_manager_test.go | 357 ++++++++++++++++-- .../impl/validators/artifact_validator.go | 36 ++ pkg/manager/interfaces/artifact.go | 1 + pkg/manager/mocks/artifact_manager.go | 65 +++- pkg/manager/mocks/dataset_manager.go | 24 +- pkg/manager/mocks/reservation_manager.go | 16 +- pkg/manager/mocks/tag_manager.go | 8 +- pkg/repositories/gormimpl/artifact.go | 62 +++ pkg/repositories/gormimpl/artifact_test.go | 211 ++++++++++- pkg/repositories/interfaces/artifact_repo.go | 3 + pkg/repositories/interfaces/dataset_repo.go | 2 + pkg/repositories/interfaces/partition_repo.go | 2 + .../interfaces/reservation_repo.go | 2 + pkg/repositories/interfaces/tag_repo.go | 2 + pkg/repositories/mocks/artifact.go | 73 ---- pkg/repositories/mocks/artifact_repo.go | 160 ++++++++ .../mocks/{dataset.go => dataset_repo.go} | 56 ++- .../mocks/{partition.go => partition_repo.go} | 23 +- .../{reservation.go => reservation_repo.go} | 32 +- .../mocks/{tag.go => tag_repo.go} | 41 +- pkg/rpc/datacatalogservice/service.go | 4 + 27 files changed, 1300 insertions(+), 246 deletions(-) delete mode 100644 pkg/repositories/mocks/artifact.go create mode 100644 pkg/repositories/mocks/artifact_repo.go rename pkg/repositories/mocks/{dataset.go => dataset_repo.go} (51%) rename pkg/repositories/mocks/{partition.go => partition_repo.go} (50%) rename pkg/repositories/mocks/{reservation.go => reservation_repo.go} (84%) rename pkg/repositories/mocks/{tag.go => tag_repo.go} (51%) diff --git a/Dockerfile b/Dockerfile index 0a13a0df..d15054a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,16 @@ FROM golang:1.18-alpine3.15 as builder RUN apk add git openssh-client make curl +# Create the artifacts directory +RUN mkdir /artifacts + +# Pull GRPC health probe binary for liveness and readiness checks +RUN GRPC_HEALTH_PROBE_VERSION=v0.4.11 && \ + wget -qO/artifacts/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /artifacts/grpc_health_probe && \ + echo 'ded15e598d887ccc47bf2321371950bbf930f5e4856b9f75712ce4b2b5120480 /artifacts/grpc_health_probe' > .grpc_checksum && \ + sha256sum -c .grpc_checksum + # COPY only the go mod files for efficient caching COPY go.mod go.sum /go/src/github.com/flyteorg/datacatalog/ WORKDIR /go/src/github.com/flyteorg/datacatalog @@ -20,19 +30,19 @@ COPY . /go/src/github.com/flyteorg/datacatalog/ # The main entrypoint should be compiled to /artifacts/datacatalog RUN make linux_compile -# install grpc-health-probe -RUN curl --silent --fail --show-error --location --output /artifacts/grpc_health_probe "https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.4.5/grpc_health_probe-linux-amd64" && \ - chmod +x /artifacts/grpc_health_probe && \ - echo '8699c46352d752d8f533cae72728b0e65663f399fc28fb9cd854b14ad5f85f44 /artifacts/grpc_health_probe' > .grpc_checksum && \ - sha256sum -c .grpc_checksum - # update the PATH to include the /artifacts directory ENV PATH="/artifacts:${PATH}" # This will eventually move to centurylink/ca-certs:latest for minimum possible image size -FROM alpine:3.14 +FROM alpine:3.16 +LABEL org.opencontainers.image.source=https://github.com/flyteorg/datacatalog + COPY --from=builder /artifacts /bin +# Ensure the latest CA certs are present to authenticate SSL connections. RUN apk --update add ca-certificates +RUN addgroup -S flyte && adduser -S flyte -G flyte +USER flyte + CMD ["datacatalog"] diff --git a/boilerplate/update.cfg b/boilerplate/update.cfg index 15359ff7..9d3bd305 100644 --- a/boilerplate/update.cfg +++ b/boilerplate/update.cfg @@ -2,6 +2,5 @@ flyte/docker_build flyte/golang_test_targets flyte/golangci_file flyte/golang_support_tools -flyte/golang_dockerfile flyte/pull_request_template flyte/ diff --git a/go.mod b/go.mod index 2448b901..5f5dea21 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.18 require ( github.com/Selvatico/go-mocket v1.0.7 - github.com/flyteorg/flyteidl v1.0.0 - github.com/flyteorg/flytestdlib v1.0.5 - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/flyteorg/flyteidl v1.2.3 + github.com/flyteorg/flytestdlib v1.0.11 + github.com/golang/glog v1.0.0 github.com/golang/protobuf v1.5.2 github.com/jackc/pgconn v1.10.1 github.com/mitchellh/mapstructure v1.4.3 @@ -14,42 +14,44 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.1 - google.golang.org/grpc v1.45.0 + google.golang.org/grpc v1.46.0 gorm.io/driver/postgres v1.2.3 gorm.io/driver/sqlite v1.1.1 gorm.io/gorm v1.22.4 ) require ( - cloud.google.com/go v0.100.2 // indirect - cloud.google.com/go/compute v1.5.0 // indirect + cloud.google.com/go v0.101.0 // indirect + cloud.google.com/go/compute v1.6.1 // indirect cloud.google.com/go/iam v0.3.0 // indirect - cloud.google.com/go/storage v1.14.0 // indirect - github.com/Azure/azure-sdk-for-go v62.3.0+incompatible // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect + cloud.google.com/go/storage v1.22.0 // indirect + github.com/Azure/azure-sdk-for-go v63.4.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v0.23.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.25 // indirect + github.com/Azure/go-autorest/autorest v0.11.27 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/aws/aws-sdk-go v1.43.37 // indirect + github.com/aws/aws-sdk-go v1.44.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coocood/freecache v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.13.0 // indirect - github.com/flyteorg/stow v0.3.4 // indirect + github.com/flyteorg/stow v0.3.6 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v0.4.0 // indirect github.com/gofrs/uuid v4.2.0+incompatible // indirect - github.com/golang-jwt/jwt/v4 v4.2.0 // indirect + github.com/golang-jwt/jwt/v4 v4.4.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/go-cmp v0.5.8 // indirect github.com/googleapis/gax-go/v2 v2.3.0 // indirect + github.com/googleapis/go-type-adapters v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect @@ -84,16 +86,16 @@ require ( github.com/stretchr/objx v0.3.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect - golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect + golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect + golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect + golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect - google.golang.org/api v0.74.0 // indirect + google.golang.org/api v0.76.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect + google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 14bd5a47..f3070603 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,9 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.101.0 h1:g+LL+JvpvdyGtcaD2xw2mSByE/6F9s471eJSoaysM84= +cloud.google.com/go v0.101.0/go.mod h1:hEiddgDb77jDQ+I80tURYNJEnuwPzFU8awCFFRLKjW0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -39,8 +40,10 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0 h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= @@ -56,24 +59,30 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= -cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.22.0 h1:NUV0NNp9nkBuW66BFRLuMgldN60C57ET3dhbwLIYio8= +cloud.google.com/go/storage v1.22.0/go.mod h1:GbaLEoMqbVm6sx3Z0R++gSiBlgMv6yUi2q1DeGFKQgE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v62.3.0+incompatible h1:Ctfsn9UoA/BB4HMYQlbPPgNXdX0tZ4tmb85+KFb2+RE= github.com/Azure/azure-sdk-for-go v62.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA= +github.com/Azure/azure-sdk-for-go v63.4.0+incompatible h1:fle3M5Q7vr8auaiPffKyUQmLbvYeqpw30bKU6PrWJFo= +github.com/Azure/azure-sdk-for-go v63.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.23.1 h1:3CVsSo4mp8NDWO11tHzN/mdo2zP0CtaSK5IcwBjfqRA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.23.1/go.mod h1:w5pDIZuawUmY3Bj4tVx3Xb8KS96ToB0j315w9rqpAg0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.14.0 h1:NVS/4LOQfkBpk+B1VopIzv1ptmYeEskA8w/3K/w7vjo= github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.2 h1:Px2KVERcYEg2Lv25AqC2hVr0xUWaq94wuEObLIkYzmA= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.2/go.mod h1:CdSJQNNzZhCkwDaV27XV1w48ZBPtxe7mlrZAsPNxD5g= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0 h1:0nJeKDmB7a1a8RDMjTltahlPsaNlWjq/LpkZleSwINk= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0/go.mod h1:mbwxKc/fW+IkF0GG591MuXw0KuEQBDkeRoZ9vmVJPxg= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= -github.com/Azure/go-autorest/autorest v0.11.25 h1:yp+V8DGur2aIUE87ebP8twPLz6k68jtJTlg61mEoByA= -github.com/Azure/go-autorest/autorest v0.11.25/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= +github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A= +github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.10/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= @@ -96,6 +105,7 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 h1:WVsrXCnHlDDX8ls+tootqRE87/hL9S/g4ewig9RsD/c= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -131,8 +141,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.23.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.37.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.43.37 h1:kyZ7UjaPZaCik+asF33UFOOYSwr9liDRr/UM/vuw8yY= -github.com/aws/aws-sdk-go v1.43.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.2 h1:5VBk5r06bgxgRKVaUtm1/4NT/rtrnH2E4cnAYv5zgQc= +github.com/aws/aws-sdk-go v1.44.2/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1/go.mod h1:jvdWlw8vowVGnZqSDC7yhPd7AifQeQbRDkZcQXV2nRg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -163,6 +173,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= @@ -209,6 +220,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607/go.mod h1:Cg4fM0vhYWOZdgM7RIOSTRNIc8/VT7CXClC3Ni86lu4= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -217,14 +229,14 @@ github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/flyteorg/flyteidl v1.0.0 h1:02V/h8cN3TzI6H9kzB2XNKR4XsJDmsGGfDWxbfmRZGs= -github.com/flyteorg/flyteidl v1.0.0/go.mod h1:JW0z1ZaHS9zWvDAwSMIyGhsf+V4zrzBBgh5IuqzMFCM= +github.com/flyteorg/flyteidl v1.2.3 h1:4A90rFyGXiUtFnQIgSPxPzBZRy9RoAPsfxs7OWYHfFA= +github.com/flyteorg/flyteidl v1.2.3/go.mod h1:f0AFl7RFycH7+JLq2th0ReH7v+Xse+QTw4jGdIxiS8I= github.com/flyteorg/flytestdlib v1.0.0/go.mod h1:QSVN5wIM1lM9d60eAEbX7NwweQXW96t5x4jbyftn89c= -github.com/flyteorg/flytestdlib v1.0.5 h1:80A/vfpAJl+pgU6vxccbsYApZPrvyGhOIsCAFngsjnk= -github.com/flyteorg/flytestdlib v1.0.5/go.mod h1:WTe0k3DmmrKFjj3hwiIbjjdCK89X63MBzBbXhQ4Yxf0= +github.com/flyteorg/flytestdlib v1.0.11 h1:f7B8x2/zMuimEVi4Jx0zqzvNhdi7aq7+ZWoqHsbp4F4= +github.com/flyteorg/flytestdlib v1.0.11/go.mod h1:nIBmBHtjTJvhZEn3e/EwVC/iMkR2tUX8hEiXjRBpH/s= github.com/flyteorg/stow v0.3.3/go.mod h1:HBld7ud0i4khMHwJjkO8v+NSP7ddKa/ruhf4I8fliaA= -github.com/flyteorg/stow v0.3.4 h1:gJVz1LCcEQ5ESWoedRxKh4uUv/V/c1eYLVAQVy07PPY= -github.com/flyteorg/stow v0.3.4/go.mod h1:2T2f6KaIWoWCLgI6EFZQgjb83Vg5SolmBwc2O06WQU4= +github.com/flyteorg/stow v0.3.6 h1:jt50ciM14qhKBaIrB+ppXXY+SXB59FNREFgTJqCyqIk= +github.com/flyteorg/stow v0.3.6/go.mod h1:5dfBitPM004dwaZdoVylVjxFT4GWAgI0ghAndhNUzCo= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= @@ -268,11 +280,14 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= +github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -323,16 +338,18 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -353,6 +370,7 @@ github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -362,6 +380,8 @@ github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/Oth github.com/googleapis/gax-go/v2 v2.3.0 h1:nRJtk3y8Fm770D42QV6T90ZnvFZyk7agSo3Q+Z9p3WI= github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= @@ -457,6 +477,7 @@ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -500,6 +521,7 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -599,6 +621,7 @@ github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -781,8 +804,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -875,8 +898,9 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -992,8 +1016,9 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1129,8 +1154,10 @@ google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tD google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.76.0 h1:UkZl25bR1FHNqtK/EKs3vCdpZtUO6gea3YElTwc8pQg= +google.golang.org/api v0.76.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1186,6 +1213,7 @@ google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= @@ -1214,8 +1242,13 @@ google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac h1:qSNTkEN+L2mvWcLgJOR+8bdHX9rN/IdU3A1Ghpfb1Rg= +google.golang.org/genproto v0.0.0-20220405205423-9d709892a2bf/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46 h1:G1IeWbjrqEq9ChWxEuRPJu6laA67+XgTFHVSAvepr38= +google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1249,8 +1282,9 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/pkg/manager/impl/artifact_data_store.go b/pkg/manager/impl/artifact_data_store.go index 4fbe8314..e4c05ef5 100644 --- a/pkg/manager/impl/artifact_data_store.go +++ b/pkg/manager/impl/artifact_data_store.go @@ -17,6 +17,7 @@ const artifactDataFile = "data.pb" type ArtifactDataStore interface { PutData(ctx context.Context, artifact *datacatalog.Artifact, data *datacatalog.ArtifactData) (storage.DataReference, error) GetData(ctx context.Context, dataModel models.ArtifactData) (*core.Literal, error) + DeleteData(ctx context.Context, dataModel models.ArtifactData) error } type artifactDataStore struct { @@ -54,6 +55,15 @@ func (m *artifactDataStore) GetData(ctx context.Context, dataModel models.Artifa return &value, nil } +// DeleteData removes the stored artifact data from the underlying blob storage +func (m *artifactDataStore) DeleteData(ctx context.Context, dataModel models.ArtifactData) error { + if err := m.store.Delete(ctx, storage.DataReference(dataModel.Location)); err != nil { + return errors.NewDataCatalogErrorf(codes.Internal, "Unable to delete artifact data in location %s, err %v", dataModel.Location, err) + } + + return nil +} + func NewArtifactDataStore(store *storage.DataStore, storagePrefix storage.DataReference) ArtifactDataStore { return &artifactDataStore{ store: store, diff --git a/pkg/manager/impl/artifact_manager.go b/pkg/manager/impl/artifact_manager.go index 7ac97d46..b31686ae 100644 --- a/pkg/manager/impl/artifact_manager.go +++ b/pkg/manager/impl/artifact_manager.go @@ -39,6 +39,13 @@ type artifactMetrics struct { validationErrorCounter labeled.Counter alreadyExistsCounter labeled.Counter doesNotExistCounter labeled.Counter + updateResponseTime labeled.StopWatch + updateSuccessCounter labeled.Counter + updateFailureCounter labeled.Counter + updateDataSuccessCounter labeled.Counter + updateDataFailureCounter labeled.Counter + deleteDataSuccessCounter labeled.Counter + deleteDataFailureCounter labeled.Counter } type artifactManager struct { @@ -137,65 +144,83 @@ func (m *artifactManager) GetArtifact(ctx context.Context, request *datacatalog. datasetID := request.Dataset + artifactModel, err := m.findArtifact(ctx, datasetID, request) + if err != nil { + logger.Errorf(ctx, "Failed to retrieve artifact for get artifact request %v, err: %v", request, err) + m.systemMetrics.getFailureCounter.Inc(ctx) + return nil, err + } + + artifact, err := transformers.FromArtifactModel(artifactModel) + if err != nil { + logger.Errorf(ctx, "Error in transforming get artifact request %+v, err %v", artifactModel, err) + m.systemMetrics.transformerErrorCounter.Inc(ctx) + return nil, err + } + + artifactDataList, err := m.getArtifactDataList(ctx, artifactModel.ArtifactData) + if err != nil { + m.systemMetrics.getFailureCounter.Inc(ctx) + return nil, err + } + artifact.Data = artifactDataList + + logger.Debugf(ctx, "Retrieved artifact dataset %v, id: %v", artifact.Dataset, artifact.Id) + m.systemMetrics.getSuccessCounter.Inc(ctx) + return &datacatalog.GetArtifactResponse{ + Artifact: artifact, + }, nil +} + +type artifactQueryHandle interface { + GetArtifactId() string + GetTagName() string +} + +func (m *artifactManager) findArtifact(ctx context.Context, datasetID *datacatalog.DatasetID, queryHandle artifactQueryHandle) (models.Artifact, error) { var artifactModel models.Artifact - switch request.QueryHandle.(type) { - case *datacatalog.GetArtifactRequest_ArtifactId: - logger.Debugf(ctx, "Get artifact by id %v", request.GetArtifactId()) - artifactKey := transformers.ToArtifactKey(datasetID, request.GetArtifactId()) + + key := queryHandle.GetArtifactId() + if len(key) > 0 { + logger.Debugf(ctx, "Get artifact by id %v", key) + artifactKey := transformers.ToArtifactKey(datasetID, key) + var err error artifactModel, err = m.repo.ArtifactRepo().Get(ctx, artifactKey) if err != nil { if errors.IsDoesNotExistError(err) { - logger.Warnf(ctx, "Artifact does not exist id: %+v, err %v", request.GetArtifactId(), err) + logger.Warnf(ctx, "Artifact does not exist id: %+v, err %v", key, err) m.systemMetrics.doesNotExistCounter.Inc(ctx) } else { - logger.Errorf(ctx, "Unable to retrieve artifact by id: %+v, err %v", request.GetArtifactId(), err) - m.systemMetrics.getFailureCounter.Inc(ctx) + logger.Errorf(ctx, "Unable to retrieve artifact by id: %+v, err %v", key, err) } - return nil, err + return models.Artifact{}, err } - case *datacatalog.GetArtifactRequest_TagName: - logger.Debugf(ctx, "Get artifact by tag %v", request.GetTagName()) - tagKey := transformers.ToTagKey(datasetID, request.GetTagName()) + } else { + key = queryHandle.GetTagName() + + logger.Debugf(ctx, "Get artifact by tag %v", key) + tagKey := transformers.ToTagKey(datasetID, key) tag, err := m.repo.TagRepo().Get(ctx, tagKey) if err != nil { if errors.IsDoesNotExistError(err) { - logger.Infof(ctx, "Artifact does not exist tag: %+v, err %v", request.GetTagName(), err) + logger.Infof(ctx, "Artifact does not exist tag: %+v, err %v", key, err) m.systemMetrics.doesNotExistCounter.Inc(ctx) } else { - logger.Errorf(ctx, "Unable to retrieve Artifact by tag %v, err: %v", request.GetTagName(), err) - m.systemMetrics.getFailureCounter.Inc(ctx) + logger.Errorf(ctx, "Unable to retrieve Artifact by tag %v, err: %v", key, err) } - return nil, err + return models.Artifact{}, err } artifactModel = tag.Artifact } if len(artifactModel.ArtifactData) == 0 { - return nil, errors.NewDataCatalogErrorf(codes.Internal, "artifact [%+v] does not have artifact data associated", request) - } - - artifact, err := transformers.FromArtifactModel(artifactModel) - if err != nil { - logger.Errorf(ctx, "Error in transforming get artifact request %+v, err %v", artifactModel, err) - m.systemMetrics.transformerErrorCounter.Inc(ctx) - return nil, err - } - - artifactDataList, err := m.getArtifactDataList(ctx, artifactModel.ArtifactData) - if err != nil { - m.systemMetrics.getFailureCounter.Inc(ctx) - return nil, err + return models.Artifact{}, errors.NewDataCatalogErrorf(codes.Internal, "artifact [%+v] with key %v does not have artifact data associated", artifactModel, key) } - artifact.Data = artifactDataList - logger.Debugf(ctx, "Retrieved artifact dataset %v, id: %v", artifact.Dataset, artifact.Id) - m.systemMetrics.getSuccessCounter.Inc(ctx) - return &datacatalog.GetArtifactResponse{ - Artifact: artifact, - }, nil + return artifactModel, nil } func (m *artifactManager) getArtifactDataList(ctx context.Context, artifactDataModels []models.ArtifactData) ([]*datacatalog.ArtifactData, error) { @@ -283,6 +308,103 @@ func (m *artifactManager) ListArtifacts(ctx context.Context, request *datacatalo return &datacatalog.ListArtifactsResponse{Artifacts: artifactsList, NextToken: token}, nil } +// UpdateArtifact updates the given artifact, currently only allowing the associated ArtifactData to be replaced. All +// stored data will be overwritten in the underlying blob storage, no longer existing data (based on ArtifactData name) +// will be deleted. +func (m *artifactManager) UpdateArtifact(ctx context.Context, request *datacatalog.UpdateArtifactRequest) (*datacatalog.UpdateArtifactResponse, error) { + ctx = contextutils.WithProjectDomain(ctx, request.Dataset.Project, request.Dataset.Domain) + + timer := m.systemMetrics.updateResponseTime.Start(ctx) + defer timer.Stop() + + err := validators.ValidateUpdateArtifactRequest(request) + if err != nil { + logger.Warningf(ctx, "Invalid update artifact request %v, err: %v", request, err) + m.systemMetrics.validationErrorCounter.Inc(ctx) + m.systemMetrics.updateFailureCounter.Inc(ctx) + return nil, err + } + + // artifact must already exist, verify first + artifactModel, err := m.findArtifact(ctx, request.GetDataset(), request) + if err != nil { + logger.Errorf(ctx, "Failed to get artifact for update artifact request %v, err: %v", request, err) + m.systemMetrics.updateFailureCounter.Inc(ctx) + return nil, err + } + + artifact, err := transformers.FromArtifactModel(artifactModel) + if err != nil { + logger.Errorf(ctx, "Error in transforming update artifact request %+v, err %v", artifactModel, err) + m.systemMetrics.transformerErrorCounter.Inc(ctx) + m.systemMetrics.updateFailureCounter.Inc(ctx) + return nil, err + } + + // overwrite existing artifact data and upload new entries, building a map of artifact data names to remove + // deleted entries from the blob storage after the upload completed + artifactDataNames := make(map[string]struct{}) + artifactDataModels := make([]models.ArtifactData, len(request.Data)) + for i, artifactData := range request.Data { + artifactDataNames[artifactData.Name] = struct{}{} + + dataLocation, err := m.artifactStore.PutData(ctx, artifact, artifactData) + if err != nil { + logger.Errorf(ctx, "Failed to store artifact data during update, err: %v", err) + m.systemMetrics.updateDataFailureCounter.Inc(ctx) + m.systemMetrics.updateFailureCounter.Inc(ctx) + return nil, err + } + + artifactDataModels[i].Name = artifactData.Name + artifactDataModels[i].Location = dataLocation.String() + m.systemMetrics.updateDataSuccessCounter.Inc(ctx) + } + + removedArtifactData := make([]models.ArtifactData, 0) + for _, artifactData := range artifactModel.ArtifactData { + if _, ok := artifactDataNames[artifactData.Name]; !ok { + removedArtifactData = append(removedArtifactData, artifactData) + } + } + + // update artifact in DB, also replaces/upserts associated artifact data + artifactModel.ArtifactData = artifactDataModels + err = m.repo.ArtifactRepo().Update(ctx, artifactModel) + if err != nil { + if errors.IsDoesNotExistError(err) { + logger.Warnf(ctx, "Artifact does not exist key: %+v, err %v", artifact.Id, err) + m.systemMetrics.doesNotExistCounter.Inc(ctx) + } else { + logger.Errorf(ctx, "Failed to update artifact %v, err: %v", artifactModel, err) + } + m.systemMetrics.updateFailureCounter.Inc(ctx) + return nil, err + } + + // delete all artifact data no longer present in the updated artifact from the blob storage. + // blob storage data is removed last in case the DB update fail, which would leave us with artifact data DB entries + // without underlying blob data. this might still leave orphaned data in blob storage, however we can more easily + // clean that up periodically and don't risk serving artifact data records that will fail when retrieved. + for _, artifactData := range removedArtifactData { + if err := m.artifactStore.DeleteData(ctx, artifactData); err != nil { + logger.Errorf(ctx, "Failed to delete artifact data during update, err: %v", err) + m.systemMetrics.deleteDataFailureCounter.Inc(ctx) + m.systemMetrics.updateFailureCounter.Inc(ctx) + return nil, err + } + + m.systemMetrics.deleteDataSuccessCounter.Inc(ctx) + } + + logger.Debugf(ctx, "Successfully updated artifact id: %v", artifact.Id) + + m.systemMetrics.updateSuccessCounter.Inc(ctx) + return &datacatalog.UpdateArtifactResponse{ + ArtifactId: artifact.Id, + }, nil +} + func NewArtifactManager(repo repositories.RepositoryInterface, store *storage.DataStore, storagePrefix storage.DataReference, artifactScope promutils.Scope) interfaces.ArtifactManager { artifactMetrics := artifactMetrics{ scope: artifactScope, @@ -300,6 +422,13 @@ func NewArtifactManager(repo repositories.RepositoryInterface, store *storage.Da doesNotExistCounter: labeled.NewCounter("does_not_exists_count", "The number of times an artifact was not found", artifactScope, labeled.EmitUnlabeledMetric), listSuccessCounter: labeled.NewCounter("list_success_count", "The number of times list artifact succeeded", artifactScope, labeled.EmitUnlabeledMetric), listFailureCounter: labeled.NewCounter("list_failure_count", "The number of times list artifact failed", artifactScope, labeled.EmitUnlabeledMetric), + updateResponseTime: labeled.NewStopWatch("update_duration", "The duration of the update artifact calls.", time.Millisecond, artifactScope, labeled.EmitUnlabeledMetric), + updateSuccessCounter: labeled.NewCounter("update_success_count", "The number of times update artifact succeeded", artifactScope, labeled.EmitUnlabeledMetric), + updateFailureCounter: labeled.NewCounter("update_failure_count", "The number of times update artifact failed", artifactScope, labeled.EmitUnlabeledMetric), + updateDataSuccessCounter: labeled.NewCounter("update_data_success_count", "The number of times update artifact data succeeded", artifactScope, labeled.EmitUnlabeledMetric), + updateDataFailureCounter: labeled.NewCounter("update_data_failure_count", "The number of times update artifact data failed", artifactScope, labeled.EmitUnlabeledMetric), + deleteDataSuccessCounter: labeled.NewCounter("delete_data_success_count", "The number of times delete artifact data succeeded", artifactScope, labeled.EmitUnlabeledMetric), + deleteDataFailureCounter: labeled.NewCounter("delete_data_failure_count", "The number of times delete artifact data failed", artifactScope, labeled.EmitUnlabeledMetric), } return &artifactManager{ diff --git a/pkg/manager/impl/artifact_manager_test.go b/pkg/manager/impl/artifact_manager_test.go index 386e15a5..e88ef09c 100644 --- a/pkg/manager/impl/artifact_manager_test.go +++ b/pkg/manager/impl/artifact_manager_test.go @@ -2,6 +2,8 @@ package impl import ( "context" + stdErrors "errors" + "os" "testing" "time" @@ -9,6 +11,7 @@ import ( "github.com/flyteorg/datacatalog/pkg/common" "github.com/flyteorg/datacatalog/pkg/errors" + repoErrors "github.com/flyteorg/datacatalog/pkg/repositories/errors" "github.com/flyteorg/datacatalog/pkg/repositories/mocks" "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -39,11 +42,15 @@ func createInmemoryDataStore(t testing.TB, scope mockScope.Scope) *storage.DataS } func getTestStringLiteral() *core.Literal { + return getTestStringLiteralWithValue("value1") +} + +func getTestStringLiteralWithValue(val string) *core.Literal { return &core.Literal{ Value: &core.Literal_Scalar{ Scalar: &core.Scalar{ Value: &core.Scalar_Primitive{ - Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "value1"}}, + Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: val}}, }, }, }, @@ -93,23 +100,35 @@ func newMockDataCatalogRepo() *mocks.DataCatalogRepo { MockDatasetRepo: &mocks.DatasetRepo{}, MockArtifactRepo: &mocks.ArtifactRepo{}, MockReservationRepo: &mocks.ReservationRepo{}, + MockTagRepo: &mocks.TagRepo{}, } } func getExpectedDatastoreLocation(ctx context.Context, store *storage.DataStore, prefix storage.DataReference, artifact *datacatalog.Artifact, idx int) (storage.DataReference, error) { + return getExpectedDatastoreLocationFromName(ctx, store, prefix, artifact, artifact.Data[idx].Name) +} + +func getExpectedDatastoreLocationFromName(ctx context.Context, store *storage.DataStore, prefix storage.DataReference, artifact *datacatalog.Artifact, artifactDataName string) (storage.DataReference, error) { dataset := artifact.Dataset - return store.ConstructReference(ctx, prefix, dataset.Project, dataset.Domain, dataset.Name, dataset.Version, artifact.Id, artifact.Data[idx].Name, artifactDataFile) + return store.ConstructReference(ctx, prefix, dataset.Project, dataset.Domain, dataset.Name, dataset.Version, artifact.Id, artifactDataName, artifactDataFile) } func getExpectedArtifactModel(ctx context.Context, t *testing.T, datastore *storage.DataStore, artifact *datacatalog.Artifact) models.Artifact { expectedDataset := artifact.Dataset + + artifactData := make([]models.ArtifactData, len(artifact.Data)) // Write sample artifact data to the expected location and see if the retrieved data matches - testStoragePrefix, err := datastore.ConstructReference(ctx, datastore.GetBaseContainerFQN(ctx), "test") - assert.NoError(t, err) - dataLocation, err := getExpectedDatastoreLocation(ctx, datastore, testStoragePrefix, artifact, 0) - assert.NoError(t, err) - err = datastore.WriteProtobuf(ctx, dataLocation, storage.Options{}, getTestStringLiteral()) - assert.NoError(t, err) + for i := range artifact.Data { + testStoragePrefix, err := datastore.ConstructReference(ctx, datastore.GetBaseContainerFQN(ctx), "test") + assert.NoError(t, err) + dataLocation, err := getExpectedDatastoreLocation(ctx, datastore, testStoragePrefix, artifact, i) + assert.NoError(t, err) + err = datastore.WriteProtobuf(ctx, dataLocation, storage.Options{}, artifact.Data[i].Value) + assert.NoError(t, err) + + artifactData[i].Name = artifact.Data[i].Name + artifactData[i].Location = dataLocation.String() + } // construct the artifact model we will return on the queries serializedMetadata, err := proto.Marshal(artifact.Metadata) @@ -129,10 +148,8 @@ func getExpectedArtifactModel(ctx context.Context, t *testing.T, datastore *stor DatasetName: expectedDataset.Name, ArtifactID: artifact.Id, }, - DatasetUUID: expectedDataset.UUID, - ArtifactData: []models.ArtifactData{ - {Name: "data1", Location: dataLocation.String()}, - }, + DatasetUUID: expectedDataset.UUID, + ArtifactData: artifactData, Dataset: models.Dataset{ DatasetKey: datasetKey, SerializedMetadata: serializedMetadata, @@ -365,17 +382,12 @@ func TestGetArtifact(t *testing.T) { testStoragePrefix, err := datastore.ConstructReference(ctx, datastore.GetBaseContainerFQN(ctx), "test") assert.NoError(t, err) - dcRepo := &mocks.DataCatalogRepo{ - MockDatasetRepo: &mocks.DatasetRepo{}, - MockArtifactRepo: &mocks.ArtifactRepo{}, - MockTagRepo: &mocks.TagRepo{}, - } + dcRepo := newMockDataCatalogRepo() expectedArtifact := getTestArtifact() mockArtifactModel := getExpectedArtifactModel(ctx, t, datastore, expectedArtifact) t.Run("Get by Id", func(t *testing.T) { - dcRepo.MockArtifactRepo.On("Get", mock.Anything, mock.MatchedBy(func(artifactKey models.ArtifactKey) bool { return artifactKey.ArtifactID == expectedArtifact.Id && @@ -454,11 +466,7 @@ func TestListArtifact(t *testing.T) { testStoragePrefix, err := datastore.ConstructReference(ctx, datastore.GetBaseContainerFQN(ctx), "test") assert.NoError(t, err) - dcRepo := &mocks.DataCatalogRepo{ - MockDatasetRepo: &mocks.DatasetRepo{}, - MockArtifactRepo: &mocks.ArtifactRepo{}, - MockTagRepo: &mocks.TagRepo{}, - } + dcRepo := newMockDataCatalogRepo() expectedDataset := getTestDataset() mockDatasetModel := models.Dataset{ @@ -600,3 +608,306 @@ func TestListArtifact(t *testing.T) { assert.NotEmpty(t, artifactResponse) }) } + +func TestUpdateArtifact(t *testing.T) { + ctx := context.Background() + datastore := createInmemoryDataStore(t, mockScope.NewTestScope()) + testStoragePrefix, err := datastore.ConstructReference(ctx, datastore.GetBaseContainerFQN(ctx), "test") + assert.NoError(t, err) + + expectedDataset := getTestDataset() + expectedArtifact := getTestArtifact() + expectedArtifact.Data = append(expectedArtifact.Data, &datacatalog.ArtifactData{ + Name: "data2", + Value: getTestStringLiteralWithValue("value2"), + }) + expectedTag := getTestTag() + + t.Run("Update by ID", func(t *testing.T) { + ctx := context.Background() + datastore := createInmemoryDataStore(t, mockScope.NewTestScope()) + mockArtifactModel := getExpectedArtifactModel(ctx, t, datastore, expectedArtifact) + + dcRepo := newMockDataCatalogRepo() + dcRepo.MockArtifactRepo.On("Get", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(artifactKey models.ArtifactKey) bool { + return artifactKey.ArtifactID == expectedArtifact.Id && + artifactKey.DatasetProject == expectedArtifact.Dataset.Project && + artifactKey.DatasetDomain == expectedArtifact.Dataset.Domain && + artifactKey.DatasetName == expectedArtifact.Dataset.Name && + artifactKey.DatasetVersion == expectedArtifact.Dataset.Version + })).Return(mockArtifactModel, nil) + + dcRepo.MockArtifactRepo.On("Update", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(artifact models.Artifact) bool { + return artifact.ArtifactID == expectedArtifact.Id && + artifact.ArtifactKey.DatasetProject == expectedArtifact.Dataset.Project && + artifact.ArtifactKey.DatasetDomain == expectedArtifact.Dataset.Domain && + artifact.ArtifactKey.DatasetName == expectedArtifact.Dataset.Name && + artifact.ArtifactKey.DatasetVersion == expectedArtifact.Dataset.Version + })).Return(nil) + + request := &datacatalog.UpdateArtifactRequest{ + Dataset: expectedDataset.Id, + QueryHandle: &datacatalog.UpdateArtifactRequest_ArtifactId{ + ArtifactId: expectedArtifact.Id, + }, + Data: []*datacatalog.ArtifactData{ + { + Name: "data1", + Value: getTestStringLiteralWithValue("value11"), + }, + { + Name: "data3", + Value: getTestStringLiteralWithValue("value3"), + }, + }, + } + + artifactManager := NewArtifactManager(dcRepo, datastore, testStoragePrefix, mockScope.NewTestScope()) + artifactResponse, err := artifactManager.UpdateArtifact(ctx, request) + assert.NoError(t, err) + assert.NotNil(t, artifactResponse) + assert.Equal(t, expectedArtifact.Id, artifactResponse.GetArtifactId()) + + // check that the datastore has the updated artifactData available + // data1 should contain updated value + dataRef, err := getExpectedDatastoreLocationFromName(ctx, datastore, testStoragePrefix, expectedArtifact, "data1") + assert.NoError(t, err) + var value core.Literal + err = datastore.ReadProtobuf(ctx, dataRef, &value) + assert.NoError(t, err) + assert.Equal(t, value, *getTestStringLiteralWithValue("value11")) + + // data2 was not included in update payload, should be removed + dataRef, err = getExpectedDatastoreLocationFromName(ctx, datastore, testStoragePrefix, expectedArtifact, "data2") + assert.NoError(t, err) + err = datastore.ReadProtobuf(ctx, dataRef, &value) + assert.Error(t, err) + assert.True(t, stdErrors.Is(err, os.ErrNotExist)) + + // data3 did not exist before, should be present after update + dataRef, err = getExpectedDatastoreLocationFromName(ctx, datastore, testStoragePrefix, expectedArtifact, "data3") + assert.NoError(t, err) + err = datastore.ReadProtobuf(ctx, dataRef, &value) + assert.NoError(t, err) + assert.Equal(t, value, *getTestStringLiteralWithValue("value3")) + }) + + t.Run("Update by artifact tag", func(t *testing.T) { + ctx := context.Background() + datastore := createInmemoryDataStore(t, mockScope.NewTestScope()) + mockArtifactModel := getExpectedArtifactModel(ctx, t, datastore, expectedArtifact) + + dcRepo := newMockDataCatalogRepo() + dcRepo.MockArtifactRepo.On("Update", + mock.MatchedBy(func(ctx context.Context) bool { return true }), + mock.MatchedBy(func(artifact models.Artifact) bool { + return artifact.ArtifactID == expectedArtifact.Id && + artifact.ArtifactKey.DatasetProject == expectedArtifact.Dataset.Project && + artifact.ArtifactKey.DatasetDomain == expectedArtifact.Dataset.Domain && + artifact.ArtifactKey.DatasetName == expectedArtifact.Dataset.Name && + artifact.ArtifactKey.DatasetVersion == expectedArtifact.Dataset.Version + })).Return(nil) + + dcRepo.MockTagRepo.On("Get", mock.Anything, + mock.MatchedBy(func(tag models.TagKey) bool { + return tag.TagName == expectedTag.TagName && + tag.DatasetProject == expectedTag.DatasetProject && + tag.DatasetDomain == expectedTag.DatasetDomain && + tag.DatasetVersion == expectedTag.DatasetVersion && + tag.DatasetName == expectedTag.DatasetName + })).Return(models.Tag{ + TagKey: models.TagKey{ + DatasetProject: expectedTag.DatasetProject, + DatasetDomain: expectedTag.DatasetDomain, + DatasetName: expectedTag.DatasetName, + DatasetVersion: expectedTag.DatasetVersion, + TagName: expectedTag.TagName, + }, + DatasetUUID: expectedTag.DatasetUUID, + Artifact: mockArtifactModel, + ArtifactID: mockArtifactModel.ArtifactID, + }, nil) + + request := &datacatalog.UpdateArtifactRequest{ + Dataset: expectedDataset.Id, + QueryHandle: &datacatalog.UpdateArtifactRequest_TagName{ + TagName: expectedTag.TagName, + }, + Data: []*datacatalog.ArtifactData{ + { + Name: "data1", + Value: getTestStringLiteralWithValue("value11"), + }, + { + Name: "data3", + Value: getTestStringLiteralWithValue("value3"), + }, + }, + } + + artifactManager := NewArtifactManager(dcRepo, datastore, testStoragePrefix, mockScope.NewTestScope()) + artifactResponse, err := artifactManager.UpdateArtifact(ctx, request) + assert.NoError(t, err) + assert.NotNil(t, artifactResponse) + assert.Equal(t, expectedArtifact.Id, artifactResponse.GetArtifactId()) + + // check that the datastore has the updated artifactData available + // data1 should contain updated value + dataRef, err := getExpectedDatastoreLocationFromName(ctx, datastore, testStoragePrefix, expectedArtifact, "data1") + assert.NoError(t, err) + var value core.Literal + err = datastore.ReadProtobuf(ctx, dataRef, &value) + assert.NoError(t, err) + assert.Equal(t, value, *getTestStringLiteralWithValue("value11")) + + // data2 was not included in update payload, should be removed + dataRef, err = getExpectedDatastoreLocationFromName(ctx, datastore, testStoragePrefix, expectedArtifact, "data2") + assert.NoError(t, err) + err = datastore.ReadProtobuf(ctx, dataRef, &value) + assert.Error(t, err) + assert.True(t, stdErrors.Is(err, os.ErrNotExist)) + + // data3 did not exist before, should be present after update + dataRef, err = getExpectedDatastoreLocationFromName(ctx, datastore, testStoragePrefix, expectedArtifact, "data3") + assert.NoError(t, err) + err = datastore.ReadProtobuf(ctx, dataRef, &value) + assert.NoError(t, err) + assert.Equal(t, value, *getTestStringLiteralWithValue("value3")) + }) + + t.Run("Artifact not found", func(t *testing.T) { + ctx := context.Background() + datastore := createInmemoryDataStore(t, mockScope.NewTestScope()) + + dcRepo := newMockDataCatalogRepo() + dcRepo.MockArtifactRepo.On("Get", mock.Anything, mock.Anything).Return(models.Artifact{}, repoErrors.GetMissingEntityError("Artifact", &datacatalog.Artifact{ + Dataset: expectedDataset.Id, + Id: expectedArtifact.Id, + })) + + request := &datacatalog.UpdateArtifactRequest{ + Dataset: expectedDataset.Id, + QueryHandle: &datacatalog.UpdateArtifactRequest_ArtifactId{ + ArtifactId: expectedArtifact.Id, + }, + Data: []*datacatalog.ArtifactData{ + { + Name: "data1", + Value: getTestStringLiteralWithValue("value11"), + }, + { + Name: "data3", + Value: getTestStringLiteralWithValue("value3"), + }, + }, + } + + artifactManager := NewArtifactManager(dcRepo, datastore, testStoragePrefix, mockScope.NewTestScope()) + artifactResponse, err := artifactManager.UpdateArtifact(ctx, request) + assert.Error(t, err) + assert.Equal(t, codes.NotFound, status.Code(err)) + assert.Nil(t, artifactResponse) + }) + + t.Run("Missing artifact ID", func(t *testing.T) { + ctx := context.Background() + datastore := createInmemoryDataStore(t, mockScope.NewTestScope()) + + dcRepo := newMockDataCatalogRepo() + + request := &datacatalog.UpdateArtifactRequest{ + Dataset: expectedDataset.Id, + QueryHandle: &datacatalog.UpdateArtifactRequest_ArtifactId{}, + Data: []*datacatalog.ArtifactData{ + { + Name: "data1", + Value: getTestStringLiteralWithValue("value11"), + }, + { + Name: "data3", + Value: getTestStringLiteralWithValue("value3"), + }, + }, + } + + artifactManager := NewArtifactManager(dcRepo, datastore, testStoragePrefix, mockScope.NewTestScope()) + artifactResponse, err := artifactManager.UpdateArtifact(ctx, request) + assert.Error(t, err) + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + assert.Nil(t, artifactResponse) + }) + + t.Run("Missing artifact tag", func(t *testing.T) { + ctx := context.Background() + datastore := createInmemoryDataStore(t, mockScope.NewTestScope()) + + dcRepo := newMockDataCatalogRepo() + + request := &datacatalog.UpdateArtifactRequest{ + Dataset: expectedDataset.Id, + QueryHandle: &datacatalog.UpdateArtifactRequest_TagName{}, + Data: []*datacatalog.ArtifactData{ + { + Name: "data1", + Value: getTestStringLiteralWithValue("value11"), + }, + { + Name: "data3", + Value: getTestStringLiteralWithValue("value3"), + }, + }, + } + + artifactManager := NewArtifactManager(dcRepo, datastore, testStoragePrefix, mockScope.NewTestScope()) + artifactResponse, err := artifactManager.UpdateArtifact(ctx, request) + assert.Error(t, err) + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + assert.Nil(t, artifactResponse) + }) + + t.Run("Missing artifact data", func(t *testing.T) { + ctx := context.Background() + datastore := createInmemoryDataStore(t, mockScope.NewTestScope()) + + dcRepo := newMockDataCatalogRepo() + + request := &datacatalog.UpdateArtifactRequest{ + Dataset: expectedDataset.Id, + QueryHandle: &datacatalog.UpdateArtifactRequest_ArtifactId{ + ArtifactId: expectedArtifact.Id, + }, + Data: nil, + } + + artifactManager := NewArtifactManager(dcRepo, datastore, testStoragePrefix, mockScope.NewTestScope()) + artifactResponse, err := artifactManager.UpdateArtifact(ctx, request) + assert.Error(t, err) + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + assert.Nil(t, artifactResponse) + }) + + t.Run("Empty artifact data", func(t *testing.T) { + ctx := context.Background() + datastore := createInmemoryDataStore(t, mockScope.NewTestScope()) + + dcRepo := newMockDataCatalogRepo() + + request := &datacatalog.UpdateArtifactRequest{ + Dataset: expectedDataset.Id, + QueryHandle: &datacatalog.UpdateArtifactRequest_ArtifactId{ + ArtifactId: expectedArtifact.Id, + }, + Data: []*datacatalog.ArtifactData{}, + } + + artifactManager := NewArtifactManager(dcRepo, datastore, testStoragePrefix, mockScope.NewTestScope()) + artifactResponse, err := artifactManager.UpdateArtifact(ctx, request) + assert.Error(t, err) + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + assert.Nil(t, artifactResponse) + }) +} diff --git a/pkg/manager/impl/validators/artifact_validator.go b/pkg/manager/impl/validators/artifact_validator.go index bdc2c549..1b3265a7 100644 --- a/pkg/manager/impl/validators/artifact_validator.go +++ b/pkg/manager/impl/validators/artifact_validator.go @@ -102,3 +102,39 @@ func ValidateArtifactFilterTypes(filters []*datacatalog.SinglePropertyFilter) er } return nil } + +func ValidateUpdateArtifactRequest(request *datacatalog.UpdateArtifactRequest) error { + if request.QueryHandle == nil { + return NewMissingArgumentError(fmt.Sprintf("one of %s/%s", artifactID, tagName)) + } + + switch request.QueryHandle.(type) { + case *datacatalog.UpdateArtifactRequest_ArtifactId: + if request.Dataset != nil { + err := ValidateDatasetID(request.Dataset) + if err != nil { + return err + } + } + + if err := ValidateEmptyStringField(request.GetArtifactId(), artifactID); err != nil { + return err + } + case *datacatalog.UpdateArtifactRequest_TagName: + if err := ValidateDatasetID(request.Dataset); err != nil { + return err + } + + if err := ValidateEmptyStringField(request.GetTagName(), tagName); err != nil { + return err + } + default: + return NewInvalidArgumentError("QueryHandle", "invalid type") + } + + if err := ValidateEmptyArtifactData(request.Data); err != nil { + return err + } + + return nil +} diff --git a/pkg/manager/interfaces/artifact.go b/pkg/manager/interfaces/artifact.go index ae80f13f..3fdae662 100644 --- a/pkg/manager/interfaces/artifact.go +++ b/pkg/manager/interfaces/artifact.go @@ -12,4 +12,5 @@ type ArtifactManager interface { CreateArtifact(ctx context.Context, request *idl_datacatalog.CreateArtifactRequest) (*idl_datacatalog.CreateArtifactResponse, error) GetArtifact(ctx context.Context, request *idl_datacatalog.GetArtifactRequest) (*idl_datacatalog.GetArtifactResponse, error) ListArtifacts(ctx context.Context, request *idl_datacatalog.ListArtifactsRequest) (*idl_datacatalog.ListArtifactsResponse, error) + UpdateArtifact(ctx context.Context, request *idl_datacatalog.UpdateArtifactRequest) (*idl_datacatalog.UpdateArtifactResponse, error) } diff --git a/pkg/manager/mocks/artifact_manager.go b/pkg/manager/mocks/artifact_manager.go index 78134c60..98cacf02 100644 --- a/pkg/manager/mocks/artifact_manager.go +++ b/pkg/manager/mocks/artifact_manager.go @@ -24,13 +24,13 @@ func (_m ArtifactManager_CreateArtifact) Return(_a0 *datacatalog.CreateArtifactR } func (_m *ArtifactManager) OnCreateArtifact(ctx context.Context, request *datacatalog.CreateArtifactRequest) *ArtifactManager_CreateArtifact { - c := _m.On("CreateArtifact", ctx, request) - return &ArtifactManager_CreateArtifact{Call: c} + c_call := _m.On("CreateArtifact", ctx, request) + return &ArtifactManager_CreateArtifact{Call: c_call} } func (_m *ArtifactManager) OnCreateArtifactMatch(matchers ...interface{}) *ArtifactManager_CreateArtifact { - c := _m.On("CreateArtifact", matchers...) - return &ArtifactManager_CreateArtifact{Call: c} + c_call := _m.On("CreateArtifact", matchers...) + return &ArtifactManager_CreateArtifact{Call: c_call} } // CreateArtifact provides a mock function with given fields: ctx, request @@ -65,13 +65,13 @@ func (_m ArtifactManager_GetArtifact) Return(_a0 *datacatalog.GetArtifactRespons } func (_m *ArtifactManager) OnGetArtifact(ctx context.Context, request *datacatalog.GetArtifactRequest) *ArtifactManager_GetArtifact { - c := _m.On("GetArtifact", ctx, request) - return &ArtifactManager_GetArtifact{Call: c} + c_call := _m.On("GetArtifact", ctx, request) + return &ArtifactManager_GetArtifact{Call: c_call} } func (_m *ArtifactManager) OnGetArtifactMatch(matchers ...interface{}) *ArtifactManager_GetArtifact { - c := _m.On("GetArtifact", matchers...) - return &ArtifactManager_GetArtifact{Call: c} + c_call := _m.On("GetArtifact", matchers...) + return &ArtifactManager_GetArtifact{Call: c_call} } // GetArtifact provides a mock function with given fields: ctx, request @@ -106,13 +106,13 @@ func (_m ArtifactManager_ListArtifacts) Return(_a0 *datacatalog.ListArtifactsRes } func (_m *ArtifactManager) OnListArtifacts(ctx context.Context, request *datacatalog.ListArtifactsRequest) *ArtifactManager_ListArtifacts { - c := _m.On("ListArtifacts", ctx, request) - return &ArtifactManager_ListArtifacts{Call: c} + c_call := _m.On("ListArtifacts", ctx, request) + return &ArtifactManager_ListArtifacts{Call: c_call} } func (_m *ArtifactManager) OnListArtifactsMatch(matchers ...interface{}) *ArtifactManager_ListArtifacts { - c := _m.On("ListArtifacts", matchers...) - return &ArtifactManager_ListArtifacts{Call: c} + c_call := _m.On("ListArtifacts", matchers...) + return &ArtifactManager_ListArtifacts{Call: c_call} } // ListArtifacts provides a mock function with given fields: ctx, request @@ -137,3 +137,44 @@ func (_m *ArtifactManager) ListArtifacts(ctx context.Context, request *datacatal return r0, r1 } + +type ArtifactManager_UpdateArtifact struct { + *mock.Call +} + +func (_m ArtifactManager_UpdateArtifact) Return(_a0 *datacatalog.UpdateArtifactResponse, _a1 error) *ArtifactManager_UpdateArtifact { + return &ArtifactManager_UpdateArtifact{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *ArtifactManager) OnUpdateArtifact(ctx context.Context, request *datacatalog.UpdateArtifactRequest) *ArtifactManager_UpdateArtifact { + c_call := _m.On("UpdateArtifact", ctx, request) + return &ArtifactManager_UpdateArtifact{Call: c_call} +} + +func (_m *ArtifactManager) OnUpdateArtifactMatch(matchers ...interface{}) *ArtifactManager_UpdateArtifact { + c_call := _m.On("UpdateArtifact", matchers...) + return &ArtifactManager_UpdateArtifact{Call: c_call} +} + +// UpdateArtifact provides a mock function with given fields: ctx, request +func (_m *ArtifactManager) UpdateArtifact(ctx context.Context, request *datacatalog.UpdateArtifactRequest) (*datacatalog.UpdateArtifactResponse, error) { + ret := _m.Called(ctx, request) + + var r0 *datacatalog.UpdateArtifactResponse + if rf, ok := ret.Get(0).(func(context.Context, *datacatalog.UpdateArtifactRequest) *datacatalog.UpdateArtifactResponse); ok { + r0 = rf(ctx, request) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*datacatalog.UpdateArtifactResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *datacatalog.UpdateArtifactRequest) error); ok { + r1 = rf(ctx, request) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/pkg/manager/mocks/dataset_manager.go b/pkg/manager/mocks/dataset_manager.go index 9127a8e6..4f97dc28 100644 --- a/pkg/manager/mocks/dataset_manager.go +++ b/pkg/manager/mocks/dataset_manager.go @@ -24,13 +24,13 @@ func (_m DatasetManager_CreateDataset) Return(_a0 *datacatalog.CreateDatasetResp } func (_m *DatasetManager) OnCreateDataset(ctx context.Context, request *datacatalog.CreateDatasetRequest) *DatasetManager_CreateDataset { - c := _m.On("CreateDataset", ctx, request) - return &DatasetManager_CreateDataset{Call: c} + c_call := _m.On("CreateDataset", ctx, request) + return &DatasetManager_CreateDataset{Call: c_call} } func (_m *DatasetManager) OnCreateDatasetMatch(matchers ...interface{}) *DatasetManager_CreateDataset { - c := _m.On("CreateDataset", matchers...) - return &DatasetManager_CreateDataset{Call: c} + c_call := _m.On("CreateDataset", matchers...) + return &DatasetManager_CreateDataset{Call: c_call} } // CreateDataset provides a mock function with given fields: ctx, request @@ -65,13 +65,13 @@ func (_m DatasetManager_GetDataset) Return(_a0 *datacatalog.GetDatasetResponse, } func (_m *DatasetManager) OnGetDataset(ctx context.Context, request *datacatalog.GetDatasetRequest) *DatasetManager_GetDataset { - c := _m.On("GetDataset", ctx, request) - return &DatasetManager_GetDataset{Call: c} + c_call := _m.On("GetDataset", ctx, request) + return &DatasetManager_GetDataset{Call: c_call} } func (_m *DatasetManager) OnGetDatasetMatch(matchers ...interface{}) *DatasetManager_GetDataset { - c := _m.On("GetDataset", matchers...) - return &DatasetManager_GetDataset{Call: c} + c_call := _m.On("GetDataset", matchers...) + return &DatasetManager_GetDataset{Call: c_call} } // GetDataset provides a mock function with given fields: ctx, request @@ -106,13 +106,13 @@ func (_m DatasetManager_ListDatasets) Return(_a0 *datacatalog.ListDatasetsRespon } func (_m *DatasetManager) OnListDatasets(ctx context.Context, request *datacatalog.ListDatasetsRequest) *DatasetManager_ListDatasets { - c := _m.On("ListDatasets", ctx, request) - return &DatasetManager_ListDatasets{Call: c} + c_call := _m.On("ListDatasets", ctx, request) + return &DatasetManager_ListDatasets{Call: c_call} } func (_m *DatasetManager) OnListDatasetsMatch(matchers ...interface{}) *DatasetManager_ListDatasets { - c := _m.On("ListDatasets", matchers...) - return &DatasetManager_ListDatasets{Call: c} + c_call := _m.On("ListDatasets", matchers...) + return &DatasetManager_ListDatasets{Call: c_call} } // ListDatasets provides a mock function with given fields: ctx, request diff --git a/pkg/manager/mocks/reservation_manager.go b/pkg/manager/mocks/reservation_manager.go index cada156e..edde09ad 100644 --- a/pkg/manager/mocks/reservation_manager.go +++ b/pkg/manager/mocks/reservation_manager.go @@ -24,13 +24,13 @@ func (_m ReservationManager_GetOrExtendReservation) Return(_a0 *datacatalog.GetO } func (_m *ReservationManager) OnGetOrExtendReservation(_a0 context.Context, _a1 *datacatalog.GetOrExtendReservationRequest) *ReservationManager_GetOrExtendReservation { - c := _m.On("GetOrExtendReservation", _a0, _a1) - return &ReservationManager_GetOrExtendReservation{Call: c} + c_call := _m.On("GetOrExtendReservation", _a0, _a1) + return &ReservationManager_GetOrExtendReservation{Call: c_call} } func (_m *ReservationManager) OnGetOrExtendReservationMatch(matchers ...interface{}) *ReservationManager_GetOrExtendReservation { - c := _m.On("GetOrExtendReservation", matchers...) - return &ReservationManager_GetOrExtendReservation{Call: c} + c_call := _m.On("GetOrExtendReservation", matchers...) + return &ReservationManager_GetOrExtendReservation{Call: c_call} } // GetOrExtendReservation provides a mock function with given fields: _a0, _a1 @@ -65,13 +65,13 @@ func (_m ReservationManager_ReleaseReservation) Return(_a0 *datacatalog.ReleaseR } func (_m *ReservationManager) OnReleaseReservation(_a0 context.Context, _a1 *datacatalog.ReleaseReservationRequest) *ReservationManager_ReleaseReservation { - c := _m.On("ReleaseReservation", _a0, _a1) - return &ReservationManager_ReleaseReservation{Call: c} + c_call := _m.On("ReleaseReservation", _a0, _a1) + return &ReservationManager_ReleaseReservation{Call: c_call} } func (_m *ReservationManager) OnReleaseReservationMatch(matchers ...interface{}) *ReservationManager_ReleaseReservation { - c := _m.On("ReleaseReservation", matchers...) - return &ReservationManager_ReleaseReservation{Call: c} + c_call := _m.On("ReleaseReservation", matchers...) + return &ReservationManager_ReleaseReservation{Call: c_call} } // ReleaseReservation provides a mock function with given fields: _a0, _a1 diff --git a/pkg/manager/mocks/tag_manager.go b/pkg/manager/mocks/tag_manager.go index ff6b79e5..b48d9cd1 100644 --- a/pkg/manager/mocks/tag_manager.go +++ b/pkg/manager/mocks/tag_manager.go @@ -24,13 +24,13 @@ func (_m TagManager_AddTag) Return(_a0 *datacatalog.AddTagResponse, _a1 error) * } func (_m *TagManager) OnAddTag(ctx context.Context, request *datacatalog.AddTagRequest) *TagManager_AddTag { - c := _m.On("AddTag", ctx, request) - return &TagManager_AddTag{Call: c} + c_call := _m.On("AddTag", ctx, request) + return &TagManager_AddTag{Call: c_call} } func (_m *TagManager) OnAddTagMatch(matchers ...interface{}) *TagManager_AddTag { - c := _m.On("AddTag", matchers...) - return &TagManager_AddTag{Call: c} + c_call := _m.On("AddTag", matchers...) + return &TagManager_AddTag{Call: c_call} } // AddTag provides a mock function with given fields: ctx, request diff --git a/pkg/repositories/gormimpl/artifact.go b/pkg/repositories/gormimpl/artifact.go index 6b1e445a..b64f2f81 100644 --- a/pkg/repositories/gormimpl/artifact.go +++ b/pkg/repositories/gormimpl/artifact.go @@ -11,6 +11,7 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" "github.com/flyteorg/flytestdlib/promutils" "gorm.io/gorm" + "gorm.io/gorm/clause" ) type artifactRepo struct { @@ -118,3 +119,64 @@ func (h *artifactRepo) List(ctx context.Context, datasetKey models.DatasetKey, i } return artifacts, nil } + +// Update updates the given artifact and its associated ArtifactData in database. The ArtifactData entries are upserted +// (ignoring conflicts, as no updates to the database model are to be expected) and any longer existing data is deleted. +func (h *artifactRepo) Update(ctx context.Context, artifact models.Artifact) error { + timer := h.repoMetrics.UpdateDuration.Start(ctx) + defer timer.Stop() + + tx := h.db.Begin() + defer func() { + if r := recover(); r != nil { + tx.Rollback() + } + }() + + if err := tx.Error; err != nil { + return err + } + + // ensure all artifact fields in DB are up-to-date + if res := tx.Model(&models.Artifact{ArtifactKey: artifact.ArtifactKey}).Updates(artifact); res.Error != nil { + tx.Rollback() + return h.errorTransformer.ToDataCatalogError(res.Error) + } else if res.RowsAffected == 0 { + // no rows affected --> artifact not found + tx.Rollback() + return errors.GetMissingEntityError(string(common.Artifact), &datacatalog.Artifact{ + Dataset: &datacatalog.DatasetID{ + Project: artifact.DatasetProject, + Domain: artifact.DatasetDomain, + Name: artifact.DatasetName, + Version: artifact.DatasetVersion, + }, + Id: artifact.ArtifactID, + }) + } + + artifactDataNames := make([]string, len(artifact.ArtifactData)) + for i := range artifact.ArtifactData { + artifactDataNames[i] = artifact.ArtifactData[i].Name + // ensure artifact data is fully associated with correct artifact + artifact.ArtifactData[i].ArtifactKey = artifact.ArtifactKey + } + + // delete all removed artifact data entries from the DB + if err := tx.Where(&models.ArtifactData{ArtifactKey: artifact.ArtifactKey}).Where("name NOT IN ?", artifactDataNames).Delete(&models.ArtifactData{}).Error; err != nil { + tx.Rollback() + return h.errorTransformer.ToDataCatalogError(err) + } + + // upsert artifact data, adding new entries and ignoring conflicts (no actual data changed) + if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(artifact.ArtifactData).Error; err != nil { + tx.Rollback() + return h.errorTransformer.ToDataCatalogError(err) + } + + if err := tx.Commit().Error; err != nil { + return h.errorTransformer.ToDataCatalogError(err) + } + + return nil +} diff --git a/pkg/repositories/gormimpl/artifact_test.go b/pkg/repositories/gormimpl/artifact_test.go index 1bff6f85..ba14f081 100644 --- a/pkg/repositories/gormimpl/artifact_test.go +++ b/pkg/repositories/gormimpl/artifact_test.go @@ -1,9 +1,8 @@ package gormimpl import ( - "testing" - "context" + "testing" mocket "github.com/Selvatico/go-mocket" "github.com/stretchr/testify/assert" @@ -352,3 +351,211 @@ func TestListArtifactsNoPartitions(t *testing.T) { assert.Len(t, artifacts[0].ArtifactData, 1) assert.Len(t, artifacts[0].Partitions, 0) } + +func TestUpdateArtifact(t *testing.T) { + ctx := context.Background() + artifact := getTestArtifact() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + artifactUpdated := false + GlobalMock.NewMock().WithQuery(`UPDATE "artifacts" SET "updated_at"=$1,"artifact_id"=$2 WHERE "artifact_id" = $3`). + WithRowsNum(1). + WithCallback(func(s string, values []driver.NamedValue) { + artifactUpdated = true + }) + artifactDataDeleted := false + GlobalMock.NewMock(). + WithQuery(`DELETE FROM "artifact_data" WHERE "artifact_data"."artifact_id" = $1 AND name NOT IN ($2,$3)`). + WithRowsNum(0). + WithCallback(func(s string, values []driver.NamedValue) { + artifactDataDeleted = true + }) + artifactDataUpserted := false + GlobalMock.NewMock().WithQuery(`INSERT INTO "artifact_data" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","name","location") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10),($11,$12,$13,$14,$15,$16,$17,$18,$19,$20) ON CONFLICT DO NOTHING`). + WithRowsNum(1). + WithCallback(func(s string, values []driver.NamedValue) { + artifactDataUpserted = true + }) + + updateInput := models.Artifact{ + ArtifactKey: models.ArtifactKey{ + ArtifactID: artifact.ArtifactID, + }, + ArtifactData: []models.ArtifactData{ + { + Name: "test-dataloc-name", + Location: "test-dataloc-location", + }, + { + Name: "additional-test-dataloc-name", + Location: "additional-test-dataloc-location", + }, + }, + } + + artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + err := artifactRepo.Update(ctx, updateInput) + assert.NoError(t, err) + assert.True(t, artifactUpdated) + assert.True(t, artifactDataDeleted) + assert.True(t, artifactDataUpserted) +} + +func TestUpdateArtifactDoesNotExist(t *testing.T) { + ctx := context.Background() + artifact := getTestArtifact() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + updateInput := models.Artifact{ + ArtifactKey: models.ArtifactKey{ + ArtifactID: artifact.ArtifactID, + }, + ArtifactData: []models.ArtifactData{ + { + Name: "test-dataloc-name", + Location: "test-dataloc-location", + }, + { + Name: "additional-test-dataloc-name", + Location: "additional-test-dataloc-location", + }, + }, + } + + artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + err := artifactRepo.Update(ctx, updateInput) + assert.Error(t, err) + dcErr, ok := err.(apiErrors.DataCatalogError) + assert.True(t, ok) + assert.Equal(t, dcErr.Code(), codes.NotFound) +} + +func TestUpdateArtifactError(t *testing.T) { + artifact := getTestArtifact() + + t.Run("ArtifactUpdate", func(t *testing.T) { + ctx := context.Background() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + GlobalMock.NewMock().WithQuery(`UPDATE "artifacts" SET "updated_at"=$1,"artifact_id"=$2 WHERE "artifact_id" = $3`). + WithExecException() + + updateInput := models.Artifact{ + ArtifactKey: models.ArtifactKey{ + ArtifactID: artifact.ArtifactID, + }, + ArtifactData: []models.ArtifactData{ + { + Name: "test-dataloc-name", + Location: "test-dataloc-location", + }, + { + Name: "additional-test-dataloc-name", + Location: "additional-test-dataloc-location", + }, + }, + } + + artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + err := artifactRepo.Update(ctx, updateInput) + assert.Error(t, err) + dcErr, ok := err.(apiErrors.DataCatalogError) + assert.True(t, ok) + assert.Equal(t, dcErr.Code(), codes.Internal) + }) + + t.Run("ArtifactDataDelete", func(t *testing.T) { + ctx := context.Background() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + artifactUpdated := false + GlobalMock.NewMock().WithQuery(`UPDATE "artifacts" SET "updated_at"=$1,"artifact_id"=$2 WHERE "artifact_id" = $3`). + WithRowsNum(1). + WithCallback(func(s string, values []driver.NamedValue) { + artifactUpdated = true + }) + GlobalMock.NewMock(). + WithQuery(`DELETE FROM "artifact_data" WHERE "artifact_data"."artifact_id" = $1 AND name NOT IN ($2,$3)`). + WithExecException() + + updateInput := models.Artifact{ + ArtifactKey: models.ArtifactKey{ + ArtifactID: artifact.ArtifactID, + }, + ArtifactData: []models.ArtifactData{ + { + Name: "test-dataloc-name", + Location: "test-dataloc-location", + }, + { + Name: "additional-test-dataloc-name", + Location: "additional-test-dataloc-location", + }, + }, + } + + artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + err := artifactRepo.Update(ctx, updateInput) + assert.Error(t, err) + dcErr, ok := err.(apiErrors.DataCatalogError) + assert.True(t, ok) + assert.Equal(t, dcErr.Code(), codes.Internal) + assert.True(t, artifactUpdated) + }) + + t.Run("ArtifactDataUpsert", func(t *testing.T) { + ctx := context.Background() + + GlobalMock := mocket.Catcher.Reset() + GlobalMock.Logging = true + + artifactUpdated := false + GlobalMock.NewMock().WithQuery(`UPDATE "artifacts" SET "updated_at"=$1,"artifact_id"=$2 WHERE "artifact_id" = $3`). + WithRowsNum(1). + WithCallback(func(s string, values []driver.NamedValue) { + artifactUpdated = true + }) + artifactDataDeleted := false + GlobalMock.NewMock(). + WithQuery(`DELETE FROM "artifact_data" WHERE "artifact_data"."artifact_id" = $1 AND name NOT IN ($2,$3)`). + WithRowsNum(0). + WithCallback(func(s string, values []driver.NamedValue) { + artifactDataDeleted = true + }) + GlobalMock.NewMock().WithQuery(`INSERT INTO "artifact_data" ("created_at","updated_at","deleted_at","dataset_project","dataset_name","dataset_domain","dataset_version","artifact_id","name","location") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10),($11,$12,$13,$14,$15,$16,$17,$18,$19,$20) ON CONFLICT DO NOTHING`). + WithExecException() + + updateInput := models.Artifact{ + ArtifactKey: models.ArtifactKey{ + ArtifactID: artifact.ArtifactID, + }, + ArtifactData: []models.ArtifactData{ + { + Name: "test-dataloc-name", + Location: "test-dataloc-location", + }, + { + Name: "additional-test-dataloc-name", + Location: "additional-test-dataloc-location", + }, + }, + } + + artifactRepo := NewArtifactRepo(utils.GetDbForTest(t), errors.NewPostgresErrorTransformer(), promutils.NewTestScope()) + err := artifactRepo.Update(ctx, updateInput) + assert.Error(t, err) + dcErr, ok := err.(apiErrors.DataCatalogError) + assert.True(t, ok) + assert.Equal(t, dcErr.Code(), codes.Internal) + assert.True(t, artifactUpdated) + assert.True(t, artifactDataDeleted) + }) +} diff --git a/pkg/repositories/interfaces/artifact_repo.go b/pkg/repositories/interfaces/artifact_repo.go index 42a30f96..9607cb62 100644 --- a/pkg/repositories/interfaces/artifact_repo.go +++ b/pkg/repositories/interfaces/artifact_repo.go @@ -6,8 +6,11 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" ) +//go:generate mockery -name=ArtifactRepo -output=../mocks -case=underscore + type ArtifactRepo interface { Create(ctx context.Context, in models.Artifact) error Get(ctx context.Context, in models.ArtifactKey) (models.Artifact, error) List(ctx context.Context, datasetKey models.DatasetKey, in models.ListModelsInput) ([]models.Artifact, error) + Update(ctx context.Context, artifact models.Artifact) error } diff --git a/pkg/repositories/interfaces/dataset_repo.go b/pkg/repositories/interfaces/dataset_repo.go index 3c059f0f..94bdb008 100644 --- a/pkg/repositories/interfaces/dataset_repo.go +++ b/pkg/repositories/interfaces/dataset_repo.go @@ -6,6 +6,8 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" ) +//go:generate mockery -name=DatasetRepo -output=../mocks -case=underscore + type DatasetRepo interface { Create(ctx context.Context, in models.Dataset) error Get(ctx context.Context, in models.DatasetKey) (models.Dataset, error) diff --git a/pkg/repositories/interfaces/partition_repo.go b/pkg/repositories/interfaces/partition_repo.go index 192d3c0d..5874a5a2 100644 --- a/pkg/repositories/interfaces/partition_repo.go +++ b/pkg/repositories/interfaces/partition_repo.go @@ -6,6 +6,8 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" ) +//go:generate mockery -name=PartitionRepo -output=../mocks -case=underscore + type PartitionRepo interface { Create(ctx context.Context, in models.Partition) error } diff --git a/pkg/repositories/interfaces/reservation_repo.go b/pkg/repositories/interfaces/reservation_repo.go index 950742b9..40b0cfd8 100644 --- a/pkg/repositories/interfaces/reservation_repo.go +++ b/pkg/repositories/interfaces/reservation_repo.go @@ -7,6 +7,8 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" ) +//go:generate mockery -name=ReservationRepo -output=../mocks -case=underscore + // Interface to interact with Reservation Table type ReservationRepo interface { diff --git a/pkg/repositories/interfaces/tag_repo.go b/pkg/repositories/interfaces/tag_repo.go index a053309a..eb3d51ac 100644 --- a/pkg/repositories/interfaces/tag_repo.go +++ b/pkg/repositories/interfaces/tag_repo.go @@ -6,6 +6,8 @@ import ( "github.com/flyteorg/datacatalog/pkg/repositories/models" ) +//go:generate mockery -name=TagRepo -output=../mocks -case=underscore + type TagRepo interface { Create(ctx context.Context, in models.Tag) error Get(ctx context.Context, in models.TagKey) (models.Tag, error) diff --git a/pkg/repositories/mocks/artifact.go b/pkg/repositories/mocks/artifact.go deleted file mode 100644 index 874a7bc1..00000000 --- a/pkg/repositories/mocks/artifact.go +++ /dev/null @@ -1,73 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package mocks - -import ( - context "context" - - models "github.com/flyteorg/datacatalog/pkg/repositories/models" - mock "github.com/stretchr/testify/mock" -) - -// ArtifactRepo is an autogenerated mock type for the ArtifactRepo type -type ArtifactRepo struct { - mock.Mock -} - -// Create provides a mock function with given fields: ctx, in -func (_m *ArtifactRepo) Create(ctx context.Context, in models.Artifact) error { - ret := _m.Called(ctx, in) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, models.Artifact) error); ok { - r0 = rf(ctx, in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Get provides a mock function with given fields: ctx, in -func (_m *ArtifactRepo) Get(ctx context.Context, in models.ArtifactKey) (models.Artifact, error) { - ret := _m.Called(ctx, in) - - var r0 models.Artifact - if rf, ok := ret.Get(0).(func(context.Context, models.ArtifactKey) models.Artifact); ok { - r0 = rf(ctx, in) - } else { - r0 = ret.Get(0).(models.Artifact) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, models.ArtifactKey) error); ok { - r1 = rf(ctx, in) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// List provides a mock function with given fields: ctx, datasetKey, in -func (_m *ArtifactRepo) List(ctx context.Context, datasetKey models.DatasetKey, in models.ListModelsInput) ([]models.Artifact, error) { - ret := _m.Called(ctx, datasetKey, in) - - var r0 []models.Artifact - if rf, ok := ret.Get(0).(func(context.Context, models.DatasetKey, models.ListModelsInput) []models.Artifact); ok { - r0 = rf(ctx, datasetKey, in) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]models.Artifact) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, models.DatasetKey, models.ListModelsInput) error); ok { - r1 = rf(ctx, datasetKey, in) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/pkg/repositories/mocks/artifact_repo.go b/pkg/repositories/mocks/artifact_repo.go new file mode 100644 index 00000000..f1677ea7 --- /dev/null +++ b/pkg/repositories/mocks/artifact_repo.go @@ -0,0 +1,160 @@ +// Code generated by mockery v1.0.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + + models "github.com/flyteorg/datacatalog/pkg/repositories/models" +) + +// ArtifactRepo is an autogenerated mock type for the ArtifactRepo type +type ArtifactRepo struct { + mock.Mock +} + +type ArtifactRepo_Create struct { + *mock.Call +} + +func (_m ArtifactRepo_Create) Return(_a0 error) *ArtifactRepo_Create { + return &ArtifactRepo_Create{Call: _m.Call.Return(_a0)} +} + +func (_m *ArtifactRepo) OnCreate(ctx context.Context, in models.Artifact) *ArtifactRepo_Create { + c_call := _m.On("Create", ctx, in) + return &ArtifactRepo_Create{Call: c_call} +} + +func (_m *ArtifactRepo) OnCreateMatch(matchers ...interface{}) *ArtifactRepo_Create { + c_call := _m.On("Create", matchers...) + return &ArtifactRepo_Create{Call: c_call} +} + +// Create provides a mock function with given fields: ctx, in +func (_m *ArtifactRepo) Create(ctx context.Context, in models.Artifact) error { + ret := _m.Called(ctx, in) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, models.Artifact) error); ok { + r0 = rf(ctx, in) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type ArtifactRepo_Get struct { + *mock.Call +} + +func (_m ArtifactRepo_Get) Return(_a0 models.Artifact, _a1 error) *ArtifactRepo_Get { + return &ArtifactRepo_Get{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *ArtifactRepo) OnGet(ctx context.Context, in models.ArtifactKey) *ArtifactRepo_Get { + c_call := _m.On("Get", ctx, in) + return &ArtifactRepo_Get{Call: c_call} +} + +func (_m *ArtifactRepo) OnGetMatch(matchers ...interface{}) *ArtifactRepo_Get { + c_call := _m.On("Get", matchers...) + return &ArtifactRepo_Get{Call: c_call} +} + +// Get provides a mock function with given fields: ctx, in +func (_m *ArtifactRepo) Get(ctx context.Context, in models.ArtifactKey) (models.Artifact, error) { + ret := _m.Called(ctx, in) + + var r0 models.Artifact + if rf, ok := ret.Get(0).(func(context.Context, models.ArtifactKey) models.Artifact); ok { + r0 = rf(ctx, in) + } else { + r0 = ret.Get(0).(models.Artifact) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, models.ArtifactKey) error); ok { + r1 = rf(ctx, in) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type ArtifactRepo_List struct { + *mock.Call +} + +func (_m ArtifactRepo_List) Return(_a0 []models.Artifact, _a1 error) *ArtifactRepo_List { + return &ArtifactRepo_List{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *ArtifactRepo) OnList(ctx context.Context, datasetKey models.DatasetKey, in models.ListModelsInput) *ArtifactRepo_List { + c_call := _m.On("List", ctx, datasetKey, in) + return &ArtifactRepo_List{Call: c_call} +} + +func (_m *ArtifactRepo) OnListMatch(matchers ...interface{}) *ArtifactRepo_List { + c_call := _m.On("List", matchers...) + return &ArtifactRepo_List{Call: c_call} +} + +// List provides a mock function with given fields: ctx, datasetKey, in +func (_m *ArtifactRepo) List(ctx context.Context, datasetKey models.DatasetKey, in models.ListModelsInput) ([]models.Artifact, error) { + ret := _m.Called(ctx, datasetKey, in) + + var r0 []models.Artifact + if rf, ok := ret.Get(0).(func(context.Context, models.DatasetKey, models.ListModelsInput) []models.Artifact); ok { + r0 = rf(ctx, datasetKey, in) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]models.Artifact) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, models.DatasetKey, models.ListModelsInput) error); ok { + r1 = rf(ctx, datasetKey, in) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type ArtifactRepo_Update struct { + *mock.Call +} + +func (_m ArtifactRepo_Update) Return(_a0 error) *ArtifactRepo_Update { + return &ArtifactRepo_Update{Call: _m.Call.Return(_a0)} +} + +func (_m *ArtifactRepo) OnUpdate(ctx context.Context, artifact models.Artifact) *ArtifactRepo_Update { + c_call := _m.On("Update", ctx, artifact) + return &ArtifactRepo_Update{Call: c_call} +} + +func (_m *ArtifactRepo) OnUpdateMatch(matchers ...interface{}) *ArtifactRepo_Update { + c_call := _m.On("Update", matchers...) + return &ArtifactRepo_Update{Call: c_call} +} + +// Update provides a mock function with given fields: ctx, artifact +func (_m *ArtifactRepo) Update(ctx context.Context, artifact models.Artifact) error { + ret := _m.Called(ctx, artifact) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, models.Artifact) error); ok { + r0 = rf(ctx, artifact) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/pkg/repositories/mocks/dataset.go b/pkg/repositories/mocks/dataset_repo.go similarity index 51% rename from pkg/repositories/mocks/dataset.go rename to pkg/repositories/mocks/dataset_repo.go index 8c75d00d..57ac32b3 100644 --- a/pkg/repositories/mocks/dataset.go +++ b/pkg/repositories/mocks/dataset_repo.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. +// Code generated by mockery v1.0.1. DO NOT EDIT. package mocks @@ -15,6 +15,24 @@ type DatasetRepo struct { mock.Mock } +type DatasetRepo_Create struct { + *mock.Call +} + +func (_m DatasetRepo_Create) Return(_a0 error) *DatasetRepo_Create { + return &DatasetRepo_Create{Call: _m.Call.Return(_a0)} +} + +func (_m *DatasetRepo) OnCreate(ctx context.Context, in models.Dataset) *DatasetRepo_Create { + c_call := _m.On("Create", ctx, in) + return &DatasetRepo_Create{Call: c_call} +} + +func (_m *DatasetRepo) OnCreateMatch(matchers ...interface{}) *DatasetRepo_Create { + c_call := _m.On("Create", matchers...) + return &DatasetRepo_Create{Call: c_call} +} + // Create provides a mock function with given fields: ctx, in func (_m *DatasetRepo) Create(ctx context.Context, in models.Dataset) error { ret := _m.Called(ctx, in) @@ -29,6 +47,24 @@ func (_m *DatasetRepo) Create(ctx context.Context, in models.Dataset) error { return r0 } +type DatasetRepo_Get struct { + *mock.Call +} + +func (_m DatasetRepo_Get) Return(_a0 models.Dataset, _a1 error) *DatasetRepo_Get { + return &DatasetRepo_Get{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *DatasetRepo) OnGet(ctx context.Context, in models.DatasetKey) *DatasetRepo_Get { + c_call := _m.On("Get", ctx, in) + return &DatasetRepo_Get{Call: c_call} +} + +func (_m *DatasetRepo) OnGetMatch(matchers ...interface{}) *DatasetRepo_Get { + c_call := _m.On("Get", matchers...) + return &DatasetRepo_Get{Call: c_call} +} + // Get provides a mock function with given fields: ctx, in func (_m *DatasetRepo) Get(ctx context.Context, in models.DatasetKey) (models.Dataset, error) { ret := _m.Called(ctx, in) @@ -50,6 +86,24 @@ func (_m *DatasetRepo) Get(ctx context.Context, in models.DatasetKey) (models.Da return r0, r1 } +type DatasetRepo_List struct { + *mock.Call +} + +func (_m DatasetRepo_List) Return(_a0 []models.Dataset, _a1 error) *DatasetRepo_List { + return &DatasetRepo_List{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *DatasetRepo) OnList(ctx context.Context, in models.ListModelsInput) *DatasetRepo_List { + c_call := _m.On("List", ctx, in) + return &DatasetRepo_List{Call: c_call} +} + +func (_m *DatasetRepo) OnListMatch(matchers ...interface{}) *DatasetRepo_List { + c_call := _m.On("List", matchers...) + return &DatasetRepo_List{Call: c_call} +} + // List provides a mock function with given fields: ctx, in func (_m *DatasetRepo) List(ctx context.Context, in models.ListModelsInput) ([]models.Dataset, error) { ret := _m.Called(ctx, in) diff --git a/pkg/repositories/mocks/partition.go b/pkg/repositories/mocks/partition_repo.go similarity index 50% rename from pkg/repositories/mocks/partition.go rename to pkg/repositories/mocks/partition_repo.go index d45dc486..e7beb1c1 100644 --- a/pkg/repositories/mocks/partition.go +++ b/pkg/repositories/mocks/partition_repo.go @@ -1,12 +1,13 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. +// Code generated by mockery v1.0.1. DO NOT EDIT. package mocks import ( context "context" - models "github.com/flyteorg/datacatalog/pkg/repositories/models" mock "github.com/stretchr/testify/mock" + + models "github.com/flyteorg/datacatalog/pkg/repositories/models" ) // PartitionRepo is an autogenerated mock type for the PartitionRepo type @@ -14,6 +15,24 @@ type PartitionRepo struct { mock.Mock } +type PartitionRepo_Create struct { + *mock.Call +} + +func (_m PartitionRepo_Create) Return(_a0 error) *PartitionRepo_Create { + return &PartitionRepo_Create{Call: _m.Call.Return(_a0)} +} + +func (_m *PartitionRepo) OnCreate(ctx context.Context, in models.Partition) *PartitionRepo_Create { + c_call := _m.On("Create", ctx, in) + return &PartitionRepo_Create{Call: c_call} +} + +func (_m *PartitionRepo) OnCreateMatch(matchers ...interface{}) *PartitionRepo_Create { + c_call := _m.On("Create", matchers...) + return &PartitionRepo_Create{Call: c_call} +} + // Create provides a mock function with given fields: ctx, in func (_m *PartitionRepo) Create(ctx context.Context, in models.Partition) error { ret := _m.Called(ctx, in) diff --git a/pkg/repositories/mocks/reservation.go b/pkg/repositories/mocks/reservation_repo.go similarity index 84% rename from pkg/repositories/mocks/reservation.go rename to pkg/repositories/mocks/reservation_repo.go index 4cabc8a2..5214107e 100644 --- a/pkg/repositories/mocks/reservation.go +++ b/pkg/repositories/mocks/reservation_repo.go @@ -26,13 +26,13 @@ func (_m ReservationRepo_Create) Return(_a0 error) *ReservationRepo_Create { } func (_m *ReservationRepo) OnCreate(ctx context.Context, reservation models.Reservation, now time.Time) *ReservationRepo_Create { - c := _m.On("Create", ctx, reservation, now) - return &ReservationRepo_Create{Call: c} + c_call := _m.On("Create", ctx, reservation, now) + return &ReservationRepo_Create{Call: c_call} } func (_m *ReservationRepo) OnCreateMatch(matchers ...interface{}) *ReservationRepo_Create { - c := _m.On("Create", matchers...) - return &ReservationRepo_Create{Call: c} + c_call := _m.On("Create", matchers...) + return &ReservationRepo_Create{Call: c_call} } // Create provides a mock function with given fields: ctx, reservation, now @@ -58,13 +58,13 @@ func (_m ReservationRepo_Delete) Return(_a0 error) *ReservationRepo_Delete { } func (_m *ReservationRepo) OnDelete(ctx context.Context, reservation models.ReservationKey, ownerID string) *ReservationRepo_Delete { - c := _m.On("Delete", ctx, reservation, ownerID) - return &ReservationRepo_Delete{Call: c} + c_call := _m.On("Delete", ctx, reservation, ownerID) + return &ReservationRepo_Delete{Call: c_call} } func (_m *ReservationRepo) OnDeleteMatch(matchers ...interface{}) *ReservationRepo_Delete { - c := _m.On("Delete", matchers...) - return &ReservationRepo_Delete{Call: c} + c_call := _m.On("Delete", matchers...) + return &ReservationRepo_Delete{Call: c_call} } // Delete provides a mock function with given fields: ctx, reservation, ownerID @@ -90,13 +90,13 @@ func (_m ReservationRepo_Get) Return(_a0 models.Reservation, _a1 error) *Reserva } func (_m *ReservationRepo) OnGet(ctx context.Context, reservationKey models.ReservationKey) *ReservationRepo_Get { - c := _m.On("Get", ctx, reservationKey) - return &ReservationRepo_Get{Call: c} + c_call := _m.On("Get", ctx, reservationKey) + return &ReservationRepo_Get{Call: c_call} } func (_m *ReservationRepo) OnGetMatch(matchers ...interface{}) *ReservationRepo_Get { - c := _m.On("Get", matchers...) - return &ReservationRepo_Get{Call: c} + c_call := _m.On("Get", matchers...) + return &ReservationRepo_Get{Call: c_call} } // Get provides a mock function with given fields: ctx, reservationKey @@ -129,13 +129,13 @@ func (_m ReservationRepo_Update) Return(_a0 error) *ReservationRepo_Update { } func (_m *ReservationRepo) OnUpdate(ctx context.Context, reservation models.Reservation, now time.Time) *ReservationRepo_Update { - c := _m.On("Update", ctx, reservation, now) - return &ReservationRepo_Update{Call: c} + c_call := _m.On("Update", ctx, reservation, now) + return &ReservationRepo_Update{Call: c_call} } func (_m *ReservationRepo) OnUpdateMatch(matchers ...interface{}) *ReservationRepo_Update { - c := _m.On("Update", matchers...) - return &ReservationRepo_Update{Call: c} + c_call := _m.On("Update", matchers...) + return &ReservationRepo_Update{Call: c_call} } // Update provides a mock function with given fields: ctx, reservation, now diff --git a/pkg/repositories/mocks/tag.go b/pkg/repositories/mocks/tag_repo.go similarity index 51% rename from pkg/repositories/mocks/tag.go rename to pkg/repositories/mocks/tag_repo.go index aa441a3a..3070cd58 100644 --- a/pkg/repositories/mocks/tag.go +++ b/pkg/repositories/mocks/tag_repo.go @@ -1,12 +1,13 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. +// Code generated by mockery v1.0.1. DO NOT EDIT. package mocks import ( context "context" - models "github.com/flyteorg/datacatalog/pkg/repositories/models" mock "github.com/stretchr/testify/mock" + + models "github.com/flyteorg/datacatalog/pkg/repositories/models" ) // TagRepo is an autogenerated mock type for the TagRepo type @@ -14,6 +15,24 @@ type TagRepo struct { mock.Mock } +type TagRepo_Create struct { + *mock.Call +} + +func (_m TagRepo_Create) Return(_a0 error) *TagRepo_Create { + return &TagRepo_Create{Call: _m.Call.Return(_a0)} +} + +func (_m *TagRepo) OnCreate(ctx context.Context, in models.Tag) *TagRepo_Create { + c_call := _m.On("Create", ctx, in) + return &TagRepo_Create{Call: c_call} +} + +func (_m *TagRepo) OnCreateMatch(matchers ...interface{}) *TagRepo_Create { + c_call := _m.On("Create", matchers...) + return &TagRepo_Create{Call: c_call} +} + // Create provides a mock function with given fields: ctx, in func (_m *TagRepo) Create(ctx context.Context, in models.Tag) error { ret := _m.Called(ctx, in) @@ -28,6 +47,24 @@ func (_m *TagRepo) Create(ctx context.Context, in models.Tag) error { return r0 } +type TagRepo_Get struct { + *mock.Call +} + +func (_m TagRepo_Get) Return(_a0 models.Tag, _a1 error) *TagRepo_Get { + return &TagRepo_Get{Call: _m.Call.Return(_a0, _a1)} +} + +func (_m *TagRepo) OnGet(ctx context.Context, in models.TagKey) *TagRepo_Get { + c_call := _m.On("Get", ctx, in) + return &TagRepo_Get{Call: c_call} +} + +func (_m *TagRepo) OnGetMatch(matchers ...interface{}) *TagRepo_Get { + c_call := _m.On("Get", matchers...) + return &TagRepo_Get{Call: c_call} +} + // Get provides a mock function with given fields: ctx, in func (_m *TagRepo) Get(ctx context.Context, in models.TagKey) (models.Tag, error) { ret := _m.Called(ctx, in) diff --git a/pkg/rpc/datacatalogservice/service.go b/pkg/rpc/datacatalogservice/service.go index 6df0064f..0591b7da 100644 --- a/pkg/rpc/datacatalogservice/service.go +++ b/pkg/rpc/datacatalogservice/service.go @@ -61,6 +61,10 @@ func (s *DataCatalogService) ListDatasets(ctx context.Context, request *catalog. return s.DatasetManager.ListDatasets(ctx, request) } +func (s *DataCatalogService) UpdateArtifact(ctx context.Context, request *catalog.UpdateArtifactRequest) (*catalog.UpdateArtifactResponse, error) { + return s.ArtifactManager.UpdateArtifact(ctx, request) +} + func (s *DataCatalogService) GetOrExtendReservation(ctx context.Context, request *catalog.GetOrExtendReservationRequest) (*catalog.GetOrExtendReservationResponse, error) { return s.ReservationManager.GetOrExtendReservation(ctx, request) }