From f9e4bc2e83da1e0bd01d67be398bb2cc5213466d Mon Sep 17 00:00:00 2001 From: Sebastian Choren Date: Thu, 27 Jul 2023 15:56:54 -0300 Subject: [PATCH] feat(server): refactor runner channels into abstract queues (#2971) --- go.work.sum | 153 +----- local-config/tracetest.provision.yaml | 1 + server/Makefile | 4 + server/app/app.go | 102 ++-- server/app/facade.go | 140 ------ server/app/test_pipeline.go | 108 ++++ server/app/transaction_pipeline.go | 28 ++ server/config/appconfig_test.go | 2 +- server/config/server.go | 17 +- server/config/server_test.go | 22 +- server/datastore/datastore_repository.go | 8 +- server/executor/assertion_runner.go | 129 ++--- server/executor/default_poller_executor.go | 199 ++++---- server/executor/linter_runner.go | 146 ++---- server/executor/pipeline.go | 94 ++++ server/executor/poller_executor_test.go | 141 +++--- .../polling_profile_entities.go | 5 +- .../polling_profile_repository_test.go | 4 +- server/executor/queue.go | 476 ++++++++++++++++++ server/executor/run_stop.go | 54 -- server/executor/run_updater.go | 8 +- server/executor/runner.go | 232 +++------ server/executor/runner_test.go | 244 ++++++--- .../selector_based_poller_executor.go | 82 ++- .../selector_based_poller_executor_test.go | 69 +-- server/executor/test_pipeline.go | 162 ++++++ server/executor/trace_poller.go | 245 +++------ server/executor/transaction_pipeline.go | 46 ++ server/executor/transaction_runner.go | 103 ++-- server/executor/transaction_runner_test.go | 27 +- server/go.mod | 13 +- server/go.sum | 21 +- server/http/controller.go | 142 +++--- server/http/controller_test.go | 11 +- server/pkg/id/generator.go | 13 +- server/test/run_repository.go | 3 +- server/testdb/postgres.go | 11 - server/testdb/postgres_options.go | 6 +- server/testdb/postgres_test.go | 6 +- server/testdb/test_run_event.go | 5 - server/tracedb/otlp.go | 9 +- server/tracedb/tracedb.go | 5 +- server/transaction/transaction_repository.go | 4 - .../transaction/transaction_run_repository.go | 14 +- 44 files changed, 1826 insertions(+), 1488 deletions(-) delete mode 100644 server/app/facade.go create mode 100644 server/app/test_pipeline.go create mode 100644 server/app/transaction_pipeline.go create mode 100644 server/executor/pipeline.go create mode 100644 server/executor/queue.go delete mode 100644 server/executor/run_stop.go create mode 100644 server/executor/test_pipeline.go create mode 100644 server/executor/transaction_pipeline.go diff --git a/go.work.sum b/go.work.sum index 17f62795f0..4c69bf41d9 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,10 +1,4 @@ -atomicgo.dev/assert v0.0.2/go.mod h1:ut4NcI3QDdJtlmAxQULOmA13Gz6e2DWbSAS8RUOmNYQ= -bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512 h1:SRsZGA7aFnCZETmov57jwPrWuTmaZK6+4R4v5FUe1/c= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.102.1 h1:vpK6iQWv/2uUeFJth4/cBHsQAGjn1iIE6AAlxipRaA0= -cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= -cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= @@ -16,7 +10,6 @@ cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= -cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= @@ -33,10 +26,6 @@ cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2c cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= @@ -78,7 +67,6 @@ cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZ cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= -cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= @@ -94,7 +82,6 @@ cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5Q cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= -cloud.google.com/go/monitoring v1.9.1/go.mod h1:iFzRDMSDMvvf/z30Ge1jwtuEe/jlPPAFusmvCkUdo+o= cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= @@ -108,7 +95,6 @@ cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7 cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= -cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= @@ -120,7 +106,6 @@ cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/ cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= -cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= @@ -129,16 +114,13 @@ cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmj cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= -cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= -cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= @@ -150,53 +132,15 @@ cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIE cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= -github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v59.4.0+incompatible h1:gDA8odnngdNd3KYHL2NoK1j9vpWBgEnFSjKKLpkC8Aw= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE= -github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= -github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= -github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= -github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= -github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgRRqzCuPshRkQ7I= -github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= -github.com/Microsoft/hcsshim v0.9.6/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 h1:KwaoQzs/WeUxxJqiJsZ4euOly1Az/IgZXXSxlD/UBNk= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= -github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= -github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= -github.com/containerd/go-cni v1.1.6/go.mod h1:BWtoWl5ghVymxu6MBjg79W9NZrCRyHIdUtk4cauMe34= -github.com/containerd/imgcrypt v1.1.4/go.mod h1:LorQnPtzL/T0IyCeftcsMEO7AqxUDbdO8j/tSUpgxvo= -github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI= -github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= -github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= -github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= -github.com/containers/ocicrypt v1.1.3/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= -github.com/deepmap/oapi-codegen v1.10.1/go.mod h1:TvVmDQlUkFli9gFij/gtW1o+tFBr4qCHyv2zG+R0YZY= -github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/elastic/go-elasticsearch/v7 v7.17.1/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= @@ -204,31 +148,14 @@ github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0+ github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= -github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/gocql/gocql v0.0.0-20210817081954-bc256bbb90de/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= -github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= -github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188/go.mod h1:vXjM/+wXQnTPR4KqTKDgJukSZ6amVRtWMPEjE6sQoK8= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= @@ -237,7 +164,6 @@ github.com/hashicorp/consul/api v1.18.0/go.mod h1:owRRGJ9M5xReDC5nfT8FTJrNAPbT4N github.com/hashicorp/consul/sdk v0.13.0/go.mod h1:0hs/l5fOVhJy/VdcoaNqUSi2AUs95eF5WKtv+EYIQqE= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= @@ -245,27 +171,25 @@ github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= -github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs= github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= -github.com/influxdata/influxdb-client-go/v2 v2.8.2/go.mod h1:x7Jo5UHHl+w8wu8UnGiNobDDHygojXwJX4mx7rXGKMk= -github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/pgconn v1.8.0 h1:FmjZ0rOyXTr1wfWs45i4a9vjnjWUAGpMuQLD9OSs+lw= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3/v2 v2.0.7 h1:6Pwi1b3QdY65cuv6SyVO0FgPd5J3Bl7wf/nQQjinHMA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgtype v1.6.2 h1:b3pDeuhbbzBYcg5kwNmNDun4pFUD/0AAr1kLXZLeNt8= +github.com/jackc/pgx/v4 v4.10.1 h1:/6Q3ye4myIj6AaplUm+eRcz4OhK9HAvFf4ePsG40LJY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/knadh/koanf v1.4.3/go.mod h1:5FAkuykKXZvLqhAbP4peWgM5CTcZmn7L1d27k/a+kfg= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mostynb/go-grpc-compression v1.1.17/go.mod h1:FUSBr0QjKqQgoDG/e0yiqlR6aqyXC39+g/hFLDfSsEY= -github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= @@ -277,9 +201,8 @@ github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= -github.com/sagikazarmark/crypt v0.9.0/go.mod h1:RnH7sEhxfdnPm1z+XMgSLjWTEIjyK4z2dw6+4vHTMuo= -github.com/segmentio/kafka-go v0.4.31/go.mod h1:m1lXeqJtIFYZayv0shM/tjrAFljvWLTprxBHd+3PnaU= github.com/shirou/gopsutil/v3 v3.22.8/go.mod h1:s648gW4IywYzUfE/KjXxUsqrqx/T2xO5VqOXxONeRfI= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= @@ -289,10 +212,8 @@ go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dY go.etcd.io/etcd/api/v3 v3.5.6/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/pkg/v3 v3.5.6/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= -go.etcd.io/etcd/client/v2 v2.305.6/go.mod h1:BHha8XJGe8vCIBfWBpbBLVZ4QjOIlfoouvOwydu63E0= go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= go.etcd.io/etcd/client/v3 v3.5.6/go.mod h1:f6GRinRMCsFVv9Ht42EyY7nfsVGwrNO0WEoS2pRKzQk= -go.mongodb.org/mongo-driver v1.9.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/collector/semconv v0.60.0/go.mod h1:aRkHuJ/OshtDFYluKEtnG5nkKTsy1HZuvZVHmakx+Vo= go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= @@ -304,104 +225,54 @@ go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85 go.opentelemetry.io/otel/exporters/prometheus v0.31.0/go.mod h1:QarXIB8L79IwIPoNgG3A6zNvBgVmcppeFogV1d8612s= go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= go.opentelemetry.io/otel/sdk v1.9.0/go.mod h1:AEZc8nt5bd2F7BC24J5R0mrjYnpEgYHyTcM/vrSple4= -go.opentelemetry.io/otel/sdk/export/metric v0.26.0/go.mod h1:UpqzSnUOjFeSIVQLPp3pYIXfB/MiMFyXXzYT/bercxQ= go.opentelemetry.io/otel/sdk/metric v0.31.0/go.mod h1:fl0SmNnX9mN9xgU6OLYLMBMrNAsaZQi7qBwprwO3abk= go.opentelemetry.io/otel/trace v1.9.0/go.mod h1:2737Q0MuG8q1uILYm2YYVkAyLtOofiTNGg6VODnOiPo= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= diff --git a/local-config/tracetest.provision.yaml b/local-config/tracetest.provision.yaml index 918e85b964..4b7018f666 100644 --- a/local-config/tracetest.provision.yaml +++ b/local-config/tracetest.provision.yaml @@ -21,6 +21,7 @@ spec: periodic: timeout: 30s retryDelay: 1s + selectorMatchRetries: 3 --- type: TestRunner spec: diff --git a/server/Makefile b/server/Makefile index 64d112630a..c03f23f675 100644 --- a/server/Makefile +++ b/server/Makefile @@ -7,6 +7,7 @@ GO_LDFLAGS := $(shell echo \ -X "'github.com/kubeshop/tracetest/server/analytics.FrontendKey=$(ANALYTICS_FE_KEY)'" \ | sed 's/ / /g') +.PHONY: help help: Makefile ## show list of commands @echo "Choose a command run:" @echo "" @@ -18,12 +19,15 @@ init-submodule: git submodule init git submodule update +.PHONY: test test: ## run go tests for this application go test -timeout 150s -coverprofile=coverage.out ./... +.PHONY: vet vet: ## run vet tool to analyze the code for suspicious, abnormal, or useless code go vet -structtag=false ./... +.PHONY: run run: ## run server locally go run -ldflags="$(GO_LDFLAGS)" main.go serve diff --git a/server/app/app.go b/server/app/app.go index 18b4775935..b811cd6d9e 100644 --- a/server/app/app.go +++ b/server/app/app.go @@ -208,7 +208,7 @@ func (app *App) Start(opts ...appOption) error { eventEmitter := executor.NewEventEmitter(testDB, subscriptionManager) registerOtlpServer(app, runRepo, eventEmitter, dataStoreRepo) - rf := newRunnerFacades( + testPipeline := buildTestPipeline( pollingProfileRepo, dataStoreRepo, linterRepo, @@ -216,36 +216,25 @@ func (app *App) Start(opts ...appOption) error { testDB, testRepo, runRepo, - transactionRunRepository, - applicationTracer, tracer, subscriptionManager, triggerRegistry, ) - - // worker count. should be configurable - rf.tracePoller.Start(5) - rf.runner.Start(5) - rf.runner.Start(5) - rf.transactionRunner.Start(5) - rf.assertionRunner.Start(5) - rf.linterRunner.Start(5) - - app.registerStopFn(func() { - fmt.Println("stopping tracePoller") - rf.tracePoller.Stop() - }) - app.registerStopFn(func() { - fmt.Println("stopping runner") - rf.runner.Stop() - }) + testPipeline.Start() app.registerStopFn(func() { - fmt.Println("stopping transactionRunner") - rf.transactionRunner.Stop() + testPipeline.Stop() }) + + transactionPipeline := buildTransactionPipeline( + transactionsRepository, + transactionRunRepository, + testPipeline, + subscriptionManager, + ) + + transactionPipeline.Start() app.registerStopFn(func() { - fmt.Println("stopping assertionRunner") - rf.assertionRunner.Stop() + transactionPipeline.Stop() }) err = analytics.SendEvent("Server Started", "beacon", "", nil) @@ -256,14 +245,17 @@ func (app *App) Start(opts ...appOption) error { provisioner := provisioning.New() router, mappers := controller(app.cfg, + tracer, + + testPipeline, + transactionPipeline, + testDB, transactionsRepository, transactionRunRepository, testRepo, runRepo, - tracer, environmentRepo, - rf, ) registerWSHandler(router, mappers, subscriptionManager) @@ -314,7 +306,7 @@ func (app *App) Start(opts ...appOption) error { var ( matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") - matchResourceName = regexp.MustCompile("(\\w)(\\.)(\\w)") + matchResourceName = regexp.MustCompile(`(\w)(\.)(\w)`) ) func toWords(str string) string { @@ -506,27 +498,36 @@ func registerWSHandler(router *mux.Router, mappers mappings.Mappings, subscripti func controller( cfg httpServerConfig, - testDB model.Repository, - transactionRepository *transaction.Repository, - transactionRunRepository *transaction.RunRepository, - testRepository test.Repository, - runRepository test.RunRepository, + tracer trace.Tracer, + + testRunner *executor.TestPipeline, + transactionRunner *executor.TransactionPipeline, + + testRunEvents model.TestRunEventRepository, + transactionRepo *transaction.Repository, + transactionRunRepo *transaction.RunRepository, + testRepo test.Repository, + testRunRepo test.RunRepository, environmentRepo *environment.Repository, - rf *runnerFacade, ) (*mux.Router, mappings.Mappings) { mappers := mappings.New(tracesConversionConfig(), comparator.DefaultRegistry()) router := openapi.NewRouter(httpRouter( cfg, - testDB, - transactionRepository, - transactionRunRepository, - testRepository, - runRepository, + tracer, + + testRunner, + transactionRunner, + + testRunEvents, + transactionRepo, + transactionRunRepo, + testRepo, + testRunRepo, environmentRepo, - rf, + mappers, )) @@ -535,27 +536,36 @@ func controller( func httpRouter( cfg httpServerConfig, - testDB model.Repository, + + tracer trace.Tracer, + + testRunner *executor.TestPipeline, + transactionRunner *executor.TransactionPipeline, + + testRunEvents model.TestRunEventRepository, transactionRepo *transaction.Repository, transactionRunRepo *transaction.RunRepository, testRepo test.Repository, testRunRepo test.RunRepository, - tracer trace.Tracer, environmentRepo *environment.Repository, - rf *runnerFacade, + mappers mappings.Mappings, ) openapi.Router { controller := httpServer.NewController( - testDB, + tracer, + + testRunner, + transactionRunner, + + testRunEvents, transactionRepo, transactionRunRepo, testRepo, testRunRepo, + environmentRepo, + tracedb.Factory(testRunRepo), - rf, mappers, - environmentRepo, - tracer, Version, ) apiApiController := openapi.NewApiApiController(controller) diff --git a/server/app/facade.go b/server/app/facade.go deleted file mode 100644 index c1a5d84cf5..0000000000 --- a/server/app/facade.go +++ /dev/null @@ -1,140 +0,0 @@ -package app - -import ( - "context" - - "github.com/kubeshop/tracetest/server/datastore" - "github.com/kubeshop/tracetest/server/environment" - "github.com/kubeshop/tracetest/server/executor" - "github.com/kubeshop/tracetest/server/executor/pollingprofile" - "github.com/kubeshop/tracetest/server/executor/testrunner" - "github.com/kubeshop/tracetest/server/executor/trigger" - "github.com/kubeshop/tracetest/server/linter/analyzer" - "github.com/kubeshop/tracetest/server/model" - "github.com/kubeshop/tracetest/server/pkg/id" - "github.com/kubeshop/tracetest/server/subscription" - "github.com/kubeshop/tracetest/server/test" - "github.com/kubeshop/tracetest/server/tracedb" - "github.com/kubeshop/tracetest/server/transaction" - "go.opentelemetry.io/otel/trace" -) - -type runnerFacade struct { - sm *subscription.Manager - runner executor.PersistentRunner - transactionRunner executor.PersistentTransactionRunner - assertionRunner executor.AssertionRunner - tracePoller executor.PersistentTracePoller - linterRunner executor.LinterRunner -} - -func (rf runnerFacade) StopTest(testID id.ID, runID int) { - sr := executor.StopRequest{ - TestID: testID, - RunID: runID, - } - - rf.sm.PublishUpdate(subscription.Message{ - ResourceID: sr.ResourceID(), - Content: sr, - }) -} - -func (rf runnerFacade) RunTest(ctx context.Context, test test.Test, rm test.RunMetadata, env environment.Environment, gates *[]testrunner.RequiredGate) test.Run { - return rf.runner.Run(ctx, test, rm, env, gates) -} - -func (rf runnerFacade) RunTransaction(ctx context.Context, tr transaction.Transaction, rm test.RunMetadata, env environment.Environment, gates *[]testrunner.RequiredGate) transaction.TransactionRun { - return rf.transactionRunner.Run(ctx, tr, rm, env, gates) -} - -func (rf runnerFacade) RunAssertions(ctx context.Context, request executor.AssertionRequest) { - rf.assertionRunner.RunAssertions(ctx, request) -} - -func newRunnerFacades( - ppRepo *pollingprofile.Repository, - dsRepo *datastore.Repository, - lintRepo *analyzer.Repository, - trRepo *testrunner.Repository, - db model.Repository, - testRepo test.Repository, - runRepo test.RunRepository, - transactionRunRepository *transaction.RunRepository, - appTracer trace.Tracer, - tracer trace.Tracer, - subscriptionManager *subscription.Manager, - triggerRegistry *trigger.Registry, -) *runnerFacade { - eventEmitter := executor.NewEventEmitter(db, subscriptionManager) - - execTestUpdater := (executor.CompositeUpdater{}). - Add(executor.NewDBUpdater(runRepo)). - Add(executor.NewSubscriptionUpdater(subscriptionManager)) - - assertionRunner := executor.NewAssertionRunner( - execTestUpdater, - executor.NewAssertionExecutor(tracer), - executor.InstrumentedOutputProcessor(tracer), - subscriptionManager, - eventEmitter, - ) - - linterRunner := executor.NewlinterRunner( - execTestUpdater, - subscriptionManager, - eventEmitter, - assertionRunner, - lintRepo, - ) - - pollerExecutor := executor.NewPollerExecutor( - ppRepo, - tracer, - execTestUpdater, - tracedb.Factory(runRepo), - dsRepo, - eventEmitter, - ) - - pollerExecutor = executor.NewSelectorBasedPoller(pollerExecutor, eventEmitter) - - tracePoller := executor.NewTracePoller( - pollerExecutor, - ppRepo, - execTestUpdater, - linterRunner, - subscriptionManager, - eventEmitter, - ) - - runner := executor.NewPersistentRunner( - triggerRegistry, - runRepo, - execTestUpdater, - tracePoller, - tracer, - subscriptionManager, - tracedb.Factory(runRepo), - dsRepo, - eventEmitter, - ppRepo, - trRepo, - ) - - transactionRunner := executor.NewTransactionRunner( - runner, - testRepo, - transactionRunRepository, - subscriptionManager, - ) - - return &runnerFacade{ - sm: subscriptionManager, - runner: runner, - transactionRunner: transactionRunner, - assertionRunner: assertionRunner, - tracePoller: tracePoller, - linterRunner: linterRunner, - } -} diff --git a/server/app/test_pipeline.go b/server/app/test_pipeline.go new file mode 100644 index 0000000000..feeb8a6b04 --- /dev/null +++ b/server/app/test_pipeline.go @@ -0,0 +1,108 @@ +package app + +import ( + "github.com/kubeshop/tracetest/server/datastore" + "github.com/kubeshop/tracetest/server/executor" + "github.com/kubeshop/tracetest/server/executor/pollingprofile" + "github.com/kubeshop/tracetest/server/executor/testrunner" + "github.com/kubeshop/tracetest/server/executor/trigger" + "github.com/kubeshop/tracetest/server/linter/analyzer" + "github.com/kubeshop/tracetest/server/model" + "github.com/kubeshop/tracetest/server/subscription" + "github.com/kubeshop/tracetest/server/test" + "github.com/kubeshop/tracetest/server/tracedb" + "go.opentelemetry.io/otel/trace" +) + +func buildTestPipeline( + ppRepo *pollingprofile.Repository, + dsRepo *datastore.Repository, + lintRepo *analyzer.Repository, + trRepo *testrunner.Repository, + treRepo model.TestRunEventRepository, + testRepo test.Repository, + runRepo test.RunRepository, + tracer trace.Tracer, + subscriptionManager *subscription.Manager, + triggerRegistry *trigger.Registry, +) *executor.TestPipeline { + eventEmitter := executor.NewEventEmitter(treRepo, subscriptionManager) + + execTestUpdater := (executor.CompositeUpdater{}). + Add(executor.NewDBUpdater(runRepo)). + Add(executor.NewSubscriptionUpdater(subscriptionManager)) + + assertionRunner := executor.NewAssertionRunner( + execTestUpdater, + executor.NewAssertionExecutor(tracer), + executor.InstrumentedOutputProcessor(tracer), + subscriptionManager, + eventEmitter, + ) + + linterRunner := executor.NewlinterRunner( + execTestUpdater, + subscriptionManager, + eventEmitter, + lintRepo, + ) + + pollerExecutor := executor.NewSelectorBasedPoller( + executor.NewPollerExecutor( + tracer, + execTestUpdater, + tracedb.Factory(runRepo), + dsRepo, + eventEmitter, + ), + eventEmitter, + ) + + tracePoller := executor.NewTracePoller( + pollerExecutor, + execTestUpdater, + subscriptionManager, + eventEmitter, + ) + + runner := executor.NewPersistentRunner( + triggerRegistry, + execTestUpdater, + tracer, + subscriptionManager, + tracedb.Factory(runRepo), + dsRepo, + eventEmitter, + ) + + cancelRunHandlerFn := executor.HandleRunCancelation(execTestUpdater, tracer, eventEmitter) + + queueBuilder := executor.NewQueueBuilder(). + WithCancelRunHandlerFn(cancelRunHandlerFn). + WithSubscriptor(subscriptionManager). + WithDataStoreGetter(dsRepo). + WithPollingProfileGetter(ppRepo). + WithTestGetter(testRepo). + WithRunGetter(runRepo) + + pipeline := executor.NewPipeline(queueBuilder, + executor.PipelineStep{Processor: runner, Driver: executor.NewInMemoryQueueDriver("runner")}, + executor.PipelineStep{Processor: tracePoller, Driver: executor.NewInMemoryQueueDriver("tracePoller")}, + executor.PipelineStep{Processor: linterRunner, Driver: executor.NewInMemoryQueueDriver("linterRunner")}, + executor.PipelineStep{Processor: assertionRunner, Driver: executor.NewInMemoryQueueDriver("assertionRunner")}, + ) + + pipeline.Start() + + const assertionRunnerStepIndex = 3 + + return executor.NewTestPipeline( + pipeline, + subscriptionManager, + pipeline.GetQueueForStep(assertionRunnerStepIndex), // assertion runner step + runRepo, + trRepo, + ppRepo, + dsRepo, + ) +} diff --git a/server/app/transaction_pipeline.go b/server/app/transaction_pipeline.go new file mode 100644 index 0000000000..5783dcc6f2 --- /dev/null +++ b/server/app/transaction_pipeline.go @@ -0,0 +1,28 @@ +package app + +import ( + "github.com/kubeshop/tracetest/server/executor" + "github.com/kubeshop/tracetest/server/subscription" + "github.com/kubeshop/tracetest/server/transaction" +) + +func buildTransactionPipeline( + tranRepo *transaction.Repository, + runRepo *transaction.RunRepository, + testRunner *executor.TestPipeline, + subscriptionManager *subscription.Manager, +) *executor.TransactionPipeline { + tranRunner := executor.NewTransactionRunner(testRunner, runRepo, subscriptionManager) + queueBuilder := executor.NewQueueBuilder(). + WithTransactionGetter(tranRepo). + WithTransactionRunGetter(runRepo) + + pipeline := executor.NewPipeline(queueBuilder, + executor.PipelineStep{Processor: tranRunner, Driver: executor.NewInMemoryQueueDriver("transactionRunner")}, + ) + + return executor.NewTransactionPipeline( + pipeline, + runRepo, + ) +} diff --git a/server/config/appconfig_test.go b/server/config/appconfig_test.go index ef7930e623..41cbe46eda 100644 --- a/server/config/appconfig_test.go +++ b/server/config/appconfig_test.go @@ -63,7 +63,7 @@ func TestFlags(t *testing.T) { cfg := configFromFile(t, "./testdata/basic.yaml") - assert.Equal(t, "host=postgres user=postgres password=postgres port=5432 dbname=tracetest sslmode=disable", cfg.PostgresConnString()) + assert.Equal(t, "postgres://postgres:postgres@postgres:5432/tracetest?sslmode=disable", cfg.PostgresConnString()) assert.Equal(t, "/tracetest", cfg.ServerPathPrefix()) assert.Equal(t, 9999, cfg.ServerPort()) diff --git a/server/config/server.go b/server/config/server.go index 73e83475a2..2f29e999f6 100644 --- a/server/config/server.go +++ b/server/config/server.go @@ -2,6 +2,8 @@ package config import ( "fmt" + "os" + "strings" ) var serverOptions = options{ @@ -80,23 +82,18 @@ func (c *AppConfig) PostgresConnString() string { defer c.mu.Unlock() if postgresConnString := c.vp.GetString("postgresConnString"); postgresConnString != "" { - return postgresConnString + fmt.Println("ERROR: postgresConnString was discontinued. Migrate to the new postgres format") + os.Exit(1) } - str := fmt.Sprintf( - "host=%s user=%s password=%s port=%d dbname=%s", - c.vp.GetString("postgres.host"), + return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?%s", c.vp.GetString("postgres.user"), c.vp.GetString("postgres.password"), + c.vp.GetString("postgres.host"), c.vp.GetInt("postgres.port"), c.vp.GetString("postgres.dbname"), + strings.ReplaceAll(c.vp.GetString("postgres.params"), " ", "&"), ) - - if params := c.vp.GetString("postgres.params"); params != "" { - str += " " + params - } - - return str } func (c *AppConfig) ServerPathPrefix() string { diff --git a/server/config/server_test.go b/server/config/server_test.go index 71474b8065..052a4ddfa5 100644 --- a/server/config/server_test.go +++ b/server/config/server_test.go @@ -11,7 +11,7 @@ func TestServerConfig(t *testing.T) { t.Run("DefaultValues", func(t *testing.T) { cfg, _ := config.New() - assert.Equal(t, "host=postgres user=postgres password=postgres port=5432 dbname=tracetest sslmode=disable", cfg.PostgresConnString()) + assert.Equal(t, "postgres://postgres:postgres@postgres:5432/tracetest?sslmode=disable", cfg.PostgresConnString()) assert.Equal(t, 11633, cfg.ServerPort()) assert.Equal(t, "", cfg.ServerPathPrefix()) @@ -40,7 +40,7 @@ func TestServerConfig(t *testing.T) { cfg := configWithFlags(t, flags) - assert.Equal(t, "host=localhost user=user password=passwd port=1234 dbname=other_dbname custom=params", cfg.PostgresConnString()) + assert.Equal(t, "postgres://user:passwd@localhost:1234/other_dbname?custom=params", cfg.PostgresConnString()) assert.Equal(t, 4321, cfg.ServerPort()) assert.Equal(t, "/prefix", cfg.ServerPathPrefix()) @@ -68,7 +68,7 @@ func TestServerConfig(t *testing.T) { cfg := configWithEnv(t, env) - assert.Equal(t, "host=localhost user=user password=passwd port=1234 dbname=other_dbname custom=params", cfg.PostgresConnString()) + assert.Equal(t, "postgres://user:passwd@localhost:1234/other_dbname?custom=params", cfg.PostgresConnString()) assert.Equal(t, 4321, cfg.ServerPort()) assert.Equal(t, "/prefix", cfg.ServerPathPrefix()) @@ -78,20 +78,4 @@ func TestServerConfig(t *testing.T) { assert.Equal(t, true, cfg.InternalTelemetryEnabled()) assert.Equal(t, "otel-collector.tracetest", cfg.InternalTelemetryOtelCollectorAddress()) }) - - t.Run("postgresConnStringCompatibility", func(t *testing.T) { - flags := []string{ - "--postgres.dbname", "other_dbname", - "--postgres.host", "localhost", - "--postgres.user", "user", - "--postgres.password", "passwd", - "--postgres.port", "1234", - "--postgres.params", "custom=params", - "--postgresConnString", "host=postgres user=postgres password=postgres port=5432 sslmode=disable", - } - - cfg := configWithFlags(t, flags) - - assert.Equal(t, cfg.PostgresConnString(), "host=postgres user=postgres password=postgres port=5432 sslmode=disable") - }) } diff --git a/server/datastore/datastore_repository.go b/server/datastore/datastore_repository.go index 08527621df..d5f0df2d28 100644 --- a/server/datastore/datastore_repository.go +++ b/server/datastore/datastore_repository.go @@ -25,7 +25,7 @@ func (r *Repository) SetID(dataStore DataStore, id id.ID) DataStore { return dataStore } -const dataStoreSingleID id.ID = "current" +const DataStoreSingleID id.ID = "current" const insertQuery = ` INSERT INTO data_stores ( @@ -66,13 +66,13 @@ func (r *Repository) getCreatedAt(ctx context.Context, dataStore DataStore) (str } func (r *Repository) Create(ctx context.Context, updated DataStore) (DataStore, error) { - updated.ID = dataStoreSingleID + updated.ID = DataStoreSingleID return r.Update(ctx, updated) } func (r *Repository) Update(ctx context.Context, dataStore DataStore) (DataStore, error) { // enforce ID and default - dataStore.ID = dataStoreSingleID + dataStore.ID = DataStoreSingleID dataStore.Default = true // reuse the created_at field for auditing purposes, @@ -91,7 +91,7 @@ func (r *Repository) Update(ctx context.Context, dataStore DataStore) (DataStore } defer tx.Rollback() - _, err = tx.ExecContext(ctx, deleteQuery, dataStoreSingleID) + _, err = tx.ExecContext(ctx, deleteQuery, DataStoreSingleID) if err != nil { return DataStore{}, fmt.Errorf("datastore repository sql exec delete: %w", err) } diff --git a/server/executor/assertion_runner.go b/server/executor/assertion_runner.go index 5c9689f9ee..59fb62aef1 100644 --- a/server/executor/assertion_runner.go +++ b/server/executor/assertion_runner.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "log" - "time" "github.com/kubeshop/tracetest/server/analytics" "github.com/kubeshop/tracetest/server/environment" @@ -14,115 +13,67 @@ import ( "github.com/kubeshop/tracetest/server/pkg/maps" "github.com/kubeshop/tracetest/server/subscription" "github.com/kubeshop/tracetest/server/test" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/propagation" ) -type AssertionRequest struct { - carrier propagation.MapCarrier - Test test.Test - Run test.Run -} - -func (r AssertionRequest) Context() context.Context { - ctx := context.Background() - return otel.GetTextMapPropagator().Extract(ctx, r.carrier) -} - -type AssertionRunner interface { - RunAssertions(ctx context.Context, request AssertionRequest) - WorkerPool -} - type defaultAssertionRunner struct { updater RunUpdater assertionExecutor AssertionExecutor outputsProcessor OutputsProcessorFn - inputChannel chan AssertionRequest - exitChannel chan bool subscriptionManager *subscription.Manager eventEmitter EventEmitter } -var _ WorkerPool = &defaultAssertionRunner{} -var _ AssertionRunner = &defaultAssertionRunner{} - func NewAssertionRunner( updater RunUpdater, assertionExecutor AssertionExecutor, op OutputsProcessorFn, subscriptionManager *subscription.Manager, eventEmitter EventEmitter, -) AssertionRunner { +) *defaultAssertionRunner { return &defaultAssertionRunner{ outputsProcessor: op, updater: updater, assertionExecutor: assertionExecutor, - inputChannel: make(chan AssertionRequest, 1), subscriptionManager: subscriptionManager, eventEmitter: eventEmitter, } } -func (e *defaultAssertionRunner) Start(workers int) { - e.exitChannel = make(chan bool, workers) - - for i := 0; i < workers; i++ { - go e.startWorker() - } +func (e *defaultAssertionRunner) SetOutputQueue(Enqueuer) { + // this is a no-op, as assertion runner does not need to enqueue anything } -func (e *defaultAssertionRunner) Stop() { - ticker := time.NewTicker(1 * time.Second) - for { - select { - case <-ticker.C: - e.exitChannel <- true - return - } - } -} +func (e *defaultAssertionRunner) ProcessItem(ctx context.Context, job Job) { + run, err := e.runAssertionsAndUpdateResult(ctx, job) -func (e *defaultAssertionRunner) startWorker() { - for { - select { - case <-e.exitChannel: - fmt.Println("Exiting assertion executor worker") - return - case request := <-e.inputChannel: - ctx := request.Context() - run, err := e.runAssertionsAndUpdateResult(ctx, request) - - log.Printf("[AssertionRunner] Test %s Run %d: update channel start\n", request.Test.ID, request.Run.ID) - e.subscriptionManager.PublishUpdate(subscription.Message{ - ResourceID: run.TransactionStepResourceID(), - Type: "run_update", - Content: RunResult{Run: run, Err: err}, - }) - log.Printf("[AssertionRunner] Test %s Run %d: update channel complete\n", request.Test.ID, request.Run.ID) - - if err != nil { - log.Printf("[AssertionRunner] Test %s Run %d: error with runAssertionsAndUpdateResult: %s\n", request.Test.ID, request.Run.ID, err.Error()) - } - } + log.Printf("[AssertionRunner] Test %s Run %d: update channel start\n", job.Test.ID, job.Run.ID) + e.subscriptionManager.PublishUpdate(subscription.Message{ + ResourceID: run.TransactionStepResourceID(), + Type: "run_update", + Content: RunResult{Run: run, Err: err}, + }) + log.Printf("[AssertionRunner] Test %s Run %d: update channel complete\n", job.Test.ID, job.Run.ID) + + if err != nil { + log.Printf("[AssertionRunner] Test %s Run %d: error with runAssertionsAndUpdateResult: %s\n", job.Test.ID, job.Run.ID, err.Error()) } } -func (e *defaultAssertionRunner) runAssertionsAndUpdateResult(ctx context.Context, request AssertionRequest) (test.Run, error) { - log.Printf("[AssertionRunner] Test %s Run %d: Starting\n", request.Test.ID, request.Run.ID) +func (e *defaultAssertionRunner) runAssertionsAndUpdateResult(ctx context.Context, job Job) (test.Run, error) { + log.Printf("[AssertionRunner] Test %s Run %d: Starting\n", job.Test.ID, job.Run.ID) - err := e.eventEmitter.Emit(ctx, events.TestSpecsRunStart(request.Test.ID, request.Run.ID)) + err := e.eventEmitter.Emit(ctx, events.TestSpecsRunStart(job.Test.ID, job.Run.ID)) if err != nil { - log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunStart event: %s\n", request.Test.ID, request.Run.ID, err.Error()) + log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunStart event: %s\n", job.Test.ID, job.Run.ID, err.Error()) } - run, err := e.executeAssertions(ctx, request) + run, err := e.executeAssertions(ctx, job) if err != nil { - log.Printf("[AssertionRunner] Test %s Run %d: error executing assertions: %s\n", request.Test.ID, request.Run.ID, err.Error()) + log.Printf("[AssertionRunner] Test %s Run %d: error executing assertions: %s\n", job.Test.ID, job.Run.ID, err.Error()) - anotherErr := e.eventEmitter.Emit(ctx, events.TestSpecsRunError(request.Test.ID, request.Run.ID, err)) + anotherErr := e.eventEmitter.Emit(ctx, events.TestSpecsRunError(job.Test.ID, job.Run.ID, err)) if anotherErr != nil { - log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunError event: %s\n", request.Test.ID, request.Run.ID, anotherErr.Error()) + log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunError event: %s\n", job.Test.ID, job.Run.ID, anotherErr.Error()) } run = run.AssertionFailed(err) @@ -132,29 +83,29 @@ func (e *defaultAssertionRunner) runAssertionsAndUpdateResult(ctx context.Contex return test.Run{}, e.updater.Update(ctx, run) } - log.Printf("[AssertionRunner] Test %s Run %d: Success. pass: %d, fail: %d\n", request.Test.ID, request.Run.ID, run.Pass, run.Fail) + log.Printf("[AssertionRunner] Test %s Run %d: Success. pass: %d, fail: %d\n", job.Test.ID, job.Run.ID, run.Pass, run.Fail) err = e.updater.Update(ctx, run) if err != nil { - log.Printf("[AssertionRunner] Test %s Run %d: error updating run: %s\n", request.Test.ID, request.Run.ID, err.Error()) + log.Printf("[AssertionRunner] Test %s Run %d: error updating run: %s\n", job.Test.ID, job.Run.ID, err.Error()) - anotherErr := e.eventEmitter.Emit(ctx, events.TestSpecsRunPersistenceError(request.Test.ID, request.Run.ID, err)) + anotherErr := e.eventEmitter.Emit(ctx, events.TestSpecsRunPersistenceError(job.Test.ID, job.Run.ID, err)) if anotherErr != nil { - log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunPersistenceError event: %s\n", request.Test.ID, request.Run.ID, anotherErr.Error()) + log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunPersistenceError event: %s\n", job.Test.ID, job.Run.ID, anotherErr.Error()) } return test.Run{}, fmt.Errorf("could not save result on database: %w", err) } - err = e.eventEmitter.Emit(ctx, events.TestSpecsRunSuccess(request.Test.ID, request.Run.ID)) + err = e.eventEmitter.Emit(ctx, events.TestSpecsRunSuccess(job.Test.ID, job.Run.ID)) if err != nil { - log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunSuccess event: %s\n", request.Test.ID, request.Run.ID, err.Error()) + log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestSpecsRunSuccess event: %s\n", job.Test.ID, job.Run.ID, err.Error()) } return run, nil } -func (e *defaultAssertionRunner) executeAssertions(ctx context.Context, req AssertionRequest) (test.Run, error) { +func (e *defaultAssertionRunner) executeAssertions(ctx context.Context, req Job) (test.Run, error) { run := req.Run if run.Trace == nil { return test.Run{}, fmt.Errorf("trace not available") @@ -192,7 +143,7 @@ func (e *defaultAssertionRunner) executeAssertions(ctx context.Context, req Asse return run, nil } -func (e *defaultAssertionRunner) emitFailedAssertions(ctx context.Context, req AssertionRequest, result maps.Ordered[test.SpanQuery, []test.AssertionResult]) { +func (e *defaultAssertionRunner) emitFailedAssertions(ctx context.Context, req Job, result maps.Ordered[test.SpanQuery, []test.AssertionResult]) { for _, assertionResults := range result.Unordered() { for _, assertionResult := range assertionResults { for _, spanAssertionResult := range assertionResult.Results { @@ -238,31 +189,21 @@ func createEnvironment(env environment.Environment, outputs maps.Ordered[string, return env.Merge(outputEnv) } - -func (e *defaultAssertionRunner) RunAssertions(ctx context.Context, request AssertionRequest) { - carrier := propagation.MapCarrier{} - otel.GetTextMapPropagator().Inject(ctx, carrier) - - request.carrier = carrier - - e.inputChannel <- request -} - -func (e *defaultAssertionRunner) validateOutputResolution(ctx context.Context, request AssertionRequest, outputs maps.Ordered[string, test.RunOutput]) { +func (e *defaultAssertionRunner) validateOutputResolution(ctx context.Context, job Job, outputs maps.Ordered[string, test.RunOutput]) { err := outputs.ForEach(func(outputName string, outputModel test.RunOutput) error { if outputModel.Resolved { return nil } - anotherErr := e.eventEmitter.Emit(ctx, events.TestOutputGenerationWarning(request.Test.ID, request.Run.ID, outputModel.Error, outputName)) + anotherErr := e.eventEmitter.Emit(ctx, events.TestOutputGenerationWarning(job.Test.ID, job.Run.ID, outputModel.Error, outputName)) if anotherErr != nil { - log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestOutputGenerationWarning event: %s\n", request.Test.ID, request.Run.ID, anotherErr.Error()) + log.Printf("[AssertionRunner] Test %s Run %d: fail to emit TestOutputGenerationWarning event: %s\n", job.Test.ID, job.Run.ID, anotherErr.Error()) } return nil }) if err != nil { - log.Printf("[AssertionRunner] Test %s Run %d: fail to validate outputs: %s\n", request.Test.ID, request.Run.ID, err.Error()) + log.Printf("[AssertionRunner] Test %s Run %d: fail to validate outputs: %s\n", job.Test.ID, job.Run.ID, err.Error()) } } diff --git a/server/executor/default_poller_executor.go b/server/executor/default_poller_executor.go index 1db6860764..a9532221ff 100644 --- a/server/executor/default_poller_executor.go +++ b/server/executor/default_poller_executor.go @@ -18,7 +18,6 @@ import ( type traceDBFactoryFn func(ds datastore.DataStore) (tracedb.TraceDB, error) type DefaultPollerExecutor struct { - ppGetter PollingProfileGetter updater RunUpdater newTraceDBFn traceDBFactoryFn dsRepo resourcemanager.Current[datastore.DataStore] @@ -27,30 +26,30 @@ type DefaultPollerExecutor struct { type InstrumentedPollerExecutor struct { tracer trace.Tracer - pollerExecutor PollerExecutor + pollerExecutor pollerExecutor } -func (pe InstrumentedPollerExecutor) ExecuteRequest(request *PollingRequest) (bool, string, test.Run, error) { - _, span := pe.tracer.Start(request.Context(), "Fetch trace") +func (pe InstrumentedPollerExecutor) ExecuteRequest(ctx context.Context, job *Job) (PollResult, error) { + _, span := pe.tracer.Start(ctx, "Fetch trace") defer span.End() - finished, finishReason, run, err := pe.pollerExecutor.ExecuteRequest(request) + res, err := pe.pollerExecutor.ExecuteRequest(ctx, job) spanCount := 0 - if run.Trace != nil { - spanCount = len(run.Trace.Flat) + if job.Run.Trace != nil { + spanCount = len(job.Run.Trace.Flat) } attrs := []attribute.KeyValue{ - attribute.String("tracetest.run.trace_poller.trace_id", request.run.TraceID.String()), - attribute.String("tracetest.run.trace_poller.span_id", request.run.SpanID.String()), - attribute.Bool("tracetest.run.trace_poller.succesful", finished), - attribute.String("tracetest.run.trace_poller.test_id", string(request.test.ID)), + attribute.String("tracetest.run.trace_poller.trace_id", job.Run.TraceID.String()), + attribute.String("tracetest.run.trace_poller.span_id", job.Run.SpanID.String()), + attribute.Bool("tracetest.run.trace_poller.succesful", res.Finished()), + attribute.String("tracetest.run.trace_poller.test_id", string(job.Test.ID)), attribute.Int("tracetest.run.trace_poller.amount_retrieved_spans", spanCount), } - if finishReason != "" { - attrs = append(attrs, attribute.String("tracetest.run.trace_poller.finish_reason", finishReason)) + if res.reason != "" { + attrs = append(attrs, attribute.String("tracetest.run.trace_poller.finish_reason", res.reason)) } if err != nil { @@ -59,29 +58,25 @@ func (pe InstrumentedPollerExecutor) ExecuteRequest(request *PollingRequest) (bo } span.SetAttributes(attrs...) - return finished, finishReason, run, err + return res, err } func NewPollerExecutor( - ppGetter PollingProfileGetter, tracer trace.Tracer, updater RunUpdater, newTraceDBFn traceDBFactoryFn, dsRepo resourcemanager.Current[datastore.DataStore], eventEmitter EventEmitter, -) PollerExecutor { - - defaultExecutor := &DefaultPollerExecutor{ - ppGetter: ppGetter, - updater: updater, - newTraceDBFn: newTraceDBFn, - dsRepo: dsRepo, - eventEmitter: eventEmitter, - } +) *InstrumentedPollerExecutor { return &InstrumentedPollerExecutor{ - tracer: tracer, - pollerExecutor: defaultExecutor, + tracer: tracer, + pollerExecutor: &DefaultPollerExecutor{ + updater: updater, + newTraceDBFn: newTraceDBFn, + dsRepo: dsRepo, + eventEmitter: eventEmitter, + }, } } @@ -99,113 +94,125 @@ func (pe DefaultPollerExecutor) traceDB(ctx context.Context) (tracedb.TraceDB, e return tdb, nil } -func (pe DefaultPollerExecutor) ExecuteRequest(request *PollingRequest) (bool, string, test.Run, error) { - log.Printf("[PollerExecutor] Test %s Run %d: ExecuteRequest\n", request.test.ID, request.run.ID) - run := request.run - ctx := request.Context() +func (pe DefaultPollerExecutor) ExecuteRequest(ctx context.Context, job *Job) (PollResult, error) { + log.Printf("[PollerExecutor] Test %s Run %d: ExecuteRequest", job.Test.ID, job.Run.ID) traceDB, err := pe.traceDB(ctx) if err != nil { - log.Printf("[PollerExecutor] Test %s Run %d: GetDataStore error: %s\n", request.test.ID, request.run.ID, err.Error()) - return false, "", test.Run{}, err + log.Printf("[PollerExecutor] Test %s Run %d: GetDataStore error: %s", job.Test.ID, job.Run.ID, err.Error()) + return PollResult{}, err } - if request.IsFirstRequest() { - if testableTraceDB, ok := traceDB.(tracedb.TestableTraceDB); ok { - connectionResult := testableTraceDB.TestConnection(ctx) - - err = pe.eventEmitter.Emit(ctx, events.TraceDataStoreConnectionInfo(request.test.ID, request.run.ID, connectionResult)) - if err != nil { - log.Printf("[PollerExecutor] Test %s Run %d: failed to emit TraceDataStoreConnectionInfo event: error: %s\n", request.test.ID, request.run.ID, err.Error()) - } - } - - endpoints := traceDB.GetEndpoints() - ds, err := pe.dsRepo.Current(ctx) + if isFirstRequest(job) { + err := pe.testConnection(ctx, traceDB, job) if err != nil { - return false, "", test.Run{}, fmt.Errorf("could not get current datastore: %w", err) - } - - err = pe.eventEmitter.Emit(ctx, events.TracePollingStart(request.test.ID, request.run.ID, string(ds.Type), endpoints)) - if err != nil { - log.Printf("[PollerExecutor] Test %s Run %d: failed to emit TracePollingStart event: error: %s\n", request.test.ID, request.run.ID, err.Error()) + return PollResult{}, err } } - traceID := run.TraceID.String() + traceID := job.Run.TraceID.String() trace, err := traceDB.GetTraceByID(ctx, traceID) if err != nil { - anotherErr := pe.eventEmitter.Emit(ctx, events.TracePollingIterationInfo(request.test.ID, request.run.ID, 0, request.count, false, err.Error())) - if anotherErr != nil { - log.Printf("[PollerExecutor] Test %s Run %d: failed to emit TracePollingIterationInfo event: error: %s\n", request.test.ID, request.run.ID, anotherErr.Error()) - } - - log.Printf("[PollerExecutor] Test %s Run %d: GetTraceByID (traceID %s) error: %s\n", request.test.ID, request.run.ID, traceID, err.Error()) - return false, "", test.Run{}, err + pe.emit(ctx, job, events.TracePollingIterationInfo(job.Test.ID, job.Run.ID, 0, job.EnqueueCount(), false, err.Error())) + log.Printf("[PollerExecutor] Test %s Run %d: GetTraceByID (traceID %s) error: %s", job.Test.ID, job.Run.ID, traceID, err.Error()) + return PollResult{}, err } - trace.ID = run.TraceID - done, reason := pe.donePollingTraces(request, traceDB, trace) + trace.ID = job.Run.TraceID + done, reason := pe.donePollingTraces(job, traceDB, trace) + // we need both values to be different to check for done, but after we want to have an updated job + job.Run.Trace = &trace + if !done { - err := pe.eventEmitter.Emit(ctx, events.TracePollingIterationInfo(request.test.ID, request.run.ID, len(trace.Flat), request.count, false, reason)) - if err != nil { - log.Printf("[PollerExecutor] Test %s Run %d: failed to emit TracePollingIterationInfo event: error: %s\n", request.test.ID, request.run.ID, err.Error()) - } + pe.emit(ctx, job, events.TracePollingIterationInfo(job.Test.ID, job.Run.ID, len(job.Run.Trace.Flat), job.EnqueueCount(), false, reason)) + log.Printf("[PollerExecutor] Test %s Run %d: Not done polling. (%s)", job.Test.ID, job.Run.ID, reason) - log.Printf("[PollerExecutor] Test %s Run %d: Not done polling. (%s)\n", request.test.ID, request.run.ID, reason) - run.Trace = &trace - request.run = run - return false, "", run, nil + return PollResult{ + finished: false, + reason: reason, + run: job.Run, + }, nil } + log.Printf("[PollerExecutor] Test %s Run %d: Done polling. (%s)", job.Test.ID, job.Run.ID, reason) - log.Printf("[PollerExecutor] Test %s Run %d: Done polling. (%s)\n", request.test.ID, request.run.ID, reason) + log.Printf("[PollerExecutor] Test %s Run %d: Start Sorting", job.Test.ID, job.Run.ID) + sorted := job.Run.Trace.Sort() + job.Run.Trace = &sorted + log.Printf("[PollerExecutor] Test %s Run %d: Sorting complete", job.Test.ID, job.Run.ID) - log.Printf("[PollerExecutor] Test %s Run %d: Start Sorting\n", request.test.ID, request.run.ID) - trace = trace.Sort() - log.Printf("[PollerExecutor] Test %s Run %d: Sorting complete\n", request.test.ID, request.run.ID) - run.Trace = &trace - request.run = run - - if !trace.HasRootSpan() { - newRoot := test.NewTracetestRootSpan(run) - run.Trace = run.Trace.InsertRootSpan(newRoot) + if !job.Run.Trace.HasRootSpan() { + newRoot := test.NewTracetestRootSpan(job.Run) + job.Run.Trace = job.Run.Trace.InsertRootSpan(newRoot) } else { - run.Trace.RootSpan = model.AugmentRootSpan(run.Trace.RootSpan, run.TriggerResult) + job.Run.Trace.RootSpan = model.AugmentRootSpan(job.Run.Trace.RootSpan, job.Run.TriggerResult) + } + job.Run = job.Run.SuccessfullyPolledTraces(job.Run.Trace) + + log.Printf("[PollerExecutor] Completed polling process for Test Run %d after %d iterations, number of spans collected: %d ", job.Run.ID, job.EnqueueCount()+1, len(job.Run.Trace.Flat)) + + log.Printf("[PollerExecutor] Test %s Run %d: Start updating", job.Test.ID, job.Run.ID) + err = pe.updater.Update(ctx, job.Run) + if err != nil { + log.Printf("[PollerExecutor] Test %s Run %d: Update error: %s", job.Test.ID, job.Run.ID, err.Error()) + return PollResult{}, err } - run = run.SuccessfullyPolledTraces(run.Trace) - fmt.Printf("[PollerExecutor] Completed polling process for Test Run %d after %d iterations, number of spans collected: %d \n", run.ID, request.count+1, len(run.Trace.Flat)) + return PollResult{ + finished: true, + reason: reason, + run: job.Run, + }, nil + +} - log.Printf("[PollerExecutor] Test %s Run %d: Start updating\n", request.test.ID, request.run.ID) - err = pe.updater.Update(ctx, run) +func (pe DefaultPollerExecutor) emit(ctx context.Context, job *Job, event model.TestRunEvent) { + err := pe.eventEmitter.Emit(ctx, event) if err != nil { - log.Printf("[PollerExecutor] Test %s Run %d: Update error: %s\n", request.test.ID, request.run.ID, err.Error()) - return false, "", test.Run{}, err + log.Printf("[PollerExecutor] Test %s Run %d: failed to emit TracePollingIterationInfo event: error: %s", job.Test.ID, job.Run.ID, err.Error()) + } +} + +func (pe DefaultPollerExecutor) testConnection(ctx context.Context, traceDB tracedb.TraceDB, job *Job) error { + if testableTraceDB, ok := traceDB.(tracedb.TestableTraceDB); ok { + connectionResult := testableTraceDB.TestConnection(ctx) + + err := pe.eventEmitter.Emit(ctx, events.TraceDataStoreConnectionInfo(job.Test.ID, job.Run.ID, connectionResult)) + if err != nil { + log.Printf("[PollerExecutor] Test %s Run %d: failed to emit TraceDataStoreConnectionInfo event: error: %s", job.Test.ID, job.Run.ID, err.Error()) + } } - return true, reason, run, nil + endpoints := traceDB.GetEndpoints() + ds, err := pe.dsRepo.Current(ctx) + if err != nil { + return fmt.Errorf("could not get current datastore: %w", err) + } + + err = pe.eventEmitter.Emit(ctx, events.TracePollingStart(job.Test.ID, job.Run.ID, string(ds.Type), endpoints)) + if err != nil { + log.Printf("[PollerExecutor] Test %s Run %d: failed to emit TracePollingStart event: error: %s", job.Test.ID, job.Run.ID, err.Error()) + } + + return nil } -func (pe DefaultPollerExecutor) donePollingTraces(job *PollingRequest, traceDB tracedb.TraceDB, trace model.Trace) (bool, string) { +func (pe DefaultPollerExecutor) donePollingTraces(job *Job, traceDB tracedb.TraceDB, trace model.Trace) (bool, string) { if !traceDB.ShouldRetry() { return true, "TraceDB is not retryable" } - pp := pe.ppGetter.GetDefault(job.Context()) - if pp.Periodic == nil { - return false, "Polling profile not configured" - } - maxTracePollRetry := pp.Periodic.MaxTracePollRetry() + maxTracePollRetry := job.PollingProfile.Periodic.MaxTracePollRetry() // we're done if we have the same amount of spans after polling or `maxTracePollRetry` times - if job.count == maxTracePollRetry { + log.Printf("[PollerExecutor] Test %s Run %d: Job count %d, max retries: %d", job.Test.ID, job.Run.ID, job.EnqueueCount(), maxTracePollRetry) + if job.EnqueueCount() == maxTracePollRetry { return true, fmt.Sprintf("Hit MaxRetry of %d", maxTracePollRetry) } - if job.run.Trace == nil { + if job.Run.Trace == nil { return false, "First iteration" } - haveNotCollectedSpansSinceLastPoll := len(trace.Flat) == len(job.run.Trace.Flat) + haveNotCollectedSpansSinceLastPoll := len(trace.Flat) == len(job.Run.Trace.Flat) haveCollectedSpansInTestRun := len(trace.Flat) > 0 haveCollectedOnlyRootNode := len(trace.Flat) == 1 && trace.HasRootSpan() @@ -218,5 +225,5 @@ func (pe DefaultPollerExecutor) donePollingTraces(job *PollingRequest, traceDB t return true, fmt.Sprintf("Trace has no new spans. Spans found: %d", len(trace.Flat)) } - return false, fmt.Sprintf("New spans found. Before: %d After: %d", len(job.run.Trace.Flat), len(trace.Flat)) + return false, fmt.Sprintf("New spans found. Before: %d After: %d", len(job.Run.Trace.Flat), len(trace.Flat)) } diff --git a/server/executor/linter_runner.go b/server/executor/linter_runner.go index 15bed63efd..f16881c1d8 100644 --- a/server/executor/linter_runner.go +++ b/server/executor/linter_runner.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log" - "time" "github.com/kubeshop/tracetest/server/analytics" "github.com/kubeshop/tracetest/server/linter" @@ -12,184 +11,121 @@ import ( "github.com/kubeshop/tracetest/server/model/events" "github.com/kubeshop/tracetest/server/subscription" "github.com/kubeshop/tracetest/server/test" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/propagation" ) -type LinterRequest struct { - carrier propagation.MapCarrier - Test test.Test - Run test.Run -} - -func (r LinterRequest) Context() context.Context { - ctx := context.Background() - return otel.GetTextMapPropagator().Extract(ctx, r.carrier) -} - -type LinterRunner interface { - RunLinter(ctx context.Context, request LinterRequest) - WorkerPool -} - type AnalyzerGetter interface { GetDefault(ctx context.Context) analyzer.Linter } -type defaultlinterRunner struct { +type defaultLinterRunner struct { updater RunUpdater - inputChannel chan LinterRequest - exitChannel chan bool subscriptionManager *subscription.Manager eventEmitter EventEmitter analyzerGetter AnalyzerGetter - assertionRunner AssertionRunner + outputQueue Enqueuer } -var _ WorkerPool = &defaultlinterRunner{} -var _ LinterRunner = &defaultlinterRunner{} - func NewlinterRunner( updater RunUpdater, subscriptionManager *subscription.Manager, eventEmitter EventEmitter, - assertionRunner AssertionRunner, analyzerGetter AnalyzerGetter, -) LinterRunner { - return &defaultlinterRunner{ +) *defaultLinterRunner { + return &defaultLinterRunner{ updater: updater, - inputChannel: make(chan LinterRequest, 1), subscriptionManager: subscriptionManager, eventEmitter: eventEmitter, - assertionRunner: assertionRunner, analyzerGetter: analyzerGetter, } } -func (e *defaultlinterRunner) Start(workers int) { - e.exitChannel = make(chan bool, workers) - - for i := 0; i < workers; i++ { - go e.startWorker() - } -} - -func (e *defaultlinterRunner) Stop() { - ticker := time.NewTicker(1 * time.Second) - for { - select { - case <-ticker.C: - e.exitChannel <- true - return - } - } +func (e *defaultLinterRunner) SetOutputQueue(queue Enqueuer) { + e.outputQueue = queue } -func (e *defaultlinterRunner) startWorker() { - for { - select { - case <-e.exitChannel: - fmt.Println("Exiting linter executor worker") - return - case request := <-e.inputChannel: - e.onRequest(request) - } - } -} - -func (e *defaultlinterRunner) onRequest(request LinterRequest) { - ctx := request.Context() +func (e *defaultLinterRunner) ProcessItem(ctx context.Context, job Job) { lintResource := e.analyzerGetter.GetDefault(ctx) shouldSkip := lintResource.ShouldSkip() if shouldSkip { - log.Printf("[linterRunner] Skipping Trace Analyzer") - err := e.eventEmitter.Emit(ctx, events.TraceLinterSkip(request.Test.ID, request.Run.ID)) - if err != nil { - log.Printf("[linterRunner] Test %s Run %d: fail to emit TracelinterSkip event: %s\n", request.Test.ID, request.Run.ID, err.Error()) - } - - e.onFinish(ctx, request, request.Run) + e.doSkip(ctx, job) return } lintResource, err := lintResource.WithMetadata() - // in the future, the registry should be dynamic based on user plugins - linter := linter.NewLinter(linter.DefaultPluginRegistry) if err != nil { - log.Printf("[linterRunner] Test %s Run %d: error with WithMetadata: %s\n", request.Test.ID, request.Run.ID, err.Error()) - e.onFinish(ctx, request, request.Run) + log.Printf("[linterRunner] Test %s Run %d: error with WithMetadata: %s\n", job.Test.ID, job.Run.ID, err.Error()) + e.outputQueue.Enqueue(ctx, job) return } - run, err := e.onRun(ctx, request, linter, lintResource) - log.Printf("[linterRunner] Test %s Run %d: update channel start\n", request.Test.ID, request.Run.ID) + // in the future, the registry should be dynamic based on user plugins + linter := linter.NewLinter(linter.DefaultPluginRegistry) + + run, err := e.lint(ctx, job, linter, lintResource) + log.Printf("[linterRunner] Test %s Run %d: update channel start\n", job.Test.ID, job.Run.ID) e.subscriptionManager.PublishUpdate(subscription.Message{ ResourceID: run.TransactionStepResourceID(), Type: "run_update", Content: RunResult{Run: run, Err: err}, }) - log.Printf("[linterRunner] Test %s Run %d: update channel complete\n", request.Test.ID, request.Run.ID) + log.Printf("[linterRunner] Test %s Run %d: update channel complete\n", job.Test.ID, job.Run.ID) if err != nil { - log.Printf("[linterRunner] Test %s Run %d: error with runlinterRunLinterAndUpdateResult: %s\n", request.Test.ID, request.Run.ID, err.Error()) + log.Printf("[linterRunner] Test %s Run %d: error with runlinterRunLinterAndUpdateResult: %s\n", job.Test.ID, job.Run.ID, err.Error()) return } - e.onFinish(ctx, request, run) -} - -func (e *defaultlinterRunner) RunLinter(ctx context.Context, request LinterRequest) { - carrier := propagation.MapCarrier{} - otel.GetTextMapPropagator().Inject(ctx, carrier) - request.carrier = carrier + job.Run = run - e.inputChannel <- request + e.outputQueue.Enqueue(ctx, job) } -func (e *defaultlinterRunner) onFinish(ctx context.Context, request LinterRequest, run test.Run) { - assertionRequest := AssertionRequest{ - Test: request.Test, - Run: run, +func (e *defaultLinterRunner) doSkip(ctx context.Context, job Job) { + log.Printf("[linterRunner] Skipping Trace Analyzer") + err := e.eventEmitter.Emit(ctx, events.TraceLinterSkip(job.Test.ID, job.Run.ID)) + if err != nil { + log.Printf("[linterRunner] Test %s Run %d: fail to emit TracelinterSkip event: %s\n", job.Test.ID, job.Run.ID, err.Error()) } - e.assertionRunner.RunAssertions(ctx, assertionRequest) + + e.outputQueue.Enqueue(ctx, job) } -func (e *defaultlinterRunner) onRun(ctx context.Context, request LinterRequest, linter linter.Linter, analyzer analyzer.Linter) (test.Run, error) { - run := request.Run - log.Printf("[linterRunner] Test %s Run %d: Starting\n", request.Test.ID, request.Run.ID) +func (e *defaultLinterRunner) lint(ctx context.Context, job Job, linterObj linter.Linter, analyzerObj analyzer.Linter) (test.Run, error) { + run := job.Run + log.Printf("[linterRunner] Test %s Run %d: Starting\n", job.Test.ID, job.Run.ID) - err := e.eventEmitter.Emit(ctx, events.TraceLinterStart(request.Test.ID, request.Run.ID)) + err := e.eventEmitter.Emit(ctx, events.TraceLinterStart(job.Test.ID, job.Run.ID)) if err != nil { - log.Printf("[linterRunner] Test %s Run %d: fail to emit TracelinterStart event: %s\n", request.Test.ID, request.Run.ID, err.Error()) + log.Printf("[linterRunner] Test %s Run %d: fail to emit TracelinterStart event: %s\n", job.Test.ID, job.Run.ID, err.Error()) } - result, err := linter.Run(ctx, *run.Trace, analyzer) + result, err := linterObj.Run(ctx, *run.Trace, analyzerObj) if err != nil { - return e.onError(ctx, request, run, err) + return e.onError(ctx, job, run, err) } - result.MinimumScore = analyzer.MinimumScore + result.MinimumScore = analyzerObj.MinimumScore run = run.SuccessfulLinterExecution(result) err = e.updater.Update(ctx, run) if err != nil { - log.Printf("[linterRunner] Test %s Run %d: error updating run: %s\n", request.Test.ID, request.Run.ID, err.Error()) + log.Printf("[linterRunner] Test %s Run %d: error updating run: %s\n", job.Test.ID, job.Run.ID, err.Error()) return test.Run{}, fmt.Errorf("could not save result on database: %w", err) } - err = e.eventEmitter.Emit(ctx, events.TraceLinterSuccess(request.Test.ID, request.Run.ID)) + err = e.eventEmitter.Emit(ctx, events.TraceLinterSuccess(job.Test.ID, job.Run.ID)) if err != nil { - log.Printf("[linterRunner] Test %s Run %d: fail to emit TracelinterSuccess event: %s\n", request.Test.ID, request.Run.ID, err.Error()) + log.Printf("[linterRunner] Test %s Run %d: fail to emit TracelinterSuccess event: %s\n", job.Test.ID, job.Run.ID, err.Error()) } return run, nil } -func (e *defaultlinterRunner) onError(ctx context.Context, request LinterRequest, run test.Run, err error) (test.Run, error) { - log.Printf("[linterRunner] Test %s Run %d: Linter failed. Reason: %s\n", request.Test.ID, request.Run.ID, err.Error()) - anotherErr := e.eventEmitter.Emit(ctx, events.TraceLinterError(request.Test.ID, request.Run.ID, err)) +func (e *defaultLinterRunner) onError(ctx context.Context, job Job, run test.Run, err error) (test.Run, error) { + log.Printf("[linterRunner] Test %s Run %d: Linter failed. Reason: %s\n", job.Test.ID, job.Run.ID, err.Error()) + anotherErr := e.eventEmitter.Emit(ctx, events.TraceLinterError(job.Test.ID, job.Run.ID, err)) if anotherErr != nil { - log.Printf("[linterRunner] Test %s Run %d: fail to emit TracelinterError event: %s\n", request.Test.ID, request.Run.ID, anotherErr.Error()) + log.Printf("[linterRunner] Test %s Run %d: fail to emit TracelinterError event: %s\n", job.Test.ID, job.Run.ID, anotherErr.Error()) } analytics.SendEvent("test_run_finished", "error", "", &map[string]string{ diff --git a/server/executor/pipeline.go b/server/executor/pipeline.go new file mode 100644 index 0000000000..32df2650eb --- /dev/null +++ b/server/executor/pipeline.go @@ -0,0 +1,94 @@ +package executor + +import ( + "context" + "fmt" + "reflect" +) + +type Pipeline struct { + steps []PipelineStep // N + queues []*Queue // N + 1 +} + +type workerDriver interface { + QueueDriver + Start() + Stop() +} + +type queueBuilder interface { + Build(QueueDriver, QueueItemProcessor) *Queue +} + +type pipelineStep interface { + QueueItemProcessor + SetOutputQueue(Enqueuer) +} + +type InputQueueSetter interface { + SetInputQueue(Enqueuer) +} + +type PipelineStep struct { + Driver workerDriver + Processor pipelineStep +} + +func NewPipeline(builder queueBuilder, steps ...PipelineStep) *Pipeline { + pipeline := &Pipeline{ + steps: steps, + queues: make([]*Queue, 0, len(steps)), + } + + // setup an input queue for each pipeline step + for _, step := range steps { + pipeline.queues = append(pipeline.queues, builder.Build(step.Driver, step.Processor)) + } + + // set the output queue for each step. the ouput queue of a processor (N) is the input queue of the next one (N+1) + // the last step has no output queue. + for i, step := range steps { + if i == len(pipeline.queues)-1 { + break + } + + step.Processor.SetOutputQueue(pipeline.queues[i+1]) + + // a processor might need to have a reference to its input queue, to requeue items for example. + // This can be done if it implements the `InputQueueSetter` interace + if setter, ok := reflect.ValueOf(step.Processor).Interface().(InputQueueSetter); ok { + setter.SetInputQueue(pipeline.queues[i]) + } + } + + return pipeline +} + +func (p *Pipeline) GetQueueForStep(i int) *Queue { + if i < 0 || i >= len(p.queues) { + return nil + } + + return p.queues[i] +} + +func (p *Pipeline) Begin(ctx context.Context, job Job) { + if len(p.queues) < 1 { + // this is a panic instead of an error because this could only happen during development + panic(fmt.Errorf("pipeline has no input queue")) + } + p.queues[0].Enqueue(ctx, job) +} + +func (p *Pipeline) Start() { + for _, step := range p.steps { + step.Driver.Start() + } +} + +func (p *Pipeline) Stop() { + for _, step := range p.steps { + step.Driver.Stop() + } +} diff --git a/server/executor/poller_executor_test.go b/server/executor/poller_executor_test.go index 66a24f85f6..6b00270643 100644 --- a/server/executor/poller_executor_test.go +++ b/server/executor/poller_executor_test.go @@ -45,16 +45,16 @@ func Test_PollerExecutor_ExecuteRequest_NoRootSpan_NoSpanCase(t *testing.T) { maxWaitTimeForTrace := 30 * time.Second tracePerIteration := []model.Trace{ - model.Trace{}, - model.Trace{}, + {}, + {}, } // mock external dependencies - pollerExecutor := getPollerExecutorWithMocks(t, retryDelay, maxWaitTimeForTrace, tracePerIteration) + pollerExecutor := getPollerExecutorWithMocks(t, tracePerIteration) // When doing polling process // Then validate outputs - executeAndValidatePollingRequests(t, pollerExecutor, []iterationExpectedValues{ + executeAndValidatePollingRequests(t, retryDelay, maxWaitTimeForTrace, pollerExecutor, []iterationExpectedValues{ {finished: false, expectNoTraceError: true}, {finished: false, expectNoTraceError: true}, }) @@ -96,17 +96,17 @@ func Test_PollerExecutor_ExecuteRequest_NoRootSpan_OneSpanCase(t *testing.T) { // test tracePerIteration := []model.Trace{ - model.Trace{}, + {}, trace, trace, } // mock external dependencies - pollerExecutor := getPollerExecutorWithMocks(t, retryDelay, maxWaitTimeForTrace, tracePerIteration) + pollerExecutor := getPollerExecutorWithMocks(t, tracePerIteration) // When doing polling process // Then validate outputs - executeAndValidatePollingRequests(t, pollerExecutor, []iterationExpectedValues{ + executeAndValidatePollingRequests(t, retryDelay, maxWaitTimeForTrace, pollerExecutor, []iterationExpectedValues{ {finished: false, expectNoTraceError: true}, {finished: false, expectNoTraceError: false, expectRootSpan: false}, {finished: true, expectNoTraceError: false, expectRootSpan: true}, @@ -161,18 +161,18 @@ func Test_PollerExecutor_ExecuteRequest_NoRootSpan_TwoSpansCase(t *testing.T) { // test tracePerIteration := []model.Trace{ - model.Trace{}, + {}, traceWithOneSpan, traceWithTwoSpans, traceWithTwoSpans, } // mock external dependencies - pollerExecutor := getPollerExecutorWithMocks(t, retryDelay, maxWaitTimeForTrace, tracePerIteration) + pollerExecutor := getPollerExecutorWithMocks(t, tracePerIteration) // When doing polling process // Then validate outputs - executeAndValidatePollingRequests(t, pollerExecutor, []iterationExpectedValues{ + executeAndValidatePollingRequests(t, retryDelay, maxWaitTimeForTrace, pollerExecutor, []iterationExpectedValues{ {finished: false, expectNoTraceError: true}, {finished: false, expectNoTraceError: false, expectRootSpan: false}, {finished: false, expectNoTraceError: false, expectRootSpan: false}, @@ -210,18 +210,18 @@ func Test_PollerExecutor_ExecuteRequest_WithRootSpan_NoSpanCase(t *testing.T) { trace := model.NewTrace(randomIDGenerator.TraceID().String(), []model.Span{rootSpan}) tracePerIteration := []model.Trace{ - model.Trace{}, + {}, trace, trace, trace, } // mock external dependencies - pollerExecutor := getPollerExecutorWithMocks(t, retryDelay, maxWaitTimeForTrace, tracePerIteration) + pollerExecutor := getPollerExecutorWithMocks(t, tracePerIteration) // When doing polling process // Then validate outputs - executeAndValidatePollingRequests(t, pollerExecutor, []iterationExpectedValues{ + executeAndValidatePollingRequests(t, retryDelay, maxWaitTimeForTrace, pollerExecutor, []iterationExpectedValues{ {finished: false, expectNoTraceError: true}, {finished: false, expectNoTraceError: false, expectRootSpan: true}, {finished: false, expectNoTraceError: false, expectRootSpan: true}, @@ -250,7 +250,7 @@ func Test_PollerExecutor_ExecuteRequest_WithRootSpan_OneSpanCase(t *testing.T) { rootSpanID := randomIDGenerator.SpanID() trace := model.NewTrace(randomIDGenerator.TraceID().String(), []model.Span{ - model.Span{ + { ID: rootSpanID, Name: model.TriggerSpanName, StartTime: time.Now(), @@ -275,17 +275,17 @@ func Test_PollerExecutor_ExecuteRequest_WithRootSpan_OneSpanCase(t *testing.T) { // test tracePerIteration := []model.Trace{ - model.Trace{}, + {}, trace, trace, } // mock external dependencies - pollerExecutor := getPollerExecutorWithMocks(t, retryDelay, maxWaitTimeForTrace, tracePerIteration) + pollerExecutor := getPollerExecutorWithMocks(t, tracePerIteration) // When doing polling process // Then validate outputs - executeAndValidatePollingRequests(t, pollerExecutor, []iterationExpectedValues{ + executeAndValidatePollingRequests(t, retryDelay, maxWaitTimeForTrace, pollerExecutor, []iterationExpectedValues{ {finished: false, expectNoTraceError: true}, {finished: false, expectNoTraceError: false, expectRootSpan: true}, {finished: true, expectNoTraceError: false, expectRootSpan: true}, @@ -338,7 +338,7 @@ func Test_PollerExecutor_ExecuteRequest_WithRootSpan_OneDelayedSpanCase(t *testi // test tracePerIteration := []model.Trace{ - model.Trace{}, + {}, traceWithOnlyRoot, traceWithOnlyRoot, completeTrace, @@ -346,11 +346,11 @@ func Test_PollerExecutor_ExecuteRequest_WithRootSpan_OneDelayedSpanCase(t *testi } // mock external dependencies - pollerExecutor := getPollerExecutorWithMocks(t, retryDelay, maxWaitTimeForTrace, tracePerIteration) + pollerExecutor := getPollerExecutorWithMocks(t, tracePerIteration) // When doing polling process // Then validate outputs - executeAndValidatePollingRequests(t, pollerExecutor, []iterationExpectedValues{ + executeAndValidatePollingRequests(t, retryDelay, maxWaitTimeForTrace, pollerExecutor, []iterationExpectedValues{ {finished: false, expectNoTraceError: true}, {finished: false, expectNoTraceError: false, expectRootSpan: true}, {finished: false, expectNoTraceError: false, expectRootSpan: true}, @@ -419,18 +419,18 @@ func Test_PollerExecutor_ExecuteRequest_WithRootSpan_TwoSpansCase(t *testing.T) // test tracePerIteration := []model.Trace{ - model.Trace{}, + {}, traceWithOneSpan, traceWithTwoSpans, traceWithTwoSpans, } // mock external dependencies - pollerExecutor := getPollerExecutorWithMocks(t, retryDelay, maxWaitTimeForTrace, tracePerIteration) + pollerExecutor := getPollerExecutorWithMocks(t, tracePerIteration) // When doing polling process // Then validate outputs - executeAndValidatePollingRequests(t, pollerExecutor, []iterationExpectedValues{ + executeAndValidatePollingRequests(t, retryDelay, maxWaitTimeForTrace, pollerExecutor, []iterationExpectedValues{ {finished: false, expectNoTraceError: true}, {finished: false, expectNoTraceError: false, expectRootSpan: true}, {finished: false, expectNoTraceError: false, expectRootSpan: true}, @@ -446,7 +446,7 @@ type iterationExpectedValues struct { expectRootSpan bool } -func executeAndValidatePollingRequests(t *testing.T, pollerExecutor executor.PollerExecutor, expectedValues []iterationExpectedValues) { +func executeAndValidatePollingRequests(t *testing.T, retryDelay, maxWaitTimeForTrace time.Duration, pollerExecutor *executor.InstrumentedPollerExecutor, expectedValues []iterationExpectedValues) { ctx := context.Background() run := test.NewRun() @@ -457,57 +457,55 @@ func executeAndValidatePollingRequests(t *testing.T, pollerExecutor executor.Pol }, } - for i, value := range expectedValues { - request := executor.NewPollingRequest(ctx, test, run, i, pollingprofile.DefaultPollingProfile) + // using `pollingprofile.DefaultPollingProfile` and changing the periodic configs directly + // causes a race condition because `DefaultPollingProfile.Periodic` is a reference, so it's shared by the copies. + // The easiest solution is to create a new polling profile for each test. + pp := pollingprofile.PollingProfile{ + Strategy: pollingprofile.Periodic, + Periodic: &pollingprofile.PeriodicPollingConfig{ + RetryDelay: retryDelay.String(), + Timeout: maxWaitTimeForTrace.String(), + }, + } - finished, finishReason, anotherRun, err := pollerExecutor.ExecuteRequest(request) - run = anotherRun // should store a run to use in another iteration + job := executor.NewJob() + job.Test = test + job.Run = run + job.PollingProfile = pp - require.NotNilf(t, run, "The test run should not be nil on iteration %d", i) + for i, expected := range expectedValues { + res, err := pollerExecutor.ExecuteRequest(ctx, &job) + run = res.Run() // should store a run to use in another iteration - if value.finished { - require.Truef(t, finished, "The poller should have finished on iteration %d", i) - require.NotEmptyf(t, finishReason, "The poller should not have finish reason on iteration %d", i) - } else { - require.Falsef(t, finished, "The poller should have not finished on iteration %d", i) - require.Emptyf(t, finishReason, "The poller should have finish reason on iteration %d", i) - } + require.NotNilf(t, run, "The test run should not be nil on iteration %d", i) - if value.expectNoTraceError { + if expected.expectNoTraceError { require.Errorf(t, err, "An error should have happened on iteration %d", i) require.ErrorIsf(t, err, connection.ErrTraceNotFound, "An connection error should have happened on iteration %d", i) } else { require.NoErrorf(t, err, "An error should not have happened on iteration %d", i) + require.NotEmptyf(t, res.Reason(), "The poller should have a reason on iteration %d", i) + + if expected.finished { + require.Truef(t, res.Finished(), "The poller should have finished on iteration %d", i) + } else { + require.Falsef(t, res.Finished(), "The poller should have not finished on iteration %d", i) + } // only validate root span if we have a root span - if value.expectRootSpan { + if expected.expectRootSpan { require.Truef(t, run.Trace.HasRootSpan(), "The trace associated with the run on iteration %d should have a root span", i) } else { require.Falsef(t, run.Trace.HasRootSpan(), "The trace associated with the run on iteration %d should not have a root span", i) } } - } -} - -type defaultProfileGetter struct { - retryDelay, maxWaitTimeForTrace time.Duration -} - -func (dpc defaultProfileGetter) GetDefault(context.Context) pollingprofile.PollingProfile { - pp := pollingprofile.DefaultPollingProfile - - if dpc.retryDelay > 0 { - pp.Periodic.RetryDelay = dpc.retryDelay.String() - } - if dpc.maxWaitTimeForTrace > 0 { - pp.Periodic.Timeout = dpc.maxWaitTimeForTrace.String() + job.IncreaseEnqueueCount() + job.Headers.SetBool("requeued", true) } - - return pp } -func getPollerExecutorWithMocks(t *testing.T, retryDelay, maxWaitTimeForTrace time.Duration, tracePerIteration []model.Trace) executor.PollerExecutor { +func getPollerExecutorWithMocks(t *testing.T, tracePerIteration []model.Trace) *executor.InstrumentedPollerExecutor { updater := getRunUpdaterMock(t) tracer := getTracerMock(t) testDB := getRunRepositoryMock(t) @@ -516,7 +514,6 @@ func getPollerExecutorWithMocks(t *testing.T, retryDelay, maxWaitTimeForTrace ti eventEmitter := getEventEmitterMock(t, testDB) return executor.NewPollerExecutor( - defaultProfileGetter{retryDelay, maxWaitTimeForTrace}, tracer, updater, traceDBFactory, @@ -532,33 +529,43 @@ type runUpdaterMock struct{} func (m runUpdaterMock) Update(context.Context, test.Run) error { return nil } -func getRunUpdaterMock(t *testing.T) executor.RunUpdater { +func getRunUpdaterMock(_ *testing.T) executor.RunUpdater { return runUpdaterMock{} } // RunRepository -func getRunRepositoryMock(t *testing.T) model.Repository { +func getRunRepositoryMock(t *testing.T) *testdb.MockRepository { t.Helper() - testDB := testdb.MockRepository{} - testDB.Mock.On("CreateTestRunEvent", mock.Anything).Return(noError) + testDB := new(testdb.MockRepository) + testDB.Test(t) + testDB.On("CreateTestRunEvent", mock.Anything).Return(noError) - return &testDB + return testDB } // DataStoreRepository -type dataStoreRepositoryMock struct{} +type dataStoreRepositoryMock struct { + mock.Mock +} -func (m *dataStoreRepositoryMock) Current(ctx context.Context) (datastore.DataStore, error) { +func (m *dataStoreRepositoryMock) Current(context.Context) (datastore.DataStore, error) { return datastore.DataStore{Type: datastore.DataStoreTypeOTLP}, nil } +func (m *dataStoreRepositoryMock) Get(_ context.Context, id id.ID) (datastore.DataStore, error) { + args := m.Called(id) + return args.Get(0).(datastore.DataStore), args.Error(1) +} + func getDataStoreRepositoryMock(t *testing.T) *dataStoreRepositoryMock { - return &dataStoreRepositoryMock{} + m := new(dataStoreRepositoryMock) + m.Test(t) + return m } // EventEmitter -func getEventEmitterMock(t *testing.T, db model.Repository) executor.EventEmitter { +func getEventEmitterMock(t *testing.T, db *testdb.MockRepository) executor.EventEmitter { t.Helper() return executor.NewEventEmitter(db, subscription.NewManager()) @@ -580,7 +587,7 @@ type traceDBMock struct { state *traceDBState } -func (db *traceDBMock) GetTraceByID(ctx context.Context, traceID string) (t model.Trace, err error) { +func (db *traceDBMock) GetTraceByID(_ context.Context, _ string) (t model.Trace, err error) { trace := db.tracePerIteration[db.state.currentIteration] db.state.currentIteration += 1 diff --git a/server/executor/pollingprofile/polling_profile_entities.go b/server/executor/pollingprofile/polling_profile_entities.go index 9fab223bc6..7aa4298b37 100644 --- a/server/executor/pollingprofile/polling_profile_entities.go +++ b/server/executor/pollingprofile/polling_profile_entities.go @@ -32,8 +32,9 @@ var DefaultPollingProfile = PollingProfile{ Default: true, Strategy: Periodic, Periodic: &PeriodicPollingConfig{ - Timeout: "1m", - RetryDelay: "5s", + Timeout: "1m", + RetryDelay: "5s", + SelectorMatchRetries: 3, }, } diff --git a/server/executor/pollingprofile/polling_profile_repository_test.go b/server/executor/pollingprofile/polling_profile_repository_test.go index c8030103c8..d90957689b 100644 --- a/server/executor/pollingprofile/polling_profile_repository_test.go +++ b/server/executor/pollingprofile/polling_profile_repository_test.go @@ -39,7 +39,7 @@ func TestPollingProfileResource(t *testing.T) { "periodic": { "timeout": "1m", "retryDelay": "5s", - "selectorMatchRetries": 0 + "selectorMatchRetries": 3 } } }`, @@ -53,7 +53,7 @@ func TestPollingProfileResource(t *testing.T) { "periodic": { "timeout": "1h", "retryDelay": "25s", - "selectorMatchRetries": 0 + "selectorMatchRetries": 3 } } }`, diff --git a/server/executor/queue.go b/server/executor/queue.go new file mode 100644 index 0000000000..d9a20dbdfb --- /dev/null +++ b/server/executor/queue.go @@ -0,0 +1,476 @@ +package executor + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log" + "strconv" + + "github.com/kubeshop/tracetest/server/datastore" + "github.com/kubeshop/tracetest/server/executor/pollingprofile" + "github.com/kubeshop/tracetest/server/pkg/id" + "github.com/kubeshop/tracetest/server/subscription" + "github.com/kubeshop/tracetest/server/test" + "github.com/kubeshop/tracetest/server/transaction" + "go.opentelemetry.io/otel/propagation" +) + +const ( + JobCountHeader string = "X-Tracetest-Job-Count" +) + +type headers map[string]string + +func (h headers) Get(key string) string { + return h[key] +} + +func (h headers) GetInt(key string) int { + if h.Get(key) == "" { + return 0 + } + + v, err := strconv.Atoi(h[key]) + if err != nil { + panic(fmt.Errorf("cannot convert header %s to int: %w", key, err)) + } + return v +} + +func (h headers) GetBool(key string) bool { + if h.Get(key) == "" { + return false + } + + v, err := strconv.ParseBool(h[key]) + if err != nil { + panic(fmt.Errorf("cannot convert header %s to bool: %w", key, err)) + } + + return v +} + +func (h *headers) Set(key, value string) { + (*h)[key] = value +} + +func (h headers) SetInt(key string, value int) { + h.Set(key, fmt.Sprintf("%d", value)) +} + +func (h headers) SetBool(key string, value bool) { + h.Set(key, fmt.Sprintf("%t", value)) +} + +type Job struct { + Headers *headers + + Transaction transaction.Transaction + TransactionRun transaction.TransactionRun + + Test test.Test + Run test.Run + + PollingProfile pollingprofile.PollingProfile + DataStore datastore.DataStore +} + +func NewJob() Job { + return Job{ + Headers: &headers{}, + } +} + +func (j Job) EnqueueCount() int { + return j.Headers.GetInt(JobCountHeader) +} + +func (j *Job) IncreaseEnqueueCount() { + j.Headers.SetInt(JobCountHeader, j.EnqueueCount()+1) +} + +type Enqueuer interface { + Enqueue(context.Context, Job) +} + +type QueueItemProcessor interface { + ProcessItem(context.Context, Job) +} + +type pollingProfileGetter interface { + Get(context.Context, id.ID) (pollingprofile.PollingProfile, error) +} + +type dataStoreGetter interface { + Get(context.Context, id.ID) (datastore.DataStore, error) +} + +type testGetter interface { + GetAugmented(context.Context, id.ID) (test.Test, error) +} + +type testRunGetter interface { + GetRun(_ context.Context, testID id.ID, runID int) (test.Run, error) +} + +type transactionGetter interface { + GetAugmented(context.Context, id.ID) (transaction.Transaction, error) +} + +type transactionRunGetter interface { + GetTransactionRun(_ context.Context, transactionID id.ID, runID int) (transaction.TransactionRun, error) +} + +type subscriptor interface { + Subscribe(string, subscription.Subscriber) +} + +type QueueDriver interface { + Enqueue(Job) + SetQueue(*Queue) +} + +type QueueBuilder struct { + cancelRunHandlerFn func(ctx context.Context, run test.Run) error + subscriptor subscriptor + + runs testRunGetter + tests testGetter + + transactionRuns transactionRunGetter + transactions transactionGetter + + pollingProfiles pollingProfileGetter + dataStores dataStoreGetter +} + +func NewQueueBuilder() *QueueBuilder { + return &QueueBuilder{} +} + +func (qb *QueueBuilder) WithCancelRunHandlerFn(fn func(ctx context.Context, run test.Run) error) *QueueBuilder { + qb.cancelRunHandlerFn = fn + return qb +} + +func (qb *QueueBuilder) WithSubscriptor(subscriptor subscriptor) *QueueBuilder { + qb.subscriptor = subscriptor + return qb +} + +func (qb *QueueBuilder) WithRunGetter(runs testRunGetter) *QueueBuilder { + qb.runs = runs + return qb +} + +func (qb *QueueBuilder) WithTestGetter(tests testGetter) *QueueBuilder { + qb.tests = tests + return qb +} + +func (qb *QueueBuilder) WithPollingProfileGetter(pollingProfiles pollingProfileGetter) *QueueBuilder { + qb.pollingProfiles = pollingProfiles + return qb +} + +func (qb *QueueBuilder) WithDataStoreGetter(dataStore dataStoreGetter) *QueueBuilder { + qb.dataStores = dataStore + return qb +} + +func (qb *QueueBuilder) WithTransactionGetter(transactions transactionGetter) *QueueBuilder { + qb.transactions = transactions + return qb +} + +func (qb *QueueBuilder) WithTransactionRunGetter(transactionRuns transactionRunGetter) *QueueBuilder { + qb.transactionRuns = transactionRuns + return qb +} + +func (qb *QueueBuilder) Build(driver QueueDriver, itemProcessor QueueItemProcessor) *Queue { + queue := &Queue{ + cancelRunHandlerFn: qb.cancelRunHandlerFn, + subscriptor: qb.subscriptor, + + runs: qb.runs, + tests: qb.tests, + + transactionRuns: qb.transactionRuns, + transactions: qb.transactions, + + pollingProfiles: qb.pollingProfiles, + dataStores: qb.dataStores, + + driver: driver, + itemProcessor: itemProcessor, + } + + driver.SetQueue(queue) + + return queue +} + +type Queue struct { + cancelRunHandlerFn func(ctx context.Context, run test.Run) error + subscriptor subscriptor + + runs testRunGetter + tests testGetter + + transactionRuns transactionRunGetter + transactions transactionGetter + + pollingProfiles pollingProfileGetter + dataStores dataStoreGetter + + itemProcessor QueueItemProcessor + driver QueueDriver +} + +func (q *Queue) SetDriver(driver QueueDriver) { + q.driver = driver + driver.SetQueue(q) +} + +func (q Queue) Enqueue(ctx context.Context, job Job) { + select { + default: + case <-ctx.Done(): + return + } + + if job.Headers == nil { + job.Headers = &headers{} + } + propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) + propagator.Inject(ctx, propagation.MapCarrier(*job.Headers)) + + newJob := Job{ + Headers: job.Headers, + + Test: test.Test{ID: job.Test.ID}, + Run: job.Run, + + Transaction: transaction.Transaction{ID: job.Transaction.ID}, + TransactionRun: transaction.TransactionRun{ID: job.TransactionRun.ID}, + + PollingProfile: pollingprofile.PollingProfile{ID: job.PollingProfile.ID}, + DataStore: datastore.DataStore{ID: job.DataStore.ID}, + } + + q.driver.Enqueue(newJob) +} + +func (q Queue) Listen(job Job) { + // this is called when a new job is put in the queue and we need to process it + propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) + ctx := propagator.Extract(context.Background(), propagation.MapCarrier(*job.Headers)) + + ctx, cancelCtx := context.WithCancel(ctx) + q.listenForStopRequests(context.Background(), cancelCtx, job) + + newJob := Job{ + Headers: job.Headers, + } + newJob.Test = q.resolveTest(ctx, job) + // todo: revert when using actual queues + // newJob.Run = q.resolveTestRun(ctx, job) + newJob.Run = job.Run + + newJob.Transaction = q.resolveTransaction(ctx, job) + newJob.TransactionRun = q.resolveTransactionRun(ctx, job) + + newJob.PollingProfile = q.resolvePollingProfile(ctx, job) + newJob.DataStore = q.resolveDataStore(ctx, job) + + // Process the item with cancellation monitoring + select { + default: + case <-ctx.Done(): + return + } + q.itemProcessor.ProcessItem(ctx, newJob) +} + +type StopRequest struct { + TestID id.ID + RunID int +} + +func (sr StopRequest) ResourceID() string { + runID := (test.Run{ID: sr.RunID, TestID: sr.TestID}).ResourceID() + return runID + "/stop" +} + +func (q Queue) listenForStopRequests(ctx context.Context, cancelCtx context.CancelFunc, job Job) { + if q.subscriptor == nil { + return + } + + sfn := subscription.NewSubscriberFunction(func(m subscription.Message) error { + cancelCtx() + stopRequest, ok := m.Content.(StopRequest) + if !ok { + return nil + } + + run, err := q.runs.GetRun(ctx, stopRequest.TestID, stopRequest.RunID) + if err != nil { + return fmt.Errorf("failed to get run %d for test %s: %w", stopRequest.RunID, stopRequest.TestID, err) + } + + if run.State == test.RunStateStopped { + return nil + } + + return q.cancelRunHandlerFn(ctx, run) + + }) + + q.subscriptor.Subscribe((StopRequest{job.Test.ID, job.Run.ID}).ResourceID(), sfn) +} + +func (q Queue) resolveTransaction(ctx context.Context, job Job) transaction.Transaction { + if q.transactions == nil { + return transaction.Transaction{} + } + + tran, err := q.transactions.GetAugmented(ctx, job.Transaction.ID) + if errors.Is(err, sql.ErrNoRows) { + return transaction.Transaction{} + } + if err != nil { + panic(err) + } + + return tran +} +func (q Queue) resolveTransactionRun(ctx context.Context, job Job) transaction.TransactionRun { + if q.transactionRuns == nil { + return transaction.TransactionRun{} + } + + tranRun, err := q.transactionRuns.GetTransactionRun(ctx, job.Transaction.ID, job.TransactionRun.ID) + if errors.Is(err, sql.ErrNoRows) { + return transaction.TransactionRun{} + } + if err != nil { + panic(err) + } + + return tranRun +} + +func (q Queue) resolveTest(ctx context.Context, job Job) test.Test { + if q.tests == nil { + return test.Test{} + } + + t, err := q.tests.GetAugmented(ctx, job.Test.ID) + if errors.Is(err, sql.ErrNoRows) { + return test.Test{} + } + if err != nil { + panic(err) + } + + return t +} + +func (q Queue) resolveTestRun(ctx context.Context, job Job) test.Run { + if q.runs == nil { + return test.Run{} + } + + run, err := q.runs.GetRun(ctx, job.Test.ID, job.Run.ID) + if errors.Is(err, sql.ErrNoRows) { + return test.Run{} + } + if err != nil { + panic(err) + } + + return run +} + +func (q Queue) resolvePollingProfile(ctx context.Context, job Job) pollingprofile.PollingProfile { + if q.pollingProfiles == nil { + return pollingprofile.PollingProfile{} + } + + profile, err := q.pollingProfiles.Get(ctx, job.PollingProfile.ID) + if errors.Is(err, sql.ErrNoRows) { + return pollingprofile.PollingProfile{} + } + if err != nil { + panic(err) + } + + return profile +} + +func (q Queue) resolveDataStore(ctx context.Context, job Job) datastore.DataStore { + if q.dataStores == nil { + return datastore.DataStore{} + } + + ds, err := q.dataStores.Get(ctx, job.DataStore.ID) + if errors.Is(err, sql.ErrNoRows) { + return datastore.DataStore{} + } + if err != nil { + panic(err) + } + + return ds +} + +func NewInMemoryQueueDriver(name string) *InMemoryQueueDriver { + return &InMemoryQueueDriver{ + queue: make(chan Job), + exit: make(chan bool), + name: name, + } +} + +type InMemoryQueueDriver struct { + queue chan Job + exit chan bool + q *Queue + name string +} + +func (r *InMemoryQueueDriver) SetQueue(q *Queue) { + r.q = q +} + +func (r InMemoryQueueDriver) Enqueue(job Job) { + r.queue <- job +} + +const inMemoryQueueWorkerCount = 5 + +func (r InMemoryQueueDriver) Start() { + for i := 0; i < inMemoryQueueWorkerCount; i++ { + go func() { + log.Printf("[InMemoryQueueDriver - %s] start", r.name) + for { + select { + case <-r.exit: + log.Printf("[InMemoryQueueDriver - %s] exit", r.name) + return + case job := <-r.queue: + r.q.Listen(job) + } + } + }() + } +} + +func (r InMemoryQueueDriver) Stop() { + log.Printf("[InMemoryQueueDriver - %s] stopping", r.name) + r.exit <- true +} diff --git a/server/executor/run_stop.go b/server/executor/run_stop.go deleted file mode 100644 index 7f3569ec67..0000000000 --- a/server/executor/run_stop.go +++ /dev/null @@ -1,54 +0,0 @@ -package executor - -import ( - "context" - "log" - - "github.com/kubeshop/tracetest/server/model/events" - "github.com/kubeshop/tracetest/server/pkg/id" - "github.com/kubeshop/tracetest/server/subscription" - "github.com/kubeshop/tracetest/server/test" -) - -type StopRequest struct { - TestID id.ID - RunID int -} - -func (sr StopRequest) ResourceID() string { - runID := (test.Run{ID: sr.RunID, TestID: sr.TestID}).ResourceID() - return runID + "/stop" -} - -func (r persistentRunner) listenForStopRequests(ctx context.Context, cancelCtx context.CancelFunc, run test.Run) { - sfn := subscription.NewSubscriberFunction(func(m subscription.Message) error { - stopRequest, ok := m.Content.(StopRequest) - if !ok { - return nil - } - - ctx, _ := r.tracer.Start(ctx, "User Requested Stop Run") - // refresh data from DB to make sure we have the latest possible data before updating - run, err := r.runs.GetRun(ctx, stopRequest.TestID, stopRequest.RunID) - if err != nil { - log.Printf("[TracePoller] Test %s Run %d: fail to get test run data: %s \n", stopRequest.TestID, stopRequest.RunID, err.Error()) - return err - } - - run = run.Stopped() - r.handleDBError(run, r.updater.Update(ctx, run)) - - evt := events.TraceStoppedInfo(stopRequest.TestID, stopRequest.RunID) - err = r.eventEmitter.Emit(ctx, evt) - if err != nil { - log.Printf("[TracePoller] Test %s Run %d: fail to emit TraceStoppedInfo event: %s \n", stopRequest.TestID, stopRequest.RunID, err.Error()) - return err - } - - cancelCtx() - - return nil - }) - - r.subscriptionManager.Subscribe((StopRequest{run.TestID, run.ID}).ResourceID(), sfn) -} diff --git a/server/executor/run_updater.go b/server/executor/run_updater.go index be547f61bc..53047f0b68 100644 --- a/server/executor/run_updater.go +++ b/server/executor/run_updater.go @@ -34,10 +34,14 @@ func (u CompositeUpdater) Update(ctx context.Context, run test.Run) error { } type dbUpdater struct { - repo test.RunRepository + repo runDBUpdater } -func NewDBUpdater(repo test.RunRepository) RunUpdater { +type runDBUpdater interface { + UpdateRun(context.Context, test.Run) error +} + +func NewDBUpdater(repo runDBUpdater) RunUpdater { return dbUpdater{repo} } diff --git a/server/executor/runner.go b/server/executor/runner.go index 7139509ae2..05e73e2199 100644 --- a/server/executor/runner.go +++ b/server/executor/runner.go @@ -10,19 +10,14 @@ import ( "github.com/kubeshop/tracetest/server/analytics" "github.com/kubeshop/tracetest/server/datastore" - "github.com/kubeshop/tracetest/server/environment" - "github.com/kubeshop/tracetest/server/executor/testrunner" triggerer "github.com/kubeshop/tracetest/server/executor/trigger" "github.com/kubeshop/tracetest/server/expression" "github.com/kubeshop/tracetest/server/model" "github.com/kubeshop/tracetest/server/model/events" - "github.com/kubeshop/tracetest/server/resourcemanager" "github.com/kubeshop/tracetest/server/subscription" "github.com/kubeshop/tracetest/server/test" "github.com/kubeshop/tracetest/server/test/trigger" "github.com/kubeshop/tracetest/server/tracedb" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" ) @@ -31,72 +26,49 @@ type RunResult struct { Err error } -type Runner interface { - Run(context.Context, test.Test, test.RunMetadata, environment.Environment, *[]testrunner.RequiredGate) test.Run -} - type PersistentRunner interface { - Runner - WorkerPool + QueueItemProcessor } -type TestRunnerGetter interface { - GetDefault(ctx context.Context) testrunner.TestRunner -} +const ProcessorName = "test_runner" func NewPersistentRunner( triggers *triggerer.Registry, - runs test.RunRepository, updater RunUpdater, - tp TracePoller, tracer trace.Tracer, subscriptionManager *subscription.Manager, newTraceDBFn traceDBFactoryFn, - dsRepo resourcemanager.Current[datastore.DataStore], + dsRepo currentDataStoreGetter, eventEmitter EventEmitter, - ppGetter PollingProfileGetter, - trGetter TestRunnerGetter, -) PersistentRunner { - return persistentRunner{ +) *persistentRunner { + return &persistentRunner{ triggers: triggers, - runs: runs, updater: updater, - tp: tp, tracer: tracer, newTraceDBFn: newTraceDBFn, dsRepo: dsRepo, subscriptionManager: subscriptionManager, eventEmitter: eventEmitter, - ppGetter: ppGetter, - trGetter: trGetter, - executeQueue: make(chan execReq, 5), - exit: make(chan bool, 1), } } +type currentDataStoreGetter interface { + Current(context.Context) (datastore.DataStore, error) +} + type persistentRunner struct { triggers *triggerer.Registry - tp TracePoller - runs test.RunRepository updater RunUpdater tracer trace.Tracer subscriptionManager *subscription.Manager newTraceDBFn traceDBFactoryFn - dsRepo resourcemanager.Current[datastore.DataStore] + dsRepo currentDataStoreGetter eventEmitter EventEmitter - ppGetter PollingProfileGetter - trGetter TestRunnerGetter - - executeQueue chan execReq - exit chan bool + outputQueue Enqueuer } -type execReq struct { - ctx context.Context - test test.Test - run test.Run - Headers propagation.MapCarrier - executor expression.Executor +func (r *persistentRunner) SetOutputQueue(queue Enqueuer) { + r.outputQueue = queue } func (r persistentRunner) handleDBError(run test.Run, err error) { @@ -111,79 +83,6 @@ func (r persistentRunner) handleError(run test.Run, err error) { } } -func (r persistentRunner) Start(workers int) { - for i := 0; i < workers; i++ { - go func() { - fmt.Println("persistentRunner start goroutine") - for { - select { - case <-r.exit: - fmt.Println("persistentRunner exit goroutine") - return - case job := <-r.executeQueue: - fmt.Printf( - "persistentRunner job. ID %d, testID %s, TraceID %s, SpanID %s\n", - job.run.ID, - job.test.ID, - job.run.TraceID, - job.run.SpanID, - ) - r.processExecQueue(job) - } - } - }() - } -} - -func (r persistentRunner) Stop() { - fmt.Println("persistentRunner stopping") - r.exit <- true -} - -func getNewCtx(ctx context.Context) context.Context { - carrier := propagation.MapCarrier{} - otel.GetTextMapPropagator().Inject(ctx, carrier) - - return otel.GetTextMapPropagator().Extract(context.Background(), carrier) -} - -func (r persistentRunner) Run(ctx context.Context, testObj test.Test, metadata test.RunMetadata, environment environment.Environment, requiredGates *[]testrunner.RequiredGate) test.Run { - ctx, cancelCtx := context.WithCancel( - getNewCtx(ctx), - ) - - run := test.NewRun() - run.Metadata = metadata - run.Environment = environment - run, err := r.runs.CreateRun(ctx, testObj, run) - r.handleDBError(run, err) - - r.listenForStopRequests(ctx, cancelCtx, run) - - // configuring required gates - if requiredGates == nil { - rg := r.trGetter.GetDefault(ctx).RequiredGates - requiredGates = &rg - } - - run = run.ConfigureRequiredGates(*requiredGates) - - ds := []expression.DataStore{expression.EnvironmentDataStore{ - Values: environment.Values, - }} - - executor := expression.NewExecutor(ds...) - - r.executeQueue <- execReq{ - ctx: ctx, - test: testObj, - run: run, - executor: executor, - } - - return run -} - func (r persistentRunner) traceDB(ctx context.Context) (tracedb.TraceDB, error) { ds, err := r.dsRepo.Current(ctx) if err != nil { @@ -198,80 +97,86 @@ func (r persistentRunner) traceDB(ctx context.Context) (tracedb.TraceDB, error) return tdb, nil } -func (r persistentRunner) processExecQueue(job execReq) { - run := job.run.Start() - r.handleDBError(run, r.updater.Update(job.ctx, run)) +func (r persistentRunner) ProcessItem(ctx context.Context, job Job) { + run := job.Run.Start() + r.handleDBError(run, r.updater.Update(ctx, run)) - err := r.eventEmitter.Emit(job.ctx, events.TriggerCreatedInfo(job.run.TestID, job.run.ID)) + err := r.eventEmitter.Emit(ctx, events.TriggerCreatedInfo(job.Run.TestID, job.Run.ID)) if err != nil { - r.handleError(job.run, err) + r.handleError(job.Run, err) } - triggererObj, err := r.triggers.Get(job.test.Trigger.Type) + triggererObj, err := r.triggers.Get(job.Test.Trigger.Type) if err != nil { - r.handleError(job.run, err) + r.handleError(job.Run, err) } - tdb, err := r.traceDB(job.ctx) + tdb, err := r.traceDB(ctx) if err != nil { - r.handleError(job.run, err) + r.handleError(job.Run, err) } traceID := tdb.GetTraceID() run.TraceID = traceID - r.handleDBError(run, r.updater.Update(job.ctx, run)) + r.handleDBError(run, r.updater.Update(ctx, run)) + + ds := []expression.DataStore{expression.EnvironmentDataStore{ + Values: run.Environment.Values, + }} + + executor := expression.NewExecutor(ds...) triggerOptions := &triggerer.TriggerOptions{ TraceID: traceID, - Executor: job.executor, + Executor: executor, } - err = r.eventEmitter.Emit(job.ctx, events.TriggerResolveStart(job.run.TestID, job.run.ID)) + err = r.eventEmitter.Emit(ctx, events.TriggerResolveStart(job.Run.TestID, job.Run.ID)) if err != nil { - r.handleError(job.run, err) + r.handleError(job.Run, err) } - resolvedTest, err := triggererObj.Resolve(job.ctx, job.test, triggerOptions) + resolvedTest, err := triggererObj.Resolve(ctx, job.Test, triggerOptions) if err != nil { - emitErr := r.eventEmitter.Emit(job.ctx, events.TriggerResolveError(job.run.TestID, job.run.ID, err)) + emitErr := r.eventEmitter.Emit(ctx, events.TriggerResolveError(job.Run.TestID, job.Run.ID, err)) if emitErr != nil { - r.handleError(job.run, emitErr) + r.handleError(job.Run, emitErr) } - r.handleError(job.run, err) + r.handleError(job.Run, err) } - err = r.eventEmitter.Emit(job.ctx, events.TriggerResolveSuccess(job.run.TestID, job.run.ID)) + err = r.eventEmitter.Emit(ctx, events.TriggerResolveSuccess(job.Run.TestID, job.Run.ID)) if err != nil { - r.handleError(job.run, err) + r.handleError(job.Run, err) } - if job.test.Trigger.Type == trigger.TriggerTypeTraceID { - traceIDFromParam, err := trace.TraceIDFromHex(job.test.Trigger.TraceID.ID) + if job.Test.Trigger.Type == trigger.TriggerTypeTraceID { + traceIDFromParam, err := trace.TraceIDFromHex(job.Test.Trigger.TraceID.ID) if err == nil { run.TraceID = traceIDFromParam } } - err = r.eventEmitter.Emit(job.ctx, events.TriggerExecutionStart(job.run.TestID, job.run.ID)) + err = r.eventEmitter.Emit(ctx, events.TriggerExecutionStart(job.Run.TestID, job.Run.ID)) if err != nil { - r.handleError(job.run, err) + r.handleError(job.Run, err) } - response, err := triggererObj.Trigger(job.ctx, resolvedTest, triggerOptions) + response, err := triggererObj.Trigger(ctx, resolvedTest, triggerOptions) run = r.handleExecutionResult(run, response, err) if err != nil { if isConnectionError(err) { - r.emitUnreachableEndpointEvent(job, err) + r.emitUnreachableEndpointEvent(ctx, job, err) if isTargetLocalhost(job) && isServerRunningInsideContainer() { - r.emitMismatchEndpointEvent(job, err) + r.emitMismatchEndpointEvent(ctx, job, err) } } - emitErr := r.eventEmitter.Emit(job.ctx, events.TriggerExecutionError(job.run.TestID, job.run.ID, err)) + emitErr := r.eventEmitter.Emit(ctx, events.TriggerExecutionError(job.Run.TestID, job.Run.ID, err)) if emitErr != nil { - r.handleError(job.run, emitErr) + r.handleError(job.Run, emitErr) } fmt.Printf("test %s run #%d trigger error: %s\n", run.TestID, run.ID, err.Error()) @@ -281,20 +186,23 @@ func (r persistentRunner) processExecQueue(job execReq) { Content: RunResult{Run: run, Err: err}, }) } else { - err = r.eventEmitter.Emit(job.ctx, events.TriggerExecutionSuccess(job.run.TestID, job.run.ID)) + err = r.eventEmitter.Emit(ctx, events.TriggerExecutionSuccess(job.Run.TestID, job.Run.ID)) if err != nil { - r.handleDBError(job.run, err) + r.handleDBError(job.Run, err) } } run.SpanID = response.SpanID - r.handleDBError(run, r.updater.Update(job.ctx, run)) - if run.State == test.RunStateAwaitingTrace { - ctx, pollingSpan := r.tracer.Start(job.ctx, "Start Polling trace") - defer pollingSpan.End() - r.tp.Poll(ctx, job.test, run, r.ppGetter.GetDefault(ctx)) + r.handleDBError(run, r.updater.Update(ctx, run)) + if run.State != test.RunStateAwaitingTrace { + return } + + job.Run = run + ctx, pollingSpan := r.tracer.Start(ctx, "Start Polling trace") + defer pollingSpan.End() + r.outputQueue.Enqueue(ctx, job) } func (r persistentRunner) handleExecutionResult(run test.Run, response triggerer.Response, err error) test.Run { @@ -312,25 +220,25 @@ func (r persistentRunner) handleExecutionResult(run test.Run, response triggerer return run.SuccessfullyTriggered() } -func (r persistentRunner) emitUnreachableEndpointEvent(job execReq, err error) { +func (r persistentRunner) emitUnreachableEndpointEvent(ctx context.Context, job Job, err error) { var event model.TestRunEvent - switch job.test.Trigger.Type { + switch job.Test.Trigger.Type { case trigger.TriggerTypeHTTP: - event = events.TriggerHTTPUnreachableHostError(job.run.TestID, job.run.ID, err) + event = events.TriggerHTTPUnreachableHostError(job.Run.TestID, job.Run.ID, err) case trigger.TriggerTypeGRPC: - event = events.TriggergRPCUnreachableHostError(job.run.TestID, job.run.ID, err) + event = events.TriggergRPCUnreachableHostError(job.Run.TestID, job.Run.ID, err) } - emitErr := r.eventEmitter.Emit(job.ctx, event) + emitErr := r.eventEmitter.Emit(ctx, event) if emitErr != nil { - r.handleError(job.run, emitErr) + r.handleError(job.Run, emitErr) } } -func (r persistentRunner) emitMismatchEndpointEvent(job execReq, err error) { - emitErr := r.eventEmitter.Emit(job.ctx, events.TriggerDockerComposeHostMismatchError(job.run.TestID, job.run.ID)) +func (r persistentRunner) emitMismatchEndpointEvent(ctx context.Context, job Job, err error) { + emitErr := r.eventEmitter.Emit(ctx, events.TriggerDockerComposeHostMismatchError(job.Run.TestID, job.Run.ID)) if emitErr != nil { - r.handleError(job.run, emitErr) + r.handleError(job.Run, emitErr) } } @@ -352,13 +260,13 @@ func isConnectionError(err error) bool { return false } -func isTargetLocalhost(job execReq) bool { +func isTargetLocalhost(job Job) bool { var endpoint string - switch job.test.Trigger.Type { + switch job.Test.Trigger.Type { case trigger.TriggerTypeHTTP: - endpoint = job.test.Trigger.HTTP.URL + endpoint = job.Test.Trigger.HTTP.URL case trigger.TriggerTypeGRPC: - endpoint = job.test.Trigger.GRPC.Address + endpoint = job.Test.Trigger.GRPC.Address } url, err := url.Parse(endpoint) diff --git a/server/executor/runner_test.go b/server/executor/runner_test.go index 1728c55671..e78014986e 100644 --- a/server/executor/runner_test.go +++ b/server/executor/runner_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/kubeshop/tracetest/server/config" + "github.com/kubeshop/tracetest/server/datastore" "github.com/kubeshop/tracetest/server/environment" "github.com/kubeshop/tracetest/server/executor" "github.com/kubeshop/tracetest/server/executor/pollingprofile" @@ -22,16 +23,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" ) -type defaultTestRunnerGetter struct{} - -func (dpc defaultTestRunnerGetter) GetDefault(context.Context) testrunner.TestRunner { - testRunner := testrunner.DefaultTestRunner - - return testRunner -} - func TestPersistentRunner(t *testing.T) { t.Run("TestIsTriggerd", func(t *testing.T) { t.Parallel() @@ -46,7 +40,7 @@ func TestPersistentRunner(t *testing.T) { f.run([]test.Test{testObj}, 10*time.Millisecond) - result := f.mockDB.runs[testObj.ID] + result := f.runsMock.runs[testObj.ID] require.NotNil(t, result) assert.Greater(t, result.ServiceTriggerCompletedAt.UnixNano(), result.CreatedAt.UnixNano()) @@ -65,8 +59,8 @@ func TestPersistentRunner(t *testing.T) { f.run([]test.Test{test1, test2}, 100*time.Millisecond) - run1 := f.mockDB.runs[test1.ID] - run2 := f.mockDB.runs[test2.ID] + run1 := f.runsMock.runs[test1.ID] + run2 := f.runsMock.runs[test2.ID] assert.Greater(t, run1.ServiceTriggerCompletedAt.UnixNano(), run2.ServiceTriggerCompletedAt.UnixNano(), "test1 did not complete after test2") }) @@ -98,99 +92,189 @@ var ( ) type runnerFixture struct { - runner executor.PersistentRunner - mockExecutor *mockTriggerer - mockDB *mockDB - mockTracePoller *mockTracePoller + runner *executor.TestPipeline + dsMock *datastoreGetterMock + ppMock *pollingprofileGetterMock + trMock *testrunnerGetterMock + testMock *testGetterMock + runsMock *runsRepoMock + triggererMock *mockTriggerer + processorMock *mockProcessor } func (f runnerFixture) run(tests []test.Test, ttl time.Duration) { - f.runner.Start(2) + // TODO - fix this test + f.runner.Start() time.Sleep(10 * time.Millisecond) for _, testObj := range tests { - f.runner.Run(context.TODO(), testObj, test.RunMetadata{}, environment.Environment{}, nil) + newRun := f.runner.Run(context.TODO(), testObj, test.RunMetadata{}, environment.Environment{}, nil) + // readd this when using not-in-memory queues + // f.runsMock. + // On("GetRun", testObj.ID, newRun.ID). + // Return(newRun, noError) + f.processorMock. + On("ProcessItem", testObj.ID, newRun.ID, datastore.DataStoreSingleID, pollingprofile.DefaultPollingProfile.ID). + Return(newRun, noError) } time.Sleep(ttl) f.runner.Stop() } +func (f runnerFixture) assert(t *testing.T) { + f.dsMock.AssertExpectations(t) + f.ppMock.AssertExpectations(t) + f.trMock.AssertExpectations(t) + f.testMock.AssertExpectations(t) + f.runsMock.AssertExpectations(t) + f.triggererMock.AssertExpectations(t) +} + func (f runnerFixture) expectSuccessExecLong(test test.Test) { - f.mockExecutor.expectTriggerTestLong(test) + f.triggererMock.expectTriggerTestLong(test) f.expectSuccessResultPersist(test) } func (f runnerFixture) expectSuccessExec(test test.Test) { - f.mockExecutor.expectTriggerTest(test) + f.testMock.On("GetAugmented", test.ID).Return(test, noError) + f.triggererMock.expectTriggerTest(test) f.expectSuccessResultPersist(test) } func (f runnerFixture) expectSuccessResultPersist(test test.Test) { - expectCreateRun(f.mockDB, test) - f.mockDB.On("UpdateRun", mock.Anything).Return(noError) - f.mockDB.On("UpdateRun", mock.Anything).Return(noError) - f.mockTracePoller.expectPoll(test) -} - -func (f runnerFixture) assert(t *testing.T) { - f.mockExecutor.AssertExpectations(t) - f.mockDB.AssertExpectations(t) + f.testMock.On("GetAugmented", test.ID).Return(test, noError) + expectCreateRun(f.runsMock, test) + f.runsMock.On("UpdateRun", mock.Anything).Return(noError) + f.runsMock.On("UpdateRun", mock.Anything).Return(noError) } func runnerSetup(t *testing.T) runnerFixture { - tr, _ := tracing.NewTracer(context.TODO(), config.Must(config.New())) - reg := triggerer.NewRegsitry(tr, tr) - me := new(mockTriggerer) - me.t = t - me.Test(t) - reg.Add(me) + dsMock := new(datastoreGetterMock) + dsMock.Test(t) - db := new(mockDB) - db.T = t - db.Test(t) + ppMock := new(pollingprofileGetterMock) + ppMock.Test(t) - mtp := new(mockTracePoller) - mtp.t = t + trMock := new(testrunnerGetterMock) + trMock.Test(t) - tracer, _ := tracing.NewTracer(context.Background(), config.Must(config.New())) + testMock := new(testGetterMock) + testMock.Test(t) + + runsMock := new(runsRepoMock) + runsMock.Test(t) - testDB := testdb.MockRepository{} - testDB.Mock.On("CreateTestRunEvent", mock.Anything).Return(noError) + triggererMock := new(mockTriggerer) + runsMock.Test(t) + + processorMock := new(mockProcessor) + processorMock.Test(t) + + sm := subscription.NewManager() + tracer, _ := tracing.NewTracer(context.Background(), config.Must(config.New())) + eventEmitter := executor.NewEventEmitter(getTestRunEventRepositoryMock(t, false), sm) - eventEmitter := executor.NewEventEmitter(&testDB, subscription.NewManager()) + registry := triggerer.NewRegsitry(tracer, tracer) + registry.Add(triggererMock) - persistentRunner := executor.NewPersistentRunner( - reg, - db, - executor.NewDBUpdater(db), - mtp, + runner := executor.NewPersistentRunner( + registry, + executor.NewDBUpdater(runsMock), tracer, - subscription.NewManager(), - tracedb.Factory(db), - getDataStoreRepositoryMock(t), + sm, + tracedb.Factory(runsMock), + dsMock, eventEmitter, - defaultProfileGetter{5 * time.Second, 30 * time.Second}, - defaultTestRunnerGetter{}, ) - mtp.Test(t) + queueBuilder := executor.NewQueueBuilder(). + WithDataStoreGetter(dsMock). + WithPollingProfileGetter(ppMock). + WithTestGetter(testMock). + WithRunGetter(runsMock) + + pipeline := executor.NewPipeline(queueBuilder, + executor.PipelineStep{Processor: runner, Driver: executor.NewInMemoryQueueDriver("runner")}, + executor.PipelineStep{Processor: processorMock, Driver: executor.NewInMemoryQueueDriver("runner")}, + ) + + testPipeline := executor.NewTestPipeline( + pipeline, + nil, + pipeline.GetQueueForStep(1), // processorMock queue + runsMock, + trMock, + ppMock, + dsMock, + ) + return runnerFixture{ - runner: persistentRunner, - mockExecutor: me, - mockDB: db, - mockTracePoller: mtp, + runner: testPipeline, + dsMock: dsMock, + ppMock: ppMock, + trMock: trMock, + testMock: testMock, + runsMock: runsMock, + triggererMock: triggererMock, + processorMock: processorMock, } } -type mockDB struct { +type mockProcessor struct { + mock.Mock +} + +func (m *mockProcessor) ProcessItem(_ context.Context, job executor.Job) { + m.Called(job.Test.ID, job.Run.ID, job.DataStore.ID, job.PollingProfile.ID) +} + +func (m *mockProcessor) SetOutputQueue(_ executor.Enqueuer) {} + +type datastoreGetterMock struct { + mock.Mock +} + +func (r *datastoreGetterMock) Get(ctx context.Context, id id.ID) (datastore.DataStore, error) { + return r.Current(ctx) +} + +func (r *datastoreGetterMock) Current(context.Context) (datastore.DataStore, error) { + return datastore.DataStore{ + ID: datastore.DataStoreSingleID, + Name: "test", + Type: datastore.DataStoreTypeOTLP, + Default: true, + }, nil +} + +type pollingprofileGetterMock struct { + mock.Mock +} + +func (r *pollingprofileGetterMock) Get(ctx context.Context, _ id.ID) (pollingprofile.PollingProfile, error) { + return r.GetDefault(ctx), nil +} + +func (r *pollingprofileGetterMock) GetDefault(context.Context) pollingprofile.PollingProfile { + return pollingprofile.DefaultPollingProfile +} + +type testGetterMock struct { + mock.Mock +} + +func (r *testGetterMock) GetAugmented(_ context.Context, id id.ID) (test.Test, error) { + args := r.Called(id) + return args.Get(0).(test.Test), args.Error(1) +} + +type runsRepoMock struct { testdb.MockRepository runs map[id.ID]test.Run } -var _ test.RunRepository = &mockDB{} - -func (m *mockDB) CreateRun(_ context.Context, testObj test.Test, run test.Run) (test.Run, error) { +func (m *runsRepoMock) CreateRun(_ context.Context, testObj test.Test, run test.Run) (test.Run, error) { args := m.Called(testObj.ID) if m.runs == nil { m.runs = map[id.ID]test.Run{} @@ -202,7 +286,7 @@ func (m *mockDB) CreateRun(_ context.Context, testObj test.Test, run test.Run) ( return run, args.Error(0) } -func (m *mockDB) UpdateRun(_ context.Context, run test.Run) error { +func (m *runsRepoMock) UpdateRun(_ context.Context, run test.Run) error { args := m.Called(run.ID) for k, v := range m.runs { if v.ID == run.ID { @@ -213,18 +297,26 @@ func (m *mockDB) UpdateRun(_ context.Context, run test.Run) error { return args.Error(0) } -func (m *mockDB) GetTransactionRunSteps(ctx context.Context, id id.ID, runID int) ([]test.Run, error) { - args := m.Called(ctx, id, runID) - return args.Get(0).([]test.Run), args.Error(1) +func (r *runsRepoMock) GetRun(_ context.Context, testID id.ID, runID int) (test.Run, error) { + args := r.Called(testID, runID) + return args.Get(0).(test.Run), args.Error(1) +} + +func (r *runsRepoMock) GetRunByTraceID(_ context.Context, id trace.TraceID) (test.Run, error) { + args := r.Called(id) + return args.Get(0).(test.Run), args.Error(1) } -type mockRunRepository struct { +type testrunnerGetterMock struct { mock.Mock } +func (r *testrunnerGetterMock) GetDefault(context.Context) testrunner.TestRunner { + return testrunner.DefaultTestRunner +} + type mockTriggerer struct { mock.Mock - t *testing.T } func (m *mockTriggerer) Type() trigger.TriggerType { @@ -258,22 +350,8 @@ func (m *mockTriggerer) expectTriggerTestLong(test test.Test) *mock.Call { Return(test, noError) } -func expectCreateRun(m *mockDB, test test.Test) *mock.Call { +func expectCreateRun(m *runsRepoMock, test test.Test) *mock.Call { return m. On("CreateRun", test.ID). Return(noError) } - -type mockTracePoller struct { - mock.Mock - t *testing.T -} - -func (m *mockTracePoller) Poll(_ context.Context, test test.Test, run test.Run, pollingProfile pollingprofile.PollingProfile) { - m.Called(test.ID) -} - -func (m *mockTracePoller) expectPoll(test test.Test) *mock.Call { - return m. - On("Poll", test.ID) -} diff --git a/server/executor/selector_based_poller_executor.go b/server/executor/selector_based_poller_executor.go index 5d1b8f60a6..e17ef05f0f 100644 --- a/server/executor/selector_based_poller_executor.go +++ b/server/executor/selector_based_poller_executor.go @@ -1,92 +1,86 @@ package executor import ( + "context" "fmt" - "strconv" "github.com/kubeshop/tracetest/server/model/events" - "github.com/kubeshop/tracetest/server/test" ) const ( selectorBasedPollerExecutorRetryHeader = "SelectorBasedPollerExecutor.retryCount" - selectorBasedPollerExecutorMaxTries = 3 ) type selectorBasedPollerExecutor struct { - pollerExecutor PollerExecutor + pollerExecutor pollerExecutor eventEmitter EventEmitter } -func NewSelectorBasedPoller(innerPoller PollerExecutor, eventEmitter EventEmitter) PollerExecutor { +func NewSelectorBasedPoller(innerPoller pollerExecutor, eventEmitter EventEmitter) selectorBasedPollerExecutor { return selectorBasedPollerExecutor{innerPoller, eventEmitter} } -func (pe selectorBasedPollerExecutor) ExecuteRequest(request *PollingRequest) (bool, string, test.Run, error) { - ready, reason, run, err := pe.pollerExecutor.ExecuteRequest(request) - if !ready { - request.SetHeaderInt(selectorBasedPollerExecutorRetryHeader, 0) - return ready, reason, run, err +func (pe selectorBasedPollerExecutor) ExecuteRequest(ctx context.Context, job *Job) (PollResult, error) { + res, err := pe.pollerExecutor.ExecuteRequest(ctx, job) + if !res.finished { + job.Headers.SetInt(selectorBasedPollerExecutorRetryHeader, 0) + return res, err } maxNumberRetries := 0 - if request.pollingProfile.Periodic != nil { - maxNumberRetries = request.pollingProfile.Periodic.SelectorMatchRetries + if job.PollingProfile.Periodic != nil { + maxNumberRetries = job.PollingProfile.Periodic.SelectorMatchRetries } - currentNumberTries := pe.getNumberTries(request) + currentNumberTries := job.Headers.GetInt(selectorBasedPollerExecutorRetryHeader) if currentNumberTries >= maxNumberRetries { - pe.eventEmitter.Emit(request.Context(), events.TracePollingIterationInfo( - request.test.ID, - request.run.ID, - len(request.run.Trace.Flat), - request.count, + pe.eventEmitter.Emit(ctx, events.TracePollingIterationInfo( + job.Test.ID, + job.Run.ID, + len(job.Run.Trace.Flat), + currentNumberTries, true, fmt.Sprintf("Some selectors did not match any spans in the current trace, but after %d tries, the trace probably won't change", currentNumberTries), )) - return true, "", run, err + res.finished = true + return res, err } - allSelectorsMatchSpans := pe.allSelectorsMatchSpans(request) + allSelectorsMatchSpans := pe.allSelectorsMatchSpans(job) if allSelectorsMatchSpans { - pe.eventEmitter.Emit(request.Context(), events.TracePollingIterationInfo( - request.test.ID, - request.run.ID, - len(request.run.Trace.Flat), - request.count, + pe.eventEmitter.Emit(ctx, events.TracePollingIterationInfo( + job.Test.ID, + job.Run.ID, + len(job.Run.Trace.Flat), + currentNumberTries, true, "All selectors from the test matched at least one span in the current trace", )) - return true, "", run, err + res.finished = true + return res, err } - request.SetHeaderInt(selectorBasedPollerExecutorRetryHeader, currentNumberTries+1) + job.Headers.SetInt(selectorBasedPollerExecutorRetryHeader, currentNumberTries+1) - pe.eventEmitter.Emit(request.Context(), events.TracePollingIterationInfo( - request.test.ID, - request.run.ID, - len(request.run.Trace.Flat), - request.count, + pe.eventEmitter.Emit(ctx, events.TracePollingIterationInfo( + job.Test.ID, + job.Run.ID, + len(job.Run.Trace.Flat), + job.Headers.GetInt(selectorBasedPollerExecutorRetryHeader), false, "All selectors from your test must match at least one span in the trace, some of them did not match any", )) - return false, "not all selectors got matching spans in the trace", run, err -} - -func (pe selectorBasedPollerExecutor) getNumberTries(request *PollingRequest) int { - value := request.Header(selectorBasedPollerExecutorRetryHeader) - if intValue, err := strconv.Atoi(value); err == nil { - return intValue - } + res.finished = false + res.reason = "not all selectors got matching spans in the trace" - return 0 + return res, err } -func (pe selectorBasedPollerExecutor) allSelectorsMatchSpans(request *PollingRequest) bool { +func (pe selectorBasedPollerExecutor) allSelectorsMatchSpans(job *Job) bool { allSelectorsHaveMatch := true - for _, spec := range request.test.Specs { - spans := selector(spec.Selector).Filter(*request.run.Trace) + for _, spec := range job.Test.Specs { + spans := selector(spec.Selector).Filter(*job.Run.Trace) if len(spans) == 0 { allSelectorsHaveMatch = false } diff --git a/server/executor/selector_based_poller_executor_test.go b/server/executor/selector_based_poller_executor_test.go index fb32ad9082..d287978773 100644 --- a/server/executor/selector_based_poller_executor_test.go +++ b/server/executor/selector_based_poller_executor_test.go @@ -16,14 +16,11 @@ type defaultPollerMock struct { mock.Mock } -// ExecuteRequest implements executor.PollerExecutor -func (m *defaultPollerMock) ExecuteRequest(request *executor.PollingRequest) (bool, string, test.Run, error) { - args := m.Called(request) - return args.Bool(0), args.String(1), args.Get(2).(test.Run), args.Error(3) +func (m *defaultPollerMock) ExecuteRequest(_ context.Context, job *executor.Job) (executor.PollResult, error) { + args := m.Called(job) + return args.Get(0).(executor.PollResult), args.Error(1) } -var _ executor.PollerExecutor = &defaultPollerMock{} - type eventEmitterMock struct { mock.Mock } @@ -41,12 +38,16 @@ func TestSelectorBasedPollerExecutor(t *testing.T) { eventEmitter := new(eventEmitterMock) eventEmitter.On("Emit", mock.Anything, mock.Anything).Return(nil) - createRequest := func(test test.Test, run test.Run) *executor.PollingRequest { + createRequest := func(test test.Test, run test.Run) *executor.Job { pollingProfile := pollingprofile.DefaultPollingProfile pollingProfile.Periodic.SelectorMatchRetries = 3 - request := executor.NewPollingRequest(context.Background(), test, run, 0, pollingProfile) - return request + job := executor.NewJob() + job.Test = test + job.Run = run + job.PollingProfile = pollingProfile + + return &job } t.Run("should return false when default poller returns false", func(t *testing.T) { @@ -55,10 +56,10 @@ func TestSelectorBasedPollerExecutor(t *testing.T) { request := createRequest(test.Test{}, test.Run{}) - defaultPoller.On("ExecuteRequest", mock.Anything).Return(false, "", test.Run{}, nil) - ready, _, _, _ := selectorBasedPoller.ExecuteRequest(request) + defaultPoller.On("ExecuteRequest", mock.Anything).Return(executor.PollResult{}, nil) + res, _ := selectorBasedPoller.ExecuteRequest(context.TODO(), request) - assert.False(t, ready) + assert.False(t, res.Finished()) }) t.Run("should return false when default poller returns true but not all selector match at least one span", func(t *testing.T) { @@ -76,11 +77,11 @@ func TestSelectorBasedPollerExecutor(t *testing.T) { request := createRequest(testObj, run) - defaultPoller.On("ExecuteRequest", mock.Anything).Return(true, "all spans found", run, nil) - ready, _, _, _ := selectorBasedPoller.ExecuteRequest(request) + defaultPoller.On("ExecuteRequest", mock.Anything).Return(executor.NewPollResult(true, "all spans found", run), nil) + res, _ := selectorBasedPoller.ExecuteRequest(context.TODO(), request) - assert.False(t, ready) - assert.Equal(t, "1", request.Header("SelectorBasedPollerExecutor.retryCount")) + assert.False(t, res.Finished()) + assert.Equal(t, 1, request.Headers.GetInt("SelectorBasedPollerExecutor.retryCount")) }) t.Run("should return true if default poller returns true and selectors don't match spans 3 times in a row", func(t *testing.T) { @@ -98,27 +99,27 @@ func TestSelectorBasedPollerExecutor(t *testing.T) { request := createRequest(testObj, run) - defaultPoller.On("ExecuteRequest", mock.Anything).Return(false, "trace not found", run, nil).Once() + defaultPoller.On("ExecuteRequest", mock.Anything).Return(executor.NewPollResult(false, "trace not found", run), nil).Once() - ready, _, _, _ := selectorBasedPoller.ExecuteRequest(request) - assert.False(t, ready) + res, _ := selectorBasedPoller.ExecuteRequest(context.TODO(), request) + assert.False(t, res.Finished()) - defaultPoller.On("ExecuteRequest", mock.Anything).Return(true, "all spans found", run, nil) + defaultPoller.On("ExecuteRequest", mock.Anything).Return(executor.NewPollResult(true, "all spans found", run), nil) - ready, _, _, _ = selectorBasedPoller.ExecuteRequest(request) - assert.False(t, ready) - assert.Equal(t, "1", request.Header("SelectorBasedPollerExecutor.retryCount")) + res, _ = selectorBasedPoller.ExecuteRequest(context.TODO(), request) + assert.False(t, res.Finished()) + assert.Equal(t, 1, request.Headers.GetInt("SelectorBasedPollerExecutor.retryCount")) - ready, _, _, _ = selectorBasedPoller.ExecuteRequest(request) - assert.False(t, ready) - assert.Equal(t, "2", request.Header("SelectorBasedPollerExecutor.retryCount")) + res, _ = selectorBasedPoller.ExecuteRequest(context.TODO(), request) + assert.False(t, res.Finished()) + assert.Equal(t, 2, request.Headers.GetInt("SelectorBasedPollerExecutor.retryCount")) - ready, _, _, _ = selectorBasedPoller.ExecuteRequest(request) - assert.False(t, ready) - assert.Equal(t, "3", request.Header("SelectorBasedPollerExecutor.retryCount")) + res, _ = selectorBasedPoller.ExecuteRequest(context.TODO(), request) + assert.False(t, res.Finished()) + assert.Equal(t, 3, request.Headers.GetInt("SelectorBasedPollerExecutor.retryCount")) - ready, _, _, _ = selectorBasedPoller.ExecuteRequest(request) - assert.True(t, ready) + res, _ = selectorBasedPoller.ExecuteRequest(context.TODO(), request) + assert.True(t, res.Finished()) }) t.Run("should return true if default poller returns true and each selector match at least one span", func(t *testing.T) { @@ -140,9 +141,9 @@ func TestSelectorBasedPollerExecutor(t *testing.T) { request := createRequest(testObj, run) - defaultPoller.On("ExecuteRequest", mock.Anything).Return(true, "all spans found", run, nil) + defaultPoller.On("ExecuteRequest", mock.Anything).Return(executor.NewPollResult(true, "all spans found", run), nil) - ready, _, _, _ := selectorBasedPoller.ExecuteRequest(request) - assert.True(t, ready) + res, _ := selectorBasedPoller.ExecuteRequest(context.TODO(), request) + assert.True(t, res.Finished()) }) } diff --git a/server/executor/test_pipeline.go b/server/executor/test_pipeline.go new file mode 100644 index 0000000000..6e105af529 --- /dev/null +++ b/server/executor/test_pipeline.go @@ -0,0 +1,162 @@ +package executor + +import ( + "context" + "fmt" + "log" + + "github.com/kubeshop/tracetest/server/environment" + "github.com/kubeshop/tracetest/server/executor/pollingprofile" + "github.com/kubeshop/tracetest/server/executor/testrunner" + "github.com/kubeshop/tracetest/server/model/events" + "github.com/kubeshop/tracetest/server/pkg/id" + "github.com/kubeshop/tracetest/server/subscription" + "github.com/kubeshop/tracetest/server/test" + "go.opentelemetry.io/otel/trace" +) + +type TestPipeline struct { + *Pipeline + updatePublisher updatePublisher + assertionQueue Enqueuer + runs runsRepo + trGetter testRunnerGetter + ppGetter defaultPollingProfileGetter + dsGetter currentDataStoreGetter +} + +type runsRepo interface { + CreateRun(context.Context, test.Test, test.Run) (test.Run, error) + UpdateRun(context.Context, test.Run) error + GetRun(_ context.Context, testID id.ID, runID int) (test.Run, error) +} + +type testRunnerGetter interface { + GetDefault(ctx context.Context) testrunner.TestRunner +} + +type defaultPollingProfileGetter interface { + GetDefault(ctx context.Context) pollingprofile.PollingProfile +} + +type updatePublisher interface { + PublishUpdate(subscription.Message) +} + +func NewTestPipeline( + pipeline *Pipeline, + + updatePublisher updatePublisher, + + assertionQueue Enqueuer, + runs runsRepo, + trGetter testRunnerGetter, + ppGetter defaultPollingProfileGetter, + dsGetter currentDataStoreGetter, +) *TestPipeline { + return &TestPipeline{ + Pipeline: pipeline, + updatePublisher: updatePublisher, + assertionQueue: assertionQueue, + runs: runs, + trGetter: trGetter, + ppGetter: ppGetter, + dsGetter: dsGetter, + } +} + +func (p *TestPipeline) Run(ctx context.Context, testObj test.Test, metadata test.RunMetadata, environment environment.Environment, requiredGates *[]testrunner.RequiredGate) test.Run { + run := test.NewRun() + run.Metadata = metadata + run.Environment = environment + + // configuring required gates + if requiredGates == nil { + rg := p.trGetter.GetDefault(ctx).RequiredGates + requiredGates = &rg + } + run = run.ConfigureRequiredGates(*requiredGates) + + run, err := p.runs.CreateRun(ctx, testObj, run) + p.handleDBError(run, err) + + datastore, err := p.dsGetter.Current(ctx) + p.handleDBError(run, err) + + job := NewJob() + job.Test = testObj + job.Run = run + job.PollingProfile = p.ppGetter.GetDefault(ctx) + job.DataStore = datastore + + p.Pipeline.Begin(ctx, job) + + return run +} + +func (p *TestPipeline) Rerun(ctx context.Context, testObj test.Test, runID int) test.Run { + run, err := p.runs.GetRun(ctx, testObj.ID, runID) + p.handleDBError(run, err) + + newTestRun, err := p.runs.CreateRun(ctx, testObj, run.Copy()) + p.handleDBError(run, err) + + err = p.runs.UpdateRun(ctx, newTestRun.SuccessfullyPolledTraces(run.Trace)) + p.handleDBError(run, err) + + ds, err := p.dsGetter.Current(ctx) + p.handleDBError(run, err) + + p.assertionQueue.Enqueue(ctx, Job{ + Test: testObj, + Run: newTestRun, + PollingProfile: p.ppGetter.GetDefault(ctx), + DataStore: ds, + }) + + return newTestRun +} + +func (p *TestPipeline) StopTest(ctx context.Context, testID id.ID, runID int) { + sr := StopRequest{ + TestID: testID, + RunID: runID, + } + + p.updatePublisher.PublishUpdate(subscription.Message{ + ResourceID: sr.ResourceID(), + Content: sr, + }) +} + +type runCancelHandlerFn func(ctx context.Context, run test.Run) error + +func HandleRunCancelation(updater RunUpdater, tracer trace.Tracer, eventEmitter EventEmitter) runCancelHandlerFn { + return func(ctx context.Context, run test.Run) error { + ctx, span := tracer.Start(ctx, "User Requested Stop Run") + defer span.End() + + if run.State == test.RunStateStopped { + return nil + } + err := updater.Update(ctx, run.Stopped()) + if err != nil { + fmt.Printf("test %s run #%d cancel DB error: %s\n", run.TestID, run.ID, err.Error()) + } + + evt := events.TraceStoppedInfo(run.TestID, run.ID) + err = eventEmitter.Emit(ctx, evt) + if err != nil { + log.Printf("[HandleRunCancelation] Test %s Run %d: fail to emit TraceStoppedInfo event: %s \n", run.TestID, run.ID, err.Error()) + return err + } + + return nil + } +} + +func (p *TestPipeline) handleDBError(run test.Run, err error) { + if err != nil { + fmt.Printf("test %s run #%d DB error: %s\n", run.TestID, run.ID, err.Error()) + } +} diff --git a/server/executor/trace_poller.go b/server/executor/trace_poller.go index b047d3e595..4ab45bf969 100644 --- a/server/executor/trace_poller.go +++ b/server/executor/trace_poller.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "log" - "strconv" "time" "github.com/kubeshop/tracetest/server/analytics" @@ -14,7 +13,6 @@ import ( "github.com/kubeshop/tracetest/server/subscription" "github.com/kubeshop/tracetest/server/test" "github.com/kubeshop/tracetest/server/tracedb/connection" - "go.opentelemetry.io/otel/propagation" v1 "go.opentelemetry.io/proto/otlp/trace/v1" ) @@ -29,33 +27,49 @@ type PersistentTracePoller interface { WorkerPool } -type PollerExecutor interface { - ExecuteRequest(*PollingRequest) (bool, string, test.Run, error) +func NewPollResult(finished bool, reason string, run test.Run) PollResult { + return PollResult{ + finished: finished, + reason: reason, + run: run, + } } -type TraceFetcher interface { - GetTraceByID(ctx context.Context, traceID string) (*v1.TracesData, error) +type PollResult struct { + finished bool + reason string + run test.Run +} + +func (pr PollResult) Finished() bool { + return pr.finished +} + +func (pr PollResult) Reason() string { + return pr.reason } -type PollingProfileGetter interface { - GetDefault(ctx context.Context) pollingprofile.PollingProfile +func (pr PollResult) Run() test.Run { + return pr.run +} + +type pollerExecutor interface { + ExecuteRequest(context.Context, *Job) (PollResult, error) +} + +type TraceFetcher interface { + GetTraceByID(ctx context.Context, traceID string) (*v1.TracesData, error) } func NewTracePoller( - pe PollerExecutor, - ppGetter PollingProfileGetter, + pe pollerExecutor, updater RunUpdater, - linterRunner LinterRunner, subscriptionManager *subscription.Manager, eventEmitter EventEmitter, -) PersistentTracePoller { - return tracePoller{ +) *tracePoller { + return &tracePoller{ updater: updater, - ppGetter: ppGetter, pollerExecutor: pe, - executeQueue: make(chan PollingRequest, 5), - exit: make(chan bool, 1), - linterRunner: linterRunner, subscriptionManager: subscriptionManager, eventEmitter: eventEmitter, } @@ -63,79 +77,11 @@ func NewTracePoller( type tracePoller struct { updater RunUpdater - ppGetter PollingProfileGetter - pollerExecutor PollerExecutor - linterRunner LinterRunner + pollerExecutor pollerExecutor subscriptionManager *subscription.Manager eventEmitter EventEmitter - - executeQueue chan PollingRequest - exit chan bool -} - -type PollingRequest struct { - test test.Test - run test.Run - pollingProfile pollingprofile.PollingProfile - count int - headers map[string]string -} - -func (r PollingRequest) Context() context.Context { - propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) - return propagator.Extract(context.Background(), propagation.MapCarrier(r.headers)) -} - -func (pr *PollingRequest) SetHeader(name, value string) { - pr.headers[name] = value -} - -func (pr *PollingRequest) SetHeaderInt(name string, value int) { - pr.headers[name] = strconv.Itoa(value) -} - -func (pr *PollingRequest) SetHeaderBool(name string, value bool) { - pr.headers[name] = fmt.Sprintf("%t", value) -} - -func (pr *PollingRequest) Header(name string) string { - return pr.headers[name] -} - -func (pr *PollingRequest) HeaderInt(name string) int { - if value, err := strconv.Atoi(pr.headers[name]); err == nil { - return value - } - - return 0 -} - -func (pr *PollingRequest) HeaderBool(name string) bool { - if value, err := strconv.ParseBool(pr.headers[name]); err == nil { - return value - } - - return false -} - -func (pr PollingRequest) IsFirstRequest() bool { - return !pr.HeaderBool("requeued") -} - -func NewPollingRequest(ctx context.Context, test test.Test, run test.Run, count int, pollingProfile pollingprofile.PollingProfile) *PollingRequest { - propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) - - request := &PollingRequest{ - test: test, - run: run, - headers: make(map[string]string), - pollingProfile: pollingProfile, - count: count, - } - - propagator.Inject(ctx, propagation.MapCarrier(request.headers)) - - return request + inputQueue Enqueuer + outputQueue Enqueuer } func (tp tracePoller) handleDBError(err error) { @@ -144,129 +90,99 @@ func (tp tracePoller) handleDBError(err error) { } } -func (tp tracePoller) Start(workers int) { - for i := 0; i < workers; i++ { - go func() { - fmt.Println("tracePoller start goroutine") - for { - select { - case <-tp.exit: - fmt.Println("tracePoller exit goroutine") - return - case job := <-tp.executeQueue: - log.Printf("[TracePoller] Test %s Run %d: Received job\n", job.test.ID, job.run.ID) - tp.processJob(job) - } - } - }() +func (tp tracePoller) enqueueJob(ctx context.Context, job Job) { + select { + default: + case <-ctx.Done(): + return } + tp.inputQueue.Enqueue(ctx, job) } -func (tp tracePoller) Stop() { - fmt.Println("tracePoller stopping") - tp.exit <- true +func (tp tracePoller) isFirstRequest(job Job) bool { + return job.EnqueueCount() == 0 } -func (tp tracePoller) Poll(ctx context.Context, test test.Test, run test.Run, pollingProfile pollingprofile.PollingProfile) { - log.Printf("[TracePoller] Test %s Run %d: Poll\n", test.ID, run.ID) - - job := NewPollingRequest(ctx, test, run, PollingRequestStartIteration, pollingProfile) - - tp.enqueueJob(*job) +func (tp *tracePoller) SetOutputQueue(queue Enqueuer) { + tp.outputQueue = queue } -func (tp tracePoller) enqueueJob(job PollingRequest) { - tp.executeQueue <- job +func (tp *tracePoller) SetInputQueue(queue Enqueuer) { + tp.inputQueue = queue } -func (tp tracePoller) processJob(job PollingRequest) { - ctx := job.Context() +func (tp *tracePoller) ProcessItem(ctx context.Context, job Job) { select { default: case <-ctx.Done(): - log.Printf("[TracePoller] Context cancelled.") return } - if job.IsFirstRequest() { - err := tp.eventEmitter.Emit(ctx, events.TraceFetchingStart(job.test.ID, job.run.ID)) + if tp.isFirstRequest(job) { + err := tp.eventEmitter.Emit(ctx, events.TraceFetchingStart(job.Test.ID, job.Run.ID)) if err != nil { - log.Printf("[TracePoller] Test %s Run %d: fail to emit TracePollingStart event: %s \n", job.test.ID, job.run.ID, err.Error()) + log.Printf("[TracePoller] Test %s Run %d: fail to emit TracePollingStart event: %s", job.Test.ID, job.Run.ID, err.Error()) } } - fmt.Println("tracePoller processJob", job.count) + log.Println("TracePoller] processJob", job.EnqueueCount()) - finished, finishReason, run, err := tp.pollerExecutor.ExecuteRequest(&job) + result, err := tp.pollerExecutor.ExecuteRequest(ctx, &job) + run := result.run if err != nil { - log.Printf("[TracePoller] Test %s Run %d: ExecuteRequest Error: %s\n", job.test.ID, job.run.ID, err.Error()) - jobFailed, reason := tp.handleTraceDBError(job, err) + log.Printf("[TracePoller] Test %s Run %d: ExecuteRequest Error: %s", job.Test.ID, job.Run.ID, err.Error()) + jobFailed, reason := tp.handleTraceDBError(ctx, job, err) if jobFailed { - anotherErr := tp.eventEmitter.Emit(ctx, events.TracePollingError(job.test.ID, job.run.ID, reason, err)) + anotherErr := tp.eventEmitter.Emit(ctx, events.TracePollingError(job.Test.ID, job.Run.ID, reason, err)) if anotherErr != nil { - log.Printf("[TracePoller] Test %s Run %d: fail to emit TracePollingError event: %s \n", job.test.ID, job.run.ID, err.Error()) + log.Printf("[TracePoller] Test %s Run %d: fail to emit TracePollingError event: %s \n", job.Test.ID, job.Run.ID, err.Error()) } - anotherErr = tp.eventEmitter.Emit(ctx, events.TraceFetchingError(job.test.ID, job.run.ID, err)) + anotherErr = tp.eventEmitter.Emit(ctx, events.TraceFetchingError(job.Test.ID, job.Run.ID, err)) if anotherErr != nil { - log.Printf("[TracePoller] Test %s Run %d: fail to emit TracePollingError event: %s \n", job.test.ID, job.run.ID, err.Error()) + log.Printf("[TracePoller] Test %s Run %d: fail to emit TracePollingError event: %s \n", job.Test.ID, job.Run.ID, err.Error()) } } return } - if !finished { - job.count += 1 - tp.requeue(job) + if !result.finished { + tp.requeue(ctx, job) return } - err = tp.eventEmitter.Emit(ctx, events.TracePollingSuccess(job.test.ID, job.run.ID, finishReason)) + err = tp.eventEmitter.Emit(ctx, events.TracePollingSuccess(job.Test.ID, job.Run.ID, result.reason)) if err != nil { - log.Printf("[PollerExecutor] Test %s Run %d: failed to emit TracePollingSuccess event: error: %s\n", job.test.ID, job.run.ID, err.Error()) + log.Printf("[PollerExecutor] Test %s Run %d: failed to emit TracePollingSuccess event: error: %s\n", job.Test.ID, job.Run.ID, err.Error()) } - log.Printf("[TracePoller] Test %s Run %d: Done polling (reason: %s). Completed polling after %d iterations, number of spans collected %d\n", job.test.ID, job.run.ID, finishReason, job.count+1, len(run.Trace.Flat)) + log.Printf("[TracePoller] Test %s Run %d: Done polling (reason: %s). Completed polling after %d iterations, number of spans collected %d\n", job.Test.ID, job.Run.ID, result.reason, job.EnqueueCount()+1, len(run.Trace.Flat)) - err = tp.eventEmitter.Emit(ctx, events.TraceFetchingSuccess(job.test.ID, job.run.ID)) + err = tp.eventEmitter.Emit(ctx, events.TraceFetchingSuccess(job.Test.ID, job.Run.ID)) if err != nil { - log.Printf("[TracePoller] Test %s Run %d: fail to emit TracePollingSuccess event: %s \n", job.test.ID, job.run.ID, err.Error()) + log.Printf("[TracePoller] Test %s Run %d: fail to emit TracePollingSuccess event: %s \n", job.Test.ID, job.Run.ID, err.Error()) } tp.handleDBError(tp.updater.Update(ctx, run)) - - job.run = run - tp.runAssertions(job) + tp.outputQueue.Enqueue(ctx, job) } -func (tp tracePoller) runAssertions(job PollingRequest) { - linterRequest := LinterRequest{ - Test: job.test, - Run: job.run, - } - - tp.linterRunner.RunLinter(job.Context(), linterRequest) -} +func (tp tracePoller) handleTraceDBError(ctx context.Context, job Job, err error) (bool, string) { + run := job.Run -func (tp tracePoller) handleTraceDBError(job PollingRequest, err error) (bool, string) { - run := job.run - ctx := job.Context() - - profile := tp.ppGetter.GetDefault(ctx) - if profile.Periodic == nil { + if job.PollingProfile.Periodic == nil { log.Println("[TracePoller] cannot get polling profile.") return true, "Cannot get polling profile" } - pp := *profile.Periodic + pp := *job.PollingProfile.Periodic // Edge case: the trace still not avaiable on Data Store during polling if errors.Is(err, connection.ErrTraceNotFound) && time.Since(run.ServiceTriggeredAt) < pp.TimeoutDuration() { log.Println("[TracePoller] Trace not found on Data Store yet. Requeuing...") - job.count += 1 - tp.requeue(job) + tp.requeue(ctx, job) return false, "Trace not found" // job without fail } @@ -300,14 +216,17 @@ func (tp tracePoller) handleTraceDBError(job PollingRequest, err error) (bool, s return true, reason // job failed } -func (tp tracePoller) requeue(job PollingRequest) { +func (tp tracePoller) requeue(ctx context.Context, job Job) { go func() { - pp := tp.ppGetter.GetDefault(job.Context()) - fmt.Printf("[TracePoller] Requeuing Test Run %d. Current iteration: %d\n", job.run.ID, job.count) - time.Sleep(pp.Periodic.RetryDelayDuration()) + fmt.Printf("[TracePoller] Requeuing Test Run %d. Current iteration: %d\n", job.Run.ID, job.EnqueueCount()) + time.Sleep(job.PollingProfile.Periodic.RetryDelayDuration()) - job.SetHeaderBool("requeued", true) - job.pollingProfile = pp - tp.enqueueJob(job) + job.IncreaseEnqueueCount() + job.Headers.SetBool("requeued", true) + tp.enqueueJob(ctx, job) }() } + +func isFirstRequest(job *Job) bool { + return !job.Headers.GetBool("requeued") +} diff --git a/server/executor/transaction_pipeline.go b/server/executor/transaction_pipeline.go new file mode 100644 index 0000000000..323a39112b --- /dev/null +++ b/server/executor/transaction_pipeline.go @@ -0,0 +1,46 @@ +package executor + +import ( + "context" + + "github.com/kubeshop/tracetest/server/environment" + "github.com/kubeshop/tracetest/server/executor/testrunner" + "github.com/kubeshop/tracetest/server/test" + "github.com/kubeshop/tracetest/server/transaction" +) + +type TransactionPipeline struct { + *Pipeline + runs transactionsRunRepo +} + +type transactionsRunRepo interface { + CreateRun(context.Context, transaction.TransactionRun) (transaction.TransactionRun, error) +} + +func NewTransactionPipeline( + pipeline *Pipeline, + runs transactionsRunRepo, +) *TransactionPipeline { + return &TransactionPipeline{ + Pipeline: pipeline, + runs: runs, + } +} + +func (p *TransactionPipeline) Run(ctx context.Context, tran transaction.Transaction, metadata test.RunMetadata, environment environment.Environment, requiredGates *[]testrunner.RequiredGate) transaction.TransactionRun { + tranRun := tran.NewRun() + tranRun.Metadata = metadata + tranRun.Environment = environment + tranRun.RequiredGates = requiredGates + + tranRun, _ = p.runs.CreateRun(ctx, tranRun) + + job := NewJob() + job.Transaction = tran + job.TransactionRun = tranRun + + p.Pipeline.Begin(ctx, job) + + return tranRun +} diff --git a/server/executor/transaction_runner.go b/server/executor/transaction_runner.go index 07f66d5f6b..f595f66330 100644 --- a/server/executor/transaction_runner.go +++ b/server/executor/transaction_runner.go @@ -3,6 +3,7 @@ package executor import ( "context" "fmt" + "log" "github.com/kubeshop/tracetest/server/environment" "github.com/kubeshop/tracetest/server/executor/testrunner" @@ -12,109 +13,60 @@ import ( "github.com/kubeshop/tracetest/server/transaction" ) -type TransactionRunner interface { - Run(context.Context, transaction.Transaction, test.RunMetadata, environment.Environment, *[]testrunner.RequiredGate) transaction.TransactionRun -} - -type PersistentTransactionRunner interface { - TransactionRunner - WorkerPool -} - type transactionRunRepository interface { transactionUpdater CreateRun(context.Context, transaction.TransactionRun) (transaction.TransactionRun, error) } +type testRunner interface { + Run(context.Context, test.Test, test.RunMetadata, environment.Environment, *[]testrunner.RequiredGate) test.Run +} + func NewTransactionRunner( - runner Runner, - db test.Repository, + testRunner testRunner, transactionRuns transactionRunRepository, subscriptionManager *subscription.Manager, -) persistentTransactionRunner { +) *persistentTransactionRunner { updater := (CompositeTransactionUpdater{}). Add(NewDBTranasctionUpdater(transactionRuns)). Add(NewSubscriptionTransactionUpdater(subscriptionManager)) - return persistentTransactionRunner{ - testRunner: runner, - db: db, + return &persistentTransactionRunner{ + testRunner: testRunner, transactionRuns: transactionRuns, updater: updater, subscriptionManager: subscriptionManager, - executionChannel: make(chan transactionRunJob, 1), - exit: make(chan bool), } } -type transactionRunJob struct { - ctx context.Context - transaction transaction.Transaction - run transaction.TransactionRun -} - type persistentTransactionRunner struct { - testRunner Runner - db test.Repository + testRunner testRunner transactionRuns transactionRunRepository updater TransactionRunUpdater subscriptionManager *subscription.Manager - executionChannel chan transactionRunJob - exit chan bool } -func (r persistentTransactionRunner) Run(ctx context.Context, transaction transaction.Transaction, metadata test.RunMetadata, environment environment.Environment, requiredGates *[]testrunner.RequiredGate) transaction.TransactionRun { - run := transaction.NewRun() - run.Metadata = metadata - run.Environment = environment - run.RequiredGates = requiredGates - - ctx = getNewCtx(ctx) - - run, _ = r.transactionRuns.CreateRun(ctx, run) - - r.executionChannel <- transactionRunJob{ - ctx: ctx, - transaction: transaction, - run: run, - } - - return run -} - -func (r persistentTransactionRunner) Stop() { - r.exit <- true +func (r *persistentTransactionRunner) SetOutputQueue(_ Enqueuer) { + // this is a no-op, as transaction runner does not need to enqueue anything } -func (r persistentTransactionRunner) Start(workers int) { - for i := 0; i < workers; i++ { - go func() { - fmt.Println("PersistentTransactionRunner start goroutine") - for { - select { - case <-r.exit: - fmt.Println("PersistentTransactionRunner exit goroutine") - return - case job := <-r.executionChannel: - err := r.runTransaction(job.ctx, job.transaction, job.run) - if err != nil { - fmt.Println(err.Error()) - } - } - } - }() - } -} +func (r persistentTransactionRunner) ProcessItem(ctx context.Context, job Job) { + tran := job.Transaction + run := job.TransactionRun -func (r persistentTransactionRunner) runTransaction(ctx context.Context, tran transaction.Transaction, run transaction.TransactionRun) error { run.State = transaction.TransactionRunStateExecuting + err := r.updater.Update(ctx, run) + if err != nil { + log.Printf("[TransactionRunner] could not update transaction run: %s", err.Error()) + return + } - var err error - + log.Printf("[TransactionRunner] running transaction %s with %d steps", run.TransactionID, len(tran.Steps)) for step, test := range tran.Steps { run, err = r.runTransactionStep(ctx, run, step, test) if err != nil { - return fmt.Errorf("could not execute step %d of transaction %s: %w", step, run.TransactionID, err) + log.Printf("[TransactionRunner] could not execute step %d of transaction %s: %s", step, run.TransactionID, err.Error()) + return } if run.State == transaction.TransactionRunStateFailed { @@ -124,7 +76,8 @@ func (r persistentTransactionRunner) runTransaction(ctx context.Context, tran tr run.Environment = mergeOutputsIntoEnv(run.Environment, run.Steps[step].Outputs) err = r.transactionRuns.UpdateRun(ctx, run) if err != nil { - return fmt.Errorf("coult not update transaction step: %w", err) + log.Printf("[TransactionRunner] could not update transaction step: %s", err.Error()) + return } } @@ -132,7 +85,11 @@ func (r persistentTransactionRunner) runTransaction(ctx context.Context, tran tr run.State = transaction.TransactionRunStateFinished } - return r.updater.Update(ctx, run) + err = r.updater.Update(ctx, run) + if err != nil { + log.Printf("[TransactionRunner] could not update transaction run: %s", err.Error()) + return + } } func (r persistentTransactionRunner) runTransactionStep(ctx context.Context, tr transaction.TransactionRun, step int, testObj test.Test) (transaction.TransactionRun, error) { diff --git a/server/executor/transaction_runner_test.go b/server/executor/transaction_runner_test.go index e8576ab031..516805cc2d 100644 --- a/server/executor/transaction_runner_test.go +++ b/server/executor/transaction_runner_test.go @@ -71,7 +71,7 @@ func TestTransactionRunner(t *testing.T) { t.Run("NoErrors", func(t *testing.T) { runTransactionRunnerTest(t, false, func(t *testing.T, actual transaction.TransactionRun) { assert.Equal(t, transaction.TransactionRunStateFinished, actual.State) - assert.Len(t, actual.Steps, 2) + require.Len(t, actual.Steps, 2) assert.Equal(t, actual.Steps[0].State, test.RunStateFinished) assert.Equal(t, actual.Steps[1].State, test.RunStateFinished) assert.Equal(t, "http://my-service.com", actual.Environment.Get("url")) @@ -132,6 +132,7 @@ func runTransactionRunnerTest(t *testing.T, withErrors bool, assert func(t *test transactionsRepo := transaction.NewRepository(rawDB, testRepo) transactionRunRepo := transaction.NewRunRepository(rawDB, runRepo) tran, err := transactionsRepo.Create(ctx, transaction.Transaction{ + ID: id.ID("tran1"), Name: "transaction", StepIDs: []id.ID{test1.ID, test2.ID}, }) @@ -157,13 +158,23 @@ func runTransactionRunnerTest(t *testing.T, withErrors bool, assert func(t *test }) require.NoError(t, err) - runner := executor.NewTransactionRunner(testRunner, testRepo, transactionRunRepo, subscriptionManager) - runner.Start(1) + runner := executor.NewTransactionRunner(testRunner, transactionRunRepo, subscriptionManager) + + queueBuilder := executor.NewQueueBuilder(). + WithTransactionGetter(transactionsRepo). + WithTransactionRunGetter(transactionRunRepo) + + pipeline := executor.NewPipeline(queueBuilder, + executor.PipelineStep{Processor: runner, Driver: executor.NewInMemoryQueueDriver("runner")}, + ) + + transactionPipeline := executor.NewTransactionPipeline(pipeline, transactionRunRepo) + transactionPipeline.Start() ctxWithTimeout, cancel := context.WithTimeout(ctx, 1*time.Second) defer cancel() - transactionRun := runner.Run(ctxWithTimeout, tran, metadata, env, nil) + transactionRun := transactionPipeline.Run(ctxWithTimeout, tran, metadata, env, nil) done := make(chan transaction.TransactionRun, 1) sf := subscription.NewSubscriberFunction(func(m subscription.Message) error { @@ -176,15 +187,13 @@ func runTransactionRunnerTest(t *testing.T, withErrors bool, assert func(t *test }) subscriptionManager.Subscribe(transactionRun.ResourceID(), sf) - var finalRun transaction.TransactionRun select { - case finalRun = <-done: + case finalRun := <-done: subscriptionManager.Unsubscribe(transactionRun.ResourceID(), sf.ID()) //cleanup to avoid race conditions - fmt.Println("run completed") + assert(t, finalRun) case <-time.After(10 * time.Second): t.Log("timeout after 10 second") t.FailNow() } - - assert(t, finalRun) + transactionPipeline.Stop() } diff --git a/server/go.mod b/server/go.mod index 7c30f17cbd..b30ff4bb22 100644 --- a/server/go.mod +++ b/server/go.mod @@ -28,9 +28,9 @@ require ( github.com/goware/urlx v0.3.2 github.com/hashicorp/go-multierror v1.1.1 github.com/j2gg0s/otsql v0.14.0 + github.com/jackc/pgx/v5 v5.4.2 github.com/jhump/protoreflect v1.12.0 github.com/json-iterator/go v1.1.12 - github.com/lib/pq v1.10.5 github.com/mitchellh/mapstructure v1.5.0 github.com/ohler55/ojg v1.14.4 github.com/opensearch-project/opensearch-go v1.1.0 @@ -90,10 +90,13 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.15.6 // indirect github.com/knadh/koanf v1.4.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lib/pq v1.10.5 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect @@ -123,11 +126,11 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect - golang.org/x/crypto v0.7.0 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/crypto v0.9.0 // indirect + golang.org/x/net v0.10.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/server/go.sum b/server/go.sum index ba0d499f85..1036029792 100644 --- a/server/go.sum +++ b/server/go.sum @@ -1034,6 +1034,7 @@ github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfG github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= @@ -1045,6 +1046,8 @@ github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= @@ -1059,6 +1062,8 @@ github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXg github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA= +github.com/jackc/pgx/v5 v5.4.2 h1:u1gmGDwbdRUZiwisBm/Ky2M14uQyUP65bG8+20nnyrg= +github.com/jackc/pgx/v5 v5.4.2/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -1866,8 +1871,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1998,8 +2003,8 @@ golang.org/x/net v0.0.0-20220111093109-d55c255bac03/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2177,8 +2182,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2196,8 +2201,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/server/http/controller.go b/server/http/controller.go index b2610a88ba..71a9d7b0e9 100644 --- a/server/http/controller.go +++ b/server/http/controller.go @@ -22,27 +22,27 @@ import ( "github.com/kubeshop/tracetest/server/openapi" "github.com/kubeshop/tracetest/server/pkg/id" "github.com/kubeshop/tracetest/server/test" - "github.com/kubeshop/tracetest/server/testdb" "github.com/kubeshop/tracetest/server/tracedb" "github.com/kubeshop/tracetest/server/transaction" "go.opentelemetry.io/otel/trace" ) -var IDGen = id.NewRandGenerator() - type controller struct { - tracer trace.Tracer - runner runner - newTraceDBFn func(ds datastore.DataStore) (tracedb.TraceDB, error) - mappers mappings.Mappings - version string + tracer trace.Tracer + + testRunner testRunner + transactionRunner transactionRunner - testDB model.Repository + testRunEvents model.TestRunEventRepository + testRunRepository test.RunRepository + testRepository testsRepository transactionRepository transactionsRepository transactionRunRepository transactionRunRepository - testRepository testsRepository - testRunRepository test.RunRepository - environmentGetter environmentGetter + + environmentGetter environmentGetter + newTraceDBFn func(ds datastore.DataStore) (tracedb.TraceDB, error) + mappers mappings.Mappings + version string } type transactionsRepository interface { @@ -67,11 +67,14 @@ type transactionRunRepository interface { DeleteTransactionRun(ctx context.Context, tr transaction.TransactionRun) error } -type runner interface { - StopTest(testID id.ID, runID int) - RunTest(ctx context.Context, test test.Test, rm test.RunMetadata, env environment.Environment, gates *[]testrunner.RequiredGate) test.Run - RunTransaction(ctx context.Context, tr transaction.Transaction, rm test.RunMetadata, env environment.Environment, gates *[]testrunner.RequiredGate) transaction.TransactionRun - RunAssertions(ctx context.Context, request executor.AssertionRequest) +type testRunner interface { + StopTest(_ context.Context, testID id.ID, runID int) + Run(context.Context, test.Test, test.RunMetadata, environment.Environment, *[]testrunner.RequiredGate) test.Run + Rerun(_ context.Context, _ test.Test, runID int) test.Run +} + +type transactionRunner interface { + Run(context.Context, transaction.Transaction, test.RunMetadata, environment.Environment, *[]testrunner.RequiredGate) transaction.TransactionRun } type environmentGetter interface { @@ -79,36 +82,43 @@ type environmentGetter interface { } func NewController( - testDB model.Repository, + tracer trace.Tracer, + + testRunner testRunner, + transactionRunner transactionRunner, + + testRunEvents model.TestRunEventRepository, transactionRepository transactionsRepository, transactionRunRepository transactionRunRepository, testRepository testsRepository, testRunRepository test.RunRepository, + environmentGetter environmentGetter, + newTraceDBFn func(ds datastore.DataStore) (tracedb.TraceDB, error), - runner runner, mappers mappings.Mappings, - envGetter environmentGetter, - tracer trace.Tracer, version string, ) openapi.ApiApiServicer { return &controller{ - tracer: tracer, - testDB: testDB, + testRunEvents: testRunEvents, transactionRepository: transactionRepository, transactionRunRepository: transactionRunRepository, testRepository: testRepository, testRunRepository: testRunRepository, - environmentGetter: envGetter, - runner: runner, - newTraceDBFn: newTraceDBFn, - mappers: mappers, - version: version, + environmentGetter: environmentGetter, + + testRunner: testRunner, + transactionRunner: transactionRunner, + + tracer: tracer, + newTraceDBFn: newTraceDBFn, + mappers: mappers, + version: version, } } func handleDBError(err error) openapi.ImplResponse { switch { - case errors.Is(testdb.ErrNotFound, err) || errors.Is(sql.ErrNoRows, err): + case errors.Is(sql.ErrNoRows, err): return openapi.Response(http.StatusNotFound, err.Error()) default: return openapi.Response(http.StatusInternalServerError, err.Error()) @@ -165,7 +175,7 @@ func (c *controller) GetTestRun(ctx context.Context, testID string, runID int32) } func (c *controller) GetTestRunEvents(ctx context.Context, testID string, runID int32) (openapi.ImplResponse, error) { - events, err := c.testDB.GetTestRunEvents(ctx, id.ID(testID), int(runID)) + events, err := c.testRunEvents.GetTestRunEvents(ctx, id.ID(testID), int(runID)) if err != nil { return openapi.Response(http.StatusInternalServerError, nil), err } @@ -214,71 +224,49 @@ func (c *controller) GetTestRuns(ctx context.Context, testID string, take, skip } func (c *controller) RerunTestRun(ctx context.Context, testID string, runID int32) (openapi.ImplResponse, error) { - test, err := c.testRepository.GetAugmented(ctx, id.ID(testID)) + testObj, err := c.testRepository.GetAugmented(ctx, id.ID(testID)) if err != nil { return handleDBError(err), err } - run, err := c.testRunRepository.GetRun(ctx, id.ID(testID), int(runID)) - if err != nil { - return handleDBError(err), err - } - - newTestRun, err := c.testRunRepository.CreateRun(ctx, test, run.Copy()) - if err != nil { - return openapi.Response(http.StatusUnprocessableEntity, err.Error()), err - } - - newTestRun = newTestRun.SuccessfullyPolledTraces(run.Trace) - err = c.testRunRepository.UpdateRun(ctx, newTestRun) - if err != nil { - return openapi.Response(http.StatusInternalServerError, err.Error()), err - } - - assertionRequest := executor.AssertionRequest{ - Test: test, - Run: newTestRun, - } - - c.runner.RunAssertions(ctx, assertionRequest) + newTestRun := c.testRunner.Rerun(ctx, testObj, int(runID)) return openapi.Response(http.StatusOK, c.mappers.Out.Run(&newTestRun)), nil } -func (c *controller) RunTest(ctx context.Context, testID string, runInformation openapi.RunInformation) (openapi.ImplResponse, error) { +func (c *controller) RunTest(ctx context.Context, testID string, runInfo openapi.RunInformation) (openapi.ImplResponse, error) { test, err := c.testRepository.GetAugmented(ctx, id.ID(testID)) if err != nil { return handleDBError(err), err } - metadata := metadata(runInformation.Metadata) variablesEnv := c.mappers.In.Environment(openapi.Environment{ - Values: runInformation.Variables, + Values: runInfo.Variables, }) - environment, err := getEnvironment(ctx, c.environmentGetter, runInformation.EnvironmentId, variablesEnv) + environment, err := c.getEnvironment(ctx, runInfo.EnvironmentId, variablesEnv) if err != nil { return handleDBError(err), err } missingVariablesError, err := validation.ValidateMissingVariables(ctx, c.testRepository, c.testRunRepository, test, environment) if err != nil { - if err == validation.ErrMissingVariables { + if errors.Is(err, validation.ErrMissingVariables) { return openapi.Response(http.StatusUnprocessableEntity, missingVariablesError), nil } return handleDBError(err), err } - requiredGates := c.mappers.In.RequiredGates(runInformation.RequiredGates) + requiredGates := c.mappers.In.RequiredGates(runInfo.RequiredGates) - run := c.runner.RunTest(ctx, test, metadata, environment, requiredGates) + run := c.testRunner.Run(ctx, test, metadata(runInfo.Metadata), environment, requiredGates) return openapi.Response(200, c.mappers.Out.Run(&run)), nil } -func (c *controller) StopTestRun(_ context.Context, testID string, runID int32) (openapi.ImplResponse, error) { - c.runner.StopTest(id.ID(testID), int(runID)) +func (c *controller) StopTestRun(ctx context.Context, testID string, runID int32) (openapi.ImplResponse, error) { + c.testRunner.StopTest(ctx, id.ID(testID), int(runID)) return openapi.Response(http.StatusOK, map[string]string{"result": "success"}), nil } @@ -401,18 +389,19 @@ func metadata(in *map[string]string) test.RunMetadata { return test.RunMetadata(*in) } -func getEnvironment(ctx context.Context, environmentRepository environmentGetter, environmentId string, variablesEnv environment.Environment) (environment.Environment, error) { - if environmentId != "" { - environment, err := environmentRepository.Get(ctx, id.ID(environmentId)) +func (c *controller) getEnvironment(ctx context.Context, environmentID string, variablesEnv environment.Environment) (environment.Environment, error) { + if environmentID == "" { + return variablesEnv, nil + } - if err != nil { - return variablesEnv, err - } + environment, err := c.environmentGetter.Get(ctx, id.ID(environmentID)) - return environment.Merge(variablesEnv), nil + if err != nil { + return variablesEnv, err } - return variablesEnv, nil + return environment.Merge(variablesEnv), nil + } // Expressions @@ -517,18 +506,18 @@ func (c *controller) GetTransactionVersion(ctx context.Context, tID string, vers } // RunTransaction implements openapi.ApiApiServicer -func (c *controller) RunTransaction(ctx context.Context, transactionID string, runInformation openapi.RunInformation) (openapi.ImplResponse, error) { +func (c *controller) RunTransaction(ctx context.Context, transactionID string, runInfo openapi.RunInformation) (openapi.ImplResponse, error) { transaction, err := c.transactionRepository.GetAugmented(ctx, id.ID(transactionID)) if err != nil { return handleDBError(err), err } - metadata := metadata(runInformation.Metadata) + metadata := metadata(runInfo.Metadata) variablesEnv := c.mappers.In.Environment(openapi.Environment{ - Values: runInformation.Variables, + Values: runInfo.Variables, }) - environment, err := getEnvironment(ctx, c.environmentGetter, runInformation.EnvironmentId, variablesEnv) + environment, err := c.getEnvironment(ctx, runInfo.EnvironmentId, variablesEnv) if err != nil { return handleDBError(err), err } @@ -542,8 +531,9 @@ func (c *controller) RunTransaction(ctx context.Context, transactionID string, r return handleDBError(err), err } - requiredGates := c.mappers.In.RequiredGates(runInformation.RequiredGates) - run := c.runner.RunTransaction(ctx, transaction, metadata, environment, requiredGates) + requiredGates := c.mappers.In.RequiredGates(runInfo.RequiredGates) + + run := c.transactionRunner.Run(ctx, transaction, metadata, environment, requiredGates) return openapi.Response(http.StatusOK, c.mappers.Out.TransactionRun(run)), nil } diff --git a/server/http/controller_test.go b/server/http/controller_test.go index 9e3f028346..7d225fdda6 100644 --- a/server/http/controller_test.go +++ b/server/http/controller_test.go @@ -24,11 +24,11 @@ var ( exampleRun = test.Run{ ID: 1, TestID: id.ID("abc123"), - TraceID: http.IDGen.TraceID(), + TraceID: id.NewRandGenerator().TraceID(), Trace: &model.Trace{ - ID: http.IDGen.TraceID(), + ID: id.NewRandGenerator().TraceID(), RootSpan: model.Span{ - ID: http.IDGen.SpanID(), + ID: id.NewRandGenerator().SpanID(), Name: "POST /pokemon/import", Attributes: model.Attributes{ "tracetest.span.type": "http", @@ -123,6 +123,9 @@ func setupController(t *testing.T) controllerFixture { db: mdb, testRunRepo: runRepo, c: http.NewController( + trace.NewNoopTracerProvider().Tracer("tracer"), + nil, + nil, mdb, nil, nil, @@ -131,8 +134,6 @@ func setupController(t *testing.T) controllerFixture { nil, nil, mappings.New(traces.NewConversionConfig(), comparator.DefaultRegistry()), - nil, - trace.NewNoopTracerProvider().Tracer("tracer"), "unit-test", ), } diff --git a/server/pkg/id/generator.go b/server/pkg/id/generator.go index 05b1c105b1..4547ac0eea 100644 --- a/server/pkg/id/generator.go +++ b/server/pkg/id/generator.go @@ -37,14 +37,10 @@ type Generator interface { } func NewRandGenerator() Generator { - return randGenerator{ - rand: rand.New(rand.NewSource(time.Now().UnixNano())), - } + return randGenerator{} } -type randGenerator struct { - rand *rand.Rand -} +type randGenerator struct{} func (g randGenerator) UUID() uuid.UUID { return uuid.New() @@ -56,12 +52,13 @@ func (g randGenerator) ID() ID { func (g randGenerator) TraceID() trace.TraceID { tid := trace.TraceID{} - g.rand.Read(tid[:]) + rndSeed := rand.NewSource(time.Now().UnixNano()) + rand.New(rndSeed).Read(tid[:]) return tid } func (g randGenerator) SpanID() trace.SpanID { sid := trace.SpanID{} - g.rand.Read(sid[:]) + rand.New(rand.NewSource(time.Now().UnixNano())).Read(sid[:]) return sid } diff --git a/server/test/run_repository.go b/server/test/run_repository.go index 86e4afedb0..546b028e7b 100644 --- a/server/test/run_repository.go +++ b/server/test/run_repository.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "fmt" + "strconv" "time" "github.com/kubeshop/tracetest/server/environment" @@ -602,7 +603,7 @@ func (r *runRepository) GetTransactionRunSteps(ctx context.Context, id id.ID, ru WHERE transaction_run_steps.transaction_run_id = $1 AND transaction_run_steps.transaction_run_transaction_id = $2 ORDER BY test_runs.completed_at ASC ` - query, params := sqlutil.Tenant(ctx, query, runID, id) + query, params := sqlutil.Tenant(ctx, query, strconv.Itoa(runID), id) stmt, err := r.db.Prepare(query) if err != nil { diff --git a/server/testdb/postgres.go b/server/testdb/postgres.go index dd628f3583..8b33f2f31b 100644 --- a/server/testdb/postgres.go +++ b/server/testdb/postgres.go @@ -2,7 +2,6 @@ package testdb import ( "database/sql" - "errors" "fmt" "log" @@ -10,22 +9,12 @@ import ( "github.com/golang-migrate/migrate/v4/database/postgres" "github.com/golang-migrate/migrate/v4/source/iofs" "github.com/kubeshop/tracetest/server/migrations" - "github.com/kubeshop/tracetest/server/pkg/id" ) type postgresDB struct { db *sql.DB } -var ( - IDGen = id.NewRandGenerator() - ErrNotFound = errors.New("record not found") -) - -type scanner interface { - Scan(dest ...interface{}) error -} - func Postgres(options ...PostgresOption) (*postgresDB, error) { ps := &postgresDB{} for _, option := range options { diff --git a/server/testdb/postgres_options.go b/server/testdb/postgres_options.go index f4295c75ac..66d1970520 100644 --- a/server/testdb/postgres_options.go +++ b/server/testdb/postgres_options.go @@ -8,7 +8,8 @@ import ( "github.com/j2gg0s/otsql" "github.com/j2gg0s/otsql/hook/trace" - pq "github.com/lib/pq" + "github.com/jackc/pgx/v5" + pgxsql "github.com/jackc/pgx/v5/stdlib" "go.opentelemetry.io/otel/attribute" ) @@ -27,10 +28,11 @@ func dbSpanNameFormatter(ctx context.Context, method, query string) string { } func Connect(dsn string) (*sql.DB, error) { - connector, err := pq.NewConnector(dsn) + config, err := pgx.ParseConfig(dsn) if err != nil { return nil, fmt.Errorf("sql open: %w", err) } + connector := pgxsql.GetConnector(*config) db := sql.OpenDB( otsql.WrapConnector(connector, otsql.WithHooks( diff --git a/server/testdb/postgres_test.go b/server/testdb/postgres_test.go index bb984d324c..ccf4eb5f54 100644 --- a/server/testdb/postgres_test.go +++ b/server/testdb/postgres_test.go @@ -6,9 +6,9 @@ import ( "time" "github.com/kubeshop/tracetest/server/model" + "github.com/kubeshop/tracetest/server/pkg/id" "github.com/kubeshop/tracetest/server/test" "github.com/kubeshop/tracetest/server/test/trigger" - "github.com/kubeshop/tracetest/server/testdb" "github.com/kubeshop/tracetest/server/testmock" ) @@ -53,8 +53,8 @@ func createTest(t *testing.T, db test.Repository) test.Test { func createRun(t *testing.T, db test.RunRepository, testObj test.Test) test.Run { t.Helper() run := test.Run{ - TraceID: testdb.IDGen.TraceID(), - SpanID: testdb.IDGen.SpanID(), + TraceID: id.NewRandGenerator().TraceID(), + SpanID: id.NewRandGenerator().SpanID(), CreatedAt: time.Now(), } updated, err := db.CreateRun(context.TODO(), testObj, run) diff --git a/server/testdb/test_run_event.go b/server/testdb/test_run_event.go index 9b9aadc492..cf7b483fbe 100644 --- a/server/testdb/test_run_event.go +++ b/server/testdb/test_run_event.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "encoding/json" - "errors" "fmt" "time" @@ -136,10 +135,6 @@ func readTestRunEventFromRows(rows *sql.Rows) (model.TestRunEvent, error) { ) if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return model.TestRunEvent{}, ErrNotFound - } - return model.TestRunEvent{}, fmt.Errorf("could not scan event: %w", err) } diff --git a/server/tracedb/otlp.go b/server/tracedb/otlp.go index 97d5fd370a..e573a814ad 100644 --- a/server/tracedb/otlp.go +++ b/server/tracedb/otlp.go @@ -8,14 +8,19 @@ import ( "github.com/kubeshop/tracetest/server/test" "github.com/kubeshop/tracetest/server/tracedb/connection" "github.com/kubeshop/tracetest/server/traces" + "go.opentelemetry.io/otel/trace" ) +type runByTraceIDGetter interface { + GetRunByTraceID(context.Context, trace.TraceID) (test.Run, error) +} + type OTLPTraceDB struct { realTraceDB - db test.RunRepository + db runByTraceIDGetter } -func newCollectorDB(repository test.RunRepository) (TraceDB, error) { +func newCollectorDB(repository runByTraceIDGetter) (TraceDB, error) { return &OTLPTraceDB{ db: repository, }, nil diff --git a/server/tracedb/tracedb.go b/server/tracedb/tracedb.go index f8891a6957..a125785247 100644 --- a/server/tracedb/tracedb.go +++ b/server/tracedb/tracedb.go @@ -7,7 +7,6 @@ import ( "github.com/kubeshop/tracetest/server/datastore" "github.com/kubeshop/tracetest/server/model" "github.com/kubeshop/tracetest/server/pkg/id" - "github.com/kubeshop/tracetest/server/test" "go.opentelemetry.io/otel/trace" ) @@ -47,10 +46,10 @@ func (db *noopTraceDB) TestConnection(ctx context.Context) model.ConnectionResul } type traceDBFactory struct { - runRepository test.RunRepository + runRepository runByTraceIDGetter } -func Factory(runRepository test.RunRepository) func(ds datastore.DataStore) (TraceDB, error) { +func Factory(runRepository runByTraceIDGetter) func(ds datastore.DataStore) (TraceDB, error) { f := traceDBFactory{ runRepository: runRepository, } diff --git a/server/transaction/transaction_repository.go b/server/transaction/transaction_repository.go index ed6b8a18a9..0f94abf4b0 100644 --- a/server/transaction/transaction_repository.go +++ b/server/transaction/transaction_repository.go @@ -27,10 +27,6 @@ type transactionStepRepository interface { GetTransactionSteps(_ context.Context, _ id.ID, version int) ([]test.Test, error) } -type transactionStepRunRepository interface { - GetTransactionRunSteps(_ context.Context, _ id.ID, runID int) ([]test.Run, error) -} - type Repository struct { db *sql.DB stepRepository transactionStepRepository diff --git a/server/transaction/transaction_run_repository.go b/server/transaction/transaction_run_repository.go index 5e3b702c2b..56b7e6caa3 100644 --- a/server/transaction/transaction_run_repository.go +++ b/server/transaction/transaction_run_repository.go @@ -7,12 +7,18 @@ import ( "encoding/hex" "encoding/json" "fmt" + "strconv" "strings" "github.com/kubeshop/tracetest/server/pkg/id" "github.com/kubeshop/tracetest/server/pkg/sqlutil" + "github.com/kubeshop/tracetest/server/test" ) +type transactionStepRunRepository interface { + GetTransactionRunSteps(_ context.Context, _ id.ID, runID int) ([]test.Run, error) +} + func NewRunRepository(db *sql.DB, stepsRepository transactionStepRunRepository) *RunRepository { return &RunRepository{ db: db, @@ -208,7 +214,7 @@ func (td *RunRepository) UpdateRun(ctx context.Context, tr TransactionRun) error allStepsRequiredGatesPassed, jsonMetadata, jsonEnvironment, - tr.ID, + strconv.Itoa(tr.ID), tr.TransactionID, ) stmt, err := tx.Prepare(query) @@ -229,7 +235,7 @@ func (td *RunRepository) UpdateRun(ctx context.Context, tr TransactionRun) error func (td *RunRepository) setTransactionRunSteps(ctx context.Context, tx *sql.Tx, tr TransactionRun) error { // delete existing steps - query, params := sqlutil.Tenant(ctx, "DELETE FROM transaction_run_steps WHERE transaction_run_id = $1 AND transaction_run_transaction_id = $2", tr.ID, tr.TransactionID) + query, params := sqlutil.Tenant(ctx, "DELETE FROM transaction_run_steps WHERE transaction_run_id = $1 AND transaction_run_transaction_id = $2", strconv.Itoa(tr.ID), tr.TransactionID) stmt, err := tx.Prepare(query) if err != nil { return err @@ -237,7 +243,7 @@ func (td *RunRepository) setTransactionRunSteps(ctx context.Context, tx *sql.Tx, _, err = stmt.ExecContext(ctx, params...) if err != nil { - return err + return fmt.Errorf("cannot reset transaction run steps: %w", err) } if len(tr.Steps) == 0 { @@ -321,7 +327,7 @@ FROM transaction_runs ` func (td *RunRepository) GetTransactionRun(ctx context.Context, transactionID id.ID, runID int) (TransactionRun, error) { - query, params := sqlutil.Tenant(ctx, selectTransactionRunQuery+" WHERE id = $1 AND transaction_id = $2", runID, transactionID) + query, params := sqlutil.Tenant(ctx, selectTransactionRunQuery+" WHERE id = $1 AND transaction_id = $2", strconv.Itoa(runID), transactionID) stmt, err := td.db.Prepare(query) if err != nil { return TransactionRun{}, fmt.Errorf("prepare: %w", err)