Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: dashboard uri for cd event #3879

Merged
merged 8 commits into from
May 24, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/api-server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,7 @@ func main() {
cfg.GraphqlPort,
artifactStorage,
cfg.CDEventsTarget,
cfg.TestkubeDashboardURI,
)

if mode == common.ModeAgent {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func TestRun_Integration(t *testing.T) {
ctx := context.Background()

params := envs.Params{DataDir: tempDir}
runner, err := NewPlaywrightRunner(ctx, "pnpm", params)
runner, err := NewPlaywrightRunner(ctx, "npm", params)
if err != nil {
t.Fail()
}
Expand Down
2 changes: 2 additions & 0 deletions docs/docs/articles/helm-chart.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ The following Helm defaults are used in the `testkube` chart:
| testkube-api.jobServiceAccountName | yes | "" |
| testkube-api.logs.storage | no | "minio" |
| testkube-api.logs.bucket | no | "testkube-logs" |
| testkube-api.cdeventsTarget | yes | "" |
| testkube-api.dashboardUri | yes | "" |

>For more configuration parameters of a `MongoDB` chart please visit:
<https://github.com/bitnami/charts/tree/master/bitnami/mongodb#parameters>
Expand Down
3 changes: 2 additions & 1 deletion internal/app/api/v1/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ func NewTestkubeAPI(
graphqlPort string,
artifactsStorage storage.ArtifactsStorage,
cdeventsTarget string,
dashboardURI string,
) TestkubeAPI {

var httpConfig server.Config
Expand Down Expand Up @@ -129,7 +130,7 @@ func NewTestkubeAPI(
s.Events.Loader.Register(s.slackLoader)

if cdeventsTarget != "" {
cdeventLoader, err := cdevent.NewCDEventLoader(cdeventsTarget, clusterId, namespace, testkube.AllEventTypes)
cdeventLoader, err := cdevent.NewCDEventLoader(cdeventsTarget, clusterId, namespace, dashboardURI, testkube.AllEventTypes)
if err == nil {
s.Events.Loader.Register(cdeventLoader)
} else {
Expand Down
1 change: 1 addition & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ type Config struct {
TestkubeRegistry string `envconfig:"TESTKUBE_REGISTRY" default:""`
TestkubePodStartTimeout time.Duration `envconfig:"TESTKUBE_POD_START_TIMEOUT" default:"30m"`
CDEventsTarget string `envconfig:"CDEVENTS_TARGET" default:""`
TestkubeDashboardURI string `envconfig:"TESTKUBE_DASHBOARD_URI" default:""`
}

func Get() (*Config, error) {
Expand Down
4 changes: 2 additions & 2 deletions pkg/api/v1/testkube/model_artifact_request.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ package testkube
// artifact request body with test artifacts
type ArtifactRequest struct {
// artifact storage class name for container executor
StorageClassName string `json:"storageClassName"`
StorageClassName string `json:"storageClassName,omitempty"`
// artifact volume mount path for container executor
VolumeMountPath string `json:"volumeMountPath"`
VolumeMountPath string `json:"volumeMountPath,omitempty"`
// artifact directories for scraping
Dirs []string `json:"dirs,omitempty"`
}
10 changes: 5 additions & 5 deletions pkg/api/v1/testkube/model_artifact_update_request.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ package testkube

// artifact request update body
type ArtifactUpdateRequest struct {
// artifact storage class name
StorageClassName *string `json:"storageClassName"`
// artifact volume mount path
VolumeMountPath *string `json:"volumeMountPath"`
// artifact directories
// artifact storage class name for container executor
StorageClassName *string `json:"storageClassName,omitempty"`
// artifact volume mount path for container executor
VolumeMountPath *string `json:"volumeMountPath,omitempty"`
// artifact directories for scraping
Dirs *[]string `json:"dirs,omitempty"`
}
4 changes: 2 additions & 2 deletions pkg/cloud/data/artifact/scraper_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func TestCloudScraper_ArchiveFilesystemExtractor_Integration(t *testing.T) {
client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(svr.URL))
assert.NoError(t, err)

s := scraper.NewExtractLoadScraper(extractor, cloudLoader, client, "")
s := scraper.NewExtractLoadScraper(extractor, cloudLoader, client, "", "")
execution := testkube.Execution{
Id: "my-execution-id",
TestName: "my-test",
Expand Down Expand Up @@ -195,7 +195,7 @@ func TestCloudScraper_RecursiveFilesystemExtractor_Integration(t *testing.T) {
client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(svr.URL))
assert.NoError(t, err)

s := scraper.NewExtractLoadScraper(extractor, cloudLoader, client, "")
s := scraper.NewExtractLoadScraper(extractor, cloudLoader, client, "", "")
execution := testkube.Execution{
Id: "my-execution-id",
TestName: "my-test",
Expand Down
2 changes: 2 additions & 0 deletions pkg/envs/variables.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type Params struct {
GitToken string // RUNNER_GITTOKEN
ClusterID string `envconfig:"RUNNER_CLUSTERID"` // RUNNER_CLUSTERID
CDEventsTarget string `envconfig:"RUNNER_CDEVENTS_TARGET"` // RUNNER_CDEVENTS_TARGET
DashboardURI string `envconfig:"RUNNER_DASHBOARD_URI"` // RUNNER_DASHBOARD_URI
CloudMode bool `envconfig:"RUNNER_CLOUD_MODE"` // RUNNER_CLOUD_MODE
CloudAPIKey string `envconfig:"RUNNER_CLOUD_API_KEY"` // RUNNER_CLOUD_API_KEY
CloudAPITLSInsecure bool `envconfig:"RUNNER_CLOUD_API_TLS_INSECURE"` // RUNNER_CLOUD_API_TLS_INSECURE
Expand Down Expand Up @@ -60,6 +61,7 @@ func printParams(params Params) {
output.PrintLogf("RUNNER_DATADIR=\"%s\"", params.DataDir)
output.PrintLogf("RUNNER_CLUSTERID=\"%s\"", params.ClusterID)
output.PrintLogf("RUNNER_CDEVENTS_TARGET=\"%s\"", params.CDEventsTarget)
output.PrintLogf("RUNNER_DASHBOARD_URI=\"%s\"", params.DashboardURI)
output.PrintLogf("RUNNER_CLOUD_MODE=\"%t\"", params.CloudMode)
output.PrintLogf("RUNNER_CLOUD_API_TLS_INSECURE=\"%t\"", params.CloudAPITLSInsecure)
output.PrintLogf("RUNNER_CLOUD_API_URL=\"%s\"", params.CloudAPIURL)
Expand Down
6 changes: 4 additions & 2 deletions pkg/event/kind/cdevent/listener.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (

var _ common.Listener = (*CDEventListener)(nil)

func NewCDEventListener(name, selector, clusterID, defaultNamespace string, events []testkube.EventType, client cloudevents.Client) *CDEventListener {
func NewCDEventListener(name, selector, clusterID, defaultNamespace, dashboardURI string, events []testkube.EventType, client cloudevents.Client) *CDEventListener {
return &CDEventListener{
name: name,
Log: log.DefaultLogger,
Expand All @@ -26,6 +26,7 @@ func NewCDEventListener(name, selector, clusterID, defaultNamespace string, even
client: client,
clusterID: clusterID,
defaultNamespace: defaultNamespace,
dashboardURI: dashboardURI,
}
}

Expand All @@ -37,6 +38,7 @@ type CDEventListener struct {
client cloudevents.Client
clusterID string
defaultNamespace string
dashboardURI string
}

func (l *CDEventListener) Name() string {
Expand All @@ -60,7 +62,7 @@ func (l *CDEventListener) Metadata() map[string]string {

func (l *CDEventListener) Notify(event testkube.Event) (result testkube.EventResult) {
// Create the base event
ev, err := cde.MapTestkubeEventToCDEvent(event, l.clusterID, l.defaultNamespace)
ev, err := cde.MapTestkubeEventToCDEvent(event, l.clusterID, l.defaultNamespace, l.dashboardURI)
if err != nil {
return testkube.NewFailedEventResult(event.Id, err)
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/event/kind/cdevent/listener_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func TestCDEventListener_Notify(t *testing.T) {
client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(svr.URL))
assert.NoError(t, err)

l := NewCDEventListener("cdeli", "", "clusterID", "", testEventTypes, client)
l := NewCDEventListener("cdeli", "", "clusterID", "", "", testEventTypes, client)

// when
r := l.Notify(testkube.Event{
Expand All @@ -57,7 +57,7 @@ func TestCDEventListener_Notify(t *testing.T) {
client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(svr.URL))
assert.NoError(t, err)

l := NewCDEventListener("cdeli", "", "clusterID", "", testEventTypes, client)
l := NewCDEventListener("cdeli", "", "clusterID", "", "", testEventTypes, client)

// when
r := l.Notify(testkube.Event{
Expand All @@ -77,7 +77,7 @@ func TestCDEventListener_Notify(t *testing.T) {
client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget("abcdef"))
assert.NoError(t, err)

l := NewCDEventListener("cdeli", "", "clusterID", "", testEventTypes, client)
l := NewCDEventListener("cdeli", "", "clusterID", "", "", testEventTypes, client)

// when
r := l.Notify(testkube.Event{
Expand Down
6 changes: 4 additions & 2 deletions pkg/event/kind/cdevent/loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (

var _ common.ListenerLoader = (*CDEventLoader)(nil)

func NewCDEventLoader(target, clusterID, defaultNamespace string, events []testkube.EventType) (*CDEventLoader, error) {
func NewCDEventLoader(target, clusterID, defaultNamespace, dashboardURI string, events []testkube.EventType) (*CDEventLoader, error) {
c, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(target))
if err != nil {
return nil, err
Expand All @@ -23,6 +23,7 @@ func NewCDEventLoader(target, clusterID, defaultNamespace string, events []testk
client: c,
clusterID: clusterID,
defaultNamespace: defaultNamespace,
dashboardURI: dashboardURI,
}, nil
}

Expand All @@ -33,6 +34,7 @@ type CDEventLoader struct {
client cloudevents.Client
clusterID string
defaultNamespace string
dashboardURI string
}

func (r *CDEventLoader) Kind() string {
Expand All @@ -41,5 +43,5 @@ func (r *CDEventLoader) Kind() string {

// Load returns single listener for cd eventt
func (r *CDEventLoader) Load() (listeners common.Listeners, err error) {
return common.Listeners{NewCDEventListener("cdevent", "", r.clusterID, r.defaultNamespace, r.events, r.client)}, nil
return common.Listeners{NewCDEventListener("cdevent", "", r.clusterID, r.defaultNamespace, r.dashboardURI, r.events, r.client)}, nil
}
2 changes: 1 addition & 1 deletion pkg/event/kind/cdevent/loader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
func TestWebhookLoader(t *testing.T) {
t.Parallel()

cdeventLoader, err := NewCDEventLoader("target", "", "", nil)
cdeventLoader, err := NewCDEventLoader("target", "", "", "", nil)
assert.NoError(t, err)

listeners, err := cdeventLoader.Load()
Expand Down
4 changes: 4 additions & 0 deletions pkg/executor/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ var RunnerEnvVars = []corev1.EnvVar{
Name: "RUNNER_CLOUD_API_URL",
Value: os.Getenv("TESTKUBE_CLOUD_URL"),
},
{
Name: "RUNNER_DASHBOARD_URI",
Value: os.Getenv("TESTKUBE_DASHBOARD_URI"),
},
}

func getOr(key, defaultVal string) string {
Expand Down
1 change: 1 addition & 0 deletions pkg/executor/containerexecutor/containerexecutor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ func TestNewExecutorJobSpecWithArgs(t *testing.T) {
{Name: "RUNNER_SCRAPPERENABLED", Value: "false"},
{Name: "RUNNER_DATADIR", Value: "/data"},
{Name: "RUNNER_CDEVENTS_TARGET", Value: ""},
{Name: "RUNNER_DASHBOARD_URI", Value: ""},
{Name: "RUNNER_CLOUD_MODE", Value: "false"},
{Name: "RUNNER_CLOUD_API_KEY", Value: ""},
{Name: "RUNNER_CLOUD_API_URL", Value: ""},
Expand Down
8 changes: 8 additions & 0 deletions pkg/executor/output/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ func TestParseRunnerOutput(t *testing.T) {
{"type":"line","content":"RUNNER_GITTOKEN=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-01-17T15:29:17.921803138Z"}
{"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"line","content":"RUNNER_CLUSTERID=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"error","content":"❌ can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-17T15:29:17.921940304Z"}
{"type":"error","content":"can't find branch or commit in params, repo:\u0026{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:\u003cnil\u003e TokenSecret:\u003cnil\u003e WorkingDir:}","time":"2023-01-17T15:29:17.921946638Z"}
Expand All @@ -146,6 +147,7 @@ RUNNER_GITUSERNAME=""
RUNNER_GITTOKEN=""
RUNNER_DATADIR="/data"
RUNNER_CDEVENTS_TARGET=""
RUNNER_DASHBOARD_URI=""
RUNNER_CLUSTERID=""
❌ can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:}
can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.com/kubeshop/testkube.git Branch: Commit: Path:test/cypress/executor-smoke/cypress-11 Username: Token: UsernameSecret:<nil> TokenSecret:<nil> WorkingDir:}
Expand Down Expand Up @@ -196,6 +198,7 @@ could not start process: fork/exec ./zap-api-scan.py: no such file or directory
{"type":"line","content":"RUNNER_GITTOKEN=\"\"","time":"2023-01-19T15:22:25.867986013Z"}
{"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-01-19T15:22:25.867987596Z"}
{"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"line","content":"RUNNER_CLUSTERID=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"event","content":"running test [63c960287104b0fa0b7a45ef]","time":"2023-01-19T15:22:25.868132888Z"}
{"type":"line","content":"🚚 Preparing for test run","time":"2023-01-19T15:22:25.868161346Z"}
Expand All @@ -216,6 +219,7 @@ RUNNER_GITUSERNAME=""
RUNNER_GITTOKEN=""
RUNNER_DATADIR="/data"
RUNNER_CDEVENTS_TARGET=""
RUNNER_DASHBOARD_URI=""
RUNNER_CLUSTERID=""
running test [63c960287104b0fa0b7a45ef]
🚚 Preparing for test run
Expand Down Expand Up @@ -249,6 +253,7 @@ can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.
{"type":"line","content":"RUNNER_GITTOKEN=\"\"","time":"2023-01-19T15:22:25.867986013Z"}
{"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-01-19T15:22:25.867987596Z"}
{"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"line","content":"RUNNER_CLUSTERID=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"event","content":"running test [63c960287104b0fa0b7a45ef]","time":"2023-01-19T15:22:25.868132888Z"}
{"type":"line","content":"🚚 Preparing for test run","time":"2023-01-19T15:22:25.868161346Z"}
Expand All @@ -275,6 +280,7 @@ RUNNER_GITUSERNAME=""
RUNNER_GITTOKEN=""
RUNNER_DATADIR="/data"
RUNNER_CDEVENTS_TARGET=""
RUNNER_DASHBOARD_URI=""
RUNNER_CLUSTERID=""
running test [63c960287104b0fa0b7a45ef]
🚚 Preparing for test run
Expand Down Expand Up @@ -309,6 +315,7 @@ can't find branch or commit in params, repo:&{Type_:git-file Uri:https://github.
{"type":"line","content":"RUNNER_GITTOKEN=\"\"","time":"2023-01-20T12:44:15.718986174Z"}
{"type":"line","content":"RUNNER_DATADIR=\"/data\"","time":"2023-01-20T12:44:15.718989049Z"}
{"type":"line","content":"RUNNER_CDEVENTS_TARGET=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"line","content":"RUNNER_DASHBOARD_URI=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"line","content":"RUNNER_CLUSTERID=\"\"","time":"2023-01-17T15:29:17.921801596Z"}
{"type":"event","content":"running test [63ca8c8988564860327a16b5]","time":"2023-01-20T12:44:15.719276383Z"}
{"type":"line","content":"🚚 Preparing for test run","time":"2023-01-20T12:44:15.719285633Z"}
Expand All @@ -329,6 +336,7 @@ RUNNER_GITUSERNAME=""
RUNNER_GITTOKEN=""
RUNNER_DATADIR="/data"
RUNNER_CDEVENTS_TARGET=""
RUNNER_DASHBOARD_URI=""
RUNNER_CLUSTERID=""
running test [63ca8c8988564860327a16b5]
🚚 Preparing for test run
Expand Down
2 changes: 1 addition & 1 deletion pkg/executor/scraper/factory/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func GetScraper(ctx context.Context, params envs.Params, extractorType Extractor
}
}

return scraper.NewExtractLoadScraper(extractor, loader, cdeventsClient, params.ClusterID), nil
return scraper.NewExtractLoadScraper(extractor, loader, cdeventsClient, params.ClusterID, params.DashboardURI), nil
}

func getCloudLoader(ctx context.Context, params envs.Params) (uploader *cloudscraper.CloudUploader, err error) {
Expand Down
4 changes: 2 additions & 2 deletions pkg/executor/scraper/minio_scraper_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func TestMinIOScraper_Archive_Integration(t *testing.T) {
client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(svr.URL))
assert.NoError(t, err)

s := scraper.NewExtractLoadScraper(extractor, loader, client, "")
s := scraper.NewExtractLoadScraper(extractor, loader, client, "", "")
err = s.Scrape(context.Background(), []string{tempDir}, execution)
if err != nil {
t.Fatalf("error scraping: %v", err)
Expand Down Expand Up @@ -139,7 +139,7 @@ func TestMinIOScraper_Recursive_Integration(t *testing.T) {
client, err := cloudevents.NewClientHTTP(cloudevents.WithTarget(svr.URL))
assert.NoError(t, err)

s := scraper.NewExtractLoadScraper(extractor, loader, client, "")
s := scraper.NewExtractLoadScraper(extractor, loader, client, "", "")
err = s.Scrape(context.Background(), []string{tempDir}, execution)
if err != nil {
t.Fatalf("error scraping: %v", err)
Expand Down
7 changes: 5 additions & 2 deletions pkg/executor/scraper/scraper.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,17 @@ type ExtractLoadScraper struct {
loader Uploader
cdeventsClient cloudevents.Client
clusterID string
dashboardURI string
}

func NewExtractLoadScraper(extractor Extractor, loader Uploader, cdeventsClient cloudevents.Client, clusterID string) *ExtractLoadScraper {
func NewExtractLoadScraper(extractor Extractor, loader Uploader, cdeventsClient cloudevents.Client,
clusterID, dashboardURI string) *ExtractLoadScraper {
return &ExtractLoadScraper{
extractor: extractor,
loader: loader,
cdeventsClient: cdeventsClient,
clusterID: clusterID,
dashboardURI: dashboardURI,
}
}

Expand Down Expand Up @@ -66,7 +69,7 @@ func (s *ExtractLoadScraper) sendCDEvent(execution testkube.Execution, path stri
log.DefaultLogger.Warnf("failed to detect mime type %w", err)
}

ev, err := cde.MapTestkubeArtifactToCDEvent(&execution, s.clusterID, mtype.String())
ev, err := cde.MapTestkubeArtifactToCDEvent(&execution, s.clusterID, path, mtype.String(), s.dashboardURI)
if err != nil {
return err
}
Expand Down