From 6cde78218bd89a3b8284700632326b361dbc2961 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Thu, 18 Jul 2024 17:40:06 +0000 Subject: [PATCH 1/4] Added more business milestones to track where failures happen Fixes #470 --- cmd/tea_ensuretoken.go | 19 +++++++++++++++++++ cmd/tea_initialisesources.go | 13 +++++++++++++ cmd/tea_plan.go | 20 ++++++++++++++++++++ cmd/tea_revlink.go | 16 ++++++++++++++++ cmd/tea_submitplan.go | 22 +++++++++++++++++++++- 5 files changed, 89 insertions(+), 1 deletion(-) diff --git a/cmd/tea_ensuretoken.go b/cmd/tea_ensuretoken.go index d6f5cbfd..c5adfed9 100644 --- a/cmd/tea_ensuretoken.go +++ b/cmd/tea_ensuretoken.go @@ -20,6 +20,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/viper" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "golang.org/x/oauth2" ) @@ -126,16 +127,34 @@ func (m ensureTokenModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.title = "Using stored token" m.deviceMessage = "" cmds = append(cmds, m.tokenAvailable(msg.token)) + + if cmdSpan != nil { + cmdSpan.AddEvent("User Authenticated", trace.WithAttributes( + attribute.String("ovm.auth.mechanism", "Token"), + )) + } case tokenReceivedMsg: m.status = taskStatusDone m.title = "Authentication successful, using API key" m.deviceMessage = "" cmds = append(cmds, m.tokenAvailable(msg.token)) + + if cmdSpan != nil { + cmdSpan.AddEvent("User Authenticated", trace.WithAttributes( + attribute.String("ovm.auth.mechanism", "API Key"), + )) + } case tokenStoredMsg: m.status = taskStatusDone m.title = fmt.Sprintf("Authentication successful, token stored locally (%v)", msg.file) m.deviceMessage = "" cmds = append(cmds, m.tokenAvailable(msg.token)) + + if cmdSpan != nil { + cmdSpan.AddEvent("User Authenticated", trace.WithAttributes( + attribute.String("ovm.auth.mechanism", "Browser"), + )) + } case otherError: if msg.id == m.spinner.ID() { m.errors = append(m.errors, fmt.Sprintf("Note: %v", msg.err)) diff --git a/cmd/tea_initialisesources.go b/cmd/tea_initialisesources.go index 485e3d4f..3accf6cd 100644 --- a/cmd/tea_initialisesources.go +++ b/cmd/tea_initialisesources.go @@ -156,6 +156,10 @@ func (m initialiseSourcesModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { ) cmds = append(cmds, input.Focus()) } else { + if cmdSpan != nil { + cmdSpan.AddEvent("Used stored AWS config") + } + cmds = append(cmds, m.storeConfigCmd(aws_config, aws_profile)) cmds = append(cmds, m.startSourcesCmd(aws_config, aws_profile)) } @@ -166,6 +170,10 @@ func (m initialiseSourcesModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.awsSourceRunning = true m.stdlibSourceRunning = true m.status = taskStatusDone + + if cmdSpan != nil { + cmdSpan.AddEvent("Sources initialised") + } case sourceInitialisationFailedMsg: m.status = taskStatusError errorHint := "Error initialising sources" @@ -259,6 +267,11 @@ func (m initialiseSourcesModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } case huh.StateCompleted: m.profileInputFormDone = true + + if cmdSpan != nil { + cmdSpan.AddEvent("User provided AWS config") + } + // store the result viper.Set("aws-profile", m.profileInputForm.GetString("aws-profile")) cmds = append(cmds, m.storeConfigCmd(viper.GetString("aws-config"), viper.GetString("aws-profile"))) diff --git a/cmd/tea_plan.go b/cmd/tea_plan.go index 2704a092..bbc059d5 100644 --- a/cmd/tea_plan.go +++ b/cmd/tea_plan.go @@ -10,6 +10,9 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/overmindtech/cli/tracing" "github.com/spf13/viper" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) type runPlanModel struct { @@ -110,10 +113,27 @@ func (m runPlanModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { })) case runPlanFinishedMsg: + var attrs []attribute.KeyValue + var eventName string + if msg.err != nil { m.taskModel.status = taskStatusError + + // Tracing info + attrs = append(attrs, attribute.String("error", msg.err.Error())) + eventName = "Terraform plan failed" } else { m.taskModel.status = taskStatusDone + + // Tracing info + eventName = "Terraform plan finished" + } + + if cmdSpan != nil { + cmdSpan.AddEvent(eventName, trace.WithAttributes(attrs...)) + if msg.err != nil { + cmdSpan.SetStatus(codes.Error, msg.err.Error()) + } } } diff --git a/cmd/tea_revlink.go b/cmd/tea_revlink.go index 02f401ab..45779f39 100644 --- a/cmd/tea_revlink.go +++ b/cmd/tea_revlink.go @@ -11,6 +11,8 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/overmindtech/sdp-go" "github.com/spf13/viper" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) type revlinkWarmupFinishedMsg struct{} @@ -110,6 +112,20 @@ func (m revlinkWarmupModel) Update(msg tea.Msg) (revlinkWarmupModel, tea.Cmd) { m.watchdogCancel() m.watchdogCancel = nil } + + if cmdSpan != nil { + var attrs []attribute.KeyValue + + if m.currentStatus != nil { + attrs = append(attrs, + attribute.String("ovm.cli.revlinkWarmupStatus", m.currentStatus.GetStatus()), + attribute.Int("ovm.cli.revlinkWarmupItems", int(m.currentStatus.Items)), + attribute.Int("ovm.cli.revlinkWarmupEdges", int(m.currentStatus.Edges)), + ) + } + + cmdSpan.AddEvent("Revlink warmup finished", trace.WithAttributes(attrs...)) + } default: var taskCmd tea.Cmd m.taskModel, taskCmd = m.taskModel.Update(msg) diff --git a/cmd/tea_submitplan.go b/cmd/tea_submitplan.go index c448a7e3..aa80d9ab 100644 --- a/cmd/tea_submitplan.go +++ b/cmd/tea_submitplan.go @@ -723,6 +723,17 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { } m.processing <- submitPlanUpdateMsg{m.blastRadiusTask.FinishMsg()} + // Add tracing that the blast radius has finished + if cmdSpan != nil { + cmdSpan.AddEvent("Blast radius calculation finished", trace.WithAttributes( + attribute.Int("ovm.blast_radius.items", int(msg.GetNumItems())), + attribute.Int("ovm.blast_radius.edges", int(msg.GetNumEdges())), + attribute.String("ovm.blast_radius.state", msg.GetState().String()), + attribute.StringSlice("ovm.blast_radius.errors", msg.GetErrors()), + attribute.String("ovm.change.uuid", changeUuid.String()), + )) + } + changeUrl := *m.oi.FrontendUrl changeUrl.Path = fmt.Sprintf("%v/changes/%v/blast-radius", changeUrl.Path, changeUuid) log.WithField("change-url", changeUrl.String()).Info("Change ready") @@ -736,8 +747,9 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { /////////////////////////////////////////////////////////////////// m.processing <- submitPlanUpdateMsg{m.riskTask.UpdateStatusMsg(taskStatusRunning)} risksErrored := false + var riskRes *connect.Response[sdp.GetChangeRisksResponse] for { - riskRes, err := client.GetChangeRisks(ctx, &connect.Request[sdp.GetChangeRisksRequest]{ + riskRes, err = client.GetChangeRisks(ctx, &connect.Request[sdp.GetChangeRisksRequest]{ Msg: &sdp.GetChangeRisksRequest{ UUID: changeUuid[:], }, @@ -783,6 +795,14 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { return nil } + // Submit milestone for tracing + if cmdSpan != nil { + cmdSpan.AddEvent("Risk calculation finished", trace.WithAttributes( + attribute.Int("ovm.risks.count", len(riskRes.Msg.GetChangeRiskMetadata().GetRisks())), + attribute.String("ovm.change.uuid", changeUuid.String()), + )) + } + m.processing <- submitPlanUpdateMsg{m.riskTask.UpdateStatusMsg(taskStatusDone)} m.processing <- submitPlanUpdateMsg{submitPlanFinishedMsg{"Done"}} From cc4dd21767ff1bd0abb66c68894ab1de378b2ba0 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Fri, 19 Jul 2024 12:58:47 +0000 Subject: [PATCH 2/4] Added sentry capture Fixes #473 --- cmd/tea_submitplan.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/tea_submitplan.go b/cmd/tea_submitplan.go index aa80d9ab..5c0bea7b 100644 --- a/cmd/tea_submitplan.go +++ b/cmd/tea_submitplan.go @@ -13,6 +13,7 @@ import ( "connectrpc.com/connect" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + "github.com/getsentry/sentry-go" "github.com/google/uuid" "github.com/muesli/reflow/wordwrap" "github.com/overmindtech/sdp-go" @@ -680,6 +681,9 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { m.processing <- submitPlanUpdateMsg{m.blastRadiusTask.UpdateStatusMsg(taskStatusError)} m.processing <- submitPlanUpdateMsg{m.risksError("failed to update planned changes", err)} close(m.processing) + + // TODO: Ask David about the position of these too + sentry.CaptureException(err) return nil } @@ -719,6 +723,9 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { m.processing <- submitPlanUpdateMsg{m.blastRadiusTask.UpdateStatusMsg(taskStatusError)} m.processing <- submitPlanUpdateMsg{m.risksError("error streaming results", err)} close(m.processing) + + // TODO: Ask David about the position of these too + sentry.CaptureException(err) return nil } m.processing <- submitPlanUpdateMsg{m.blastRadiusTask.FinishMsg()} From cf3f2530397adfa3bdc2db5aea388ce37235a11a Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Fri, 19 Jul 2024 13:37:37 +0000 Subject: [PATCH 3/4] Fixed linting --- cmd/tea_revlink.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/tea_revlink.go b/cmd/tea_revlink.go index 45779f39..7abdd877 100644 --- a/cmd/tea_revlink.go +++ b/cmd/tea_revlink.go @@ -119,8 +119,8 @@ func (m revlinkWarmupModel) Update(msg tea.Msg) (revlinkWarmupModel, tea.Cmd) { if m.currentStatus != nil { attrs = append(attrs, attribute.String("ovm.cli.revlinkWarmupStatus", m.currentStatus.GetStatus()), - attribute.Int("ovm.cli.revlinkWarmupItems", int(m.currentStatus.Items)), - attribute.Int("ovm.cli.revlinkWarmupEdges", int(m.currentStatus.Edges)), + attribute.Int("ovm.cli.revlinkWarmupItems", int(m.currentStatus.GetItems())), + attribute.Int("ovm.cli.revlinkWarmupEdges", int(m.currentStatus.GetEdges())), ) } From f3428d76be4127aa73daad3b4445a3ccddbda077 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Fri, 19 Jul 2024 13:39:28 +0000 Subject: [PATCH 4/4] Removed sentry since they should already be captured --- cmd/tea_submitplan.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cmd/tea_submitplan.go b/cmd/tea_submitplan.go index 5c0bea7b..aa80d9ab 100644 --- a/cmd/tea_submitplan.go +++ b/cmd/tea_submitplan.go @@ -13,7 +13,6 @@ import ( "connectrpc.com/connect" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" - "github.com/getsentry/sentry-go" "github.com/google/uuid" "github.com/muesli/reflow/wordwrap" "github.com/overmindtech/sdp-go" @@ -681,9 +680,6 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { m.processing <- submitPlanUpdateMsg{m.blastRadiusTask.UpdateStatusMsg(taskStatusError)} m.processing <- submitPlanUpdateMsg{m.risksError("failed to update planned changes", err)} close(m.processing) - - // TODO: Ask David about the position of these too - sentry.CaptureException(err) return nil } @@ -723,9 +719,6 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { m.processing <- submitPlanUpdateMsg{m.blastRadiusTask.UpdateStatusMsg(taskStatusError)} m.processing <- submitPlanUpdateMsg{m.risksError("error streaming results", err)} close(m.processing) - - // TODO: Ask David about the position of these too - sentry.CaptureException(err) return nil } m.processing <- submitPlanUpdateMsg{m.blastRadiusTask.FinishMsg()}