From 9dd6c5102fe2b9ad1bb302269c10383b5f084b74 Mon Sep 17 00:00:00 2001 From: Jeff Wendling Date: Tue, 19 Mar 2024 11:35:50 -0400 Subject: [PATCH] cmd/uplink: add mon.Task and allow writing stats to a file adds mon.Task calls to all command Execute methods as well as some other functions that had contexts available already and adds a --debug-monkit-stats flag to write the stats to when the command finishes. Change-Id: I06581f278403054893ac83403c18410c121a96f8 --- cmd/uplink/access_maker.go | 2 ++ cmd/uplink/cmd_access_create.go | 2 ++ cmd/uplink/cmd_access_export.go | 4 +++- cmd/uplink/cmd_access_import.go | 2 ++ cmd/uplink/cmd_access_inspect.go | 4 +++- cmd/uplink/cmd_access_list.go | 4 +++- cmd/uplink/cmd_access_register.go | 2 ++ cmd/uplink/cmd_access_remove.go | 4 +++- cmd/uplink/cmd_access_restrict.go | 4 +++- cmd/uplink/cmd_access_revoke.go | 4 +++- cmd/uplink/cmd_access_setup.go | 2 ++ cmd/uplink/cmd_access_use.go | 4 +++- cmd/uplink/cmd_cp.go | 17 +++++++++++++---- cmd/uplink/cmd_ls.go | 12 +++++++++--- cmd/uplink/cmd_mb.go | 4 +++- cmd/uplink/cmd_meta_get.go | 4 +++- cmd/uplink/cmd_mv.go | 12 +++++++++--- cmd/uplink/cmd_rb.go | 4 +++- cmd/uplink/cmd_rm.go | 4 +++- cmd/uplink/cmd_share.go | 4 +++- cmd/uplink/cmd_version.go | 4 +++- cmd/uplink/external.go | 19 +++++++++++++++++++ cmd/uplink/external_access.go | 8 ++++++-- cmd/uplink/external_project.go | 8 ++++++-- 24 files changed, 111 insertions(+), 27 deletions(-) diff --git a/cmd/uplink/access_maker.go b/cmd/uplink/access_maker.go index 7c36db4e9b72..65092e38f42a 100644 --- a/cmd/uplink/access_maker.go +++ b/cmd/uplink/access_maker.go @@ -38,6 +38,8 @@ func (am *accessMaker) Setup(params clingy.Parameters, ex ulext.External) { } func (am *accessMaker) Execute(ctx context.Context, name string, access *uplink.Access) (_ *uplink.Access, err error) { + defer mon.Task()(&ctx)(&err) + defaultName, accesses, err := am.ex.GetAccessInfo(false) if err != nil { return nil, err diff --git a/cmd/uplink/cmd_access_create.go b/cmd/uplink/cmd_access_create.go index cce534c1177a..8d888cfbad5c 100644 --- a/cmd/uplink/cmd_access_create.go +++ b/cmd/uplink/cmd_access_create.go @@ -53,6 +53,8 @@ func (c *cmdAccessCreate) Setup(params clingy.Parameters) { } func (c *cmdAccessCreate) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + if c.satelliteAddr == "" { if c.passphraseStdin { return errs.New("Must specify the satellite address as a flag when passphrase-stdin is set.") diff --git a/cmd/uplink/cmd_access_export.go b/cmd/uplink/cmd_access_export.go index d6dba43c4287..ec93b5951d5e 100644 --- a/cmd/uplink/cmd_access_export.go +++ b/cmd/uplink/cmd_access_export.go @@ -28,7 +28,9 @@ func (c *cmdAccessExport) Setup(params clingy.Parameters) { c.filename = params.Arg("filename", "Name of the file to save to").(string) } -func (c *cmdAccessExport) Execute(ctx context.Context) error { +func (c *cmdAccessExport) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + if c.filename == "" { return errs.New("Must specify a filename to write to.") } diff --git a/cmd/uplink/cmd_access_import.go b/cmd/uplink/cmd_access_import.go index acc5f79bf9c5..90a1c9e38f7b 100644 --- a/cmd/uplink/cmd_access_import.go +++ b/cmd/uplink/cmd_access_import.go @@ -32,6 +32,8 @@ func (c *cmdAccessImport) Setup(params clingy.Parameters) { } func (c *cmdAccessImport) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + if c.name == "" { return errs.New("Must specify a name to import the access as.") } diff --git a/cmd/uplink/cmd_access_inspect.go b/cmd/uplink/cmd_access_inspect.go index 7089891f00ed..f1ca4bae4075 100644 --- a/cmd/uplink/cmd_access_inspect.go +++ b/cmd/uplink/cmd_access_inspect.go @@ -38,7 +38,9 @@ func (c *cmdAccessInspect) Setup(params clingy.Parameters) { } // Execute runs the command. -func (c *cmdAccessInspect) Execute(ctx context.Context) error { +func (c *cmdAccessInspect) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + toOpen := "" if c.access != nil { toOpen = *c.access diff --git a/cmd/uplink/cmd_access_list.go b/cmd/uplink/cmd_access_list.go index 47d68e61a70f..77e049ba1e76 100644 --- a/cmd/uplink/cmd_access_list.go +++ b/cmd/uplink/cmd_access_list.go @@ -32,7 +32,9 @@ func (c *cmdAccessList) Setup(params clingy.Parameters) { ).(bool) } -func (c *cmdAccessList) Execute(ctx context.Context) error { +func (c *cmdAccessList) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + defaultName, accesses, err := c.ex.GetAccessInfo(true) if err != nil { return err diff --git a/cmd/uplink/cmd_access_register.go b/cmd/uplink/cmd_access_register.go index 1f7528241bdb..de92be7c563e 100644 --- a/cmd/uplink/cmd_access_register.go +++ b/cmd/uplink/cmd_access_register.go @@ -38,6 +38,8 @@ func (c *cmdAccessRegister) Setup(params clingy.Parameters) { } func (c *cmdAccessRegister) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + accessNameOrValue := "" if c.accessNameOrValue != nil && len(*c.accessNameOrValue) > 0 { accessNameOrValue = *c.accessNameOrValue diff --git a/cmd/uplink/cmd_access_remove.go b/cmd/uplink/cmd_access_remove.go index 0a3bf5bbdaa8..8f97cb1bfbe5 100644 --- a/cmd/uplink/cmd_access_remove.go +++ b/cmd/uplink/cmd_access_remove.go @@ -27,7 +27,9 @@ func (c *cmdAccessRemove) Setup(params clingy.Parameters) { c.access = params.Arg("name", "Access name to delete").(string) } -func (c *cmdAccessRemove) Execute(ctx context.Context) error { +func (c *cmdAccessRemove) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + defaultName, accesses, err := c.ex.GetAccessInfo(true) if err != nil { return err diff --git a/cmd/uplink/cmd_access_restrict.go b/cmd/uplink/cmd_access_restrict.go index 45f3b2d9f622..3d853f1e4236 100644 --- a/cmd/uplink/cmd_access_restrict.go +++ b/cmd/uplink/cmd_access_restrict.go @@ -37,7 +37,9 @@ func (c *cmdAccessRestrict) Setup(params clingy.Parameters) { c.am.perms.Setup(params, true) } -func (c *cmdAccessRestrict) Execute(ctx context.Context) error { +func (c *cmdAccessRestrict) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + access, err := c.ex.OpenAccess(c.access) if err != nil { return err diff --git a/cmd/uplink/cmd_access_revoke.go b/cmd/uplink/cmd_access_revoke.go index 2cff32a77420..8d1908e6fd52 100644 --- a/cmd/uplink/cmd_access_revoke.go +++ b/cmd/uplink/cmd_access_revoke.go @@ -28,7 +28,9 @@ func (c *cmdAccessRevoke) Setup(params clingy.Parameters) { c.revokee = params.Arg("revokee", "Access name or value revoke").(string) } -func (c *cmdAccessRevoke) Execute(ctx context.Context) error { +func (c *cmdAccessRevoke) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + project, err := c.ex.OpenProject(ctx, c.access) if err != nil { return err diff --git a/cmd/uplink/cmd_access_setup.go b/cmd/uplink/cmd_access_setup.go index 05345a5722c6..04ab142d2f3c 100644 --- a/cmd/uplink/cmd_access_setup.go +++ b/cmd/uplink/cmd_access_setup.go @@ -39,6 +39,8 @@ func (c *cmdAccessSetup) Setup(params clingy.Parameters) { } func (c *cmdAccessSetup) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + name, err := c.ex.PromptInput(ctx, "Enter name to import as [default: main]:") if err != nil { return errs.Wrap(err) diff --git a/cmd/uplink/cmd_access_use.go b/cmd/uplink/cmd_access_use.go index d55582f5e7a3..e337eab02385 100644 --- a/cmd/uplink/cmd_access_use.go +++ b/cmd/uplink/cmd_access_use.go @@ -26,7 +26,9 @@ func (c *cmdAccessUse) Setup(params clingy.Parameters) { c.access = params.Arg("access", "Access name to use").(string) } -func (c *cmdAccessUse) Execute(ctx context.Context) error { +func (c *cmdAccessUse) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + _, accesses, err := c.ex.GetAccessInfo(true) if err != nil { return err diff --git a/cmd/uplink/cmd_cp.go b/cmd/uplink/cmd_cp.go index 1ab308e56813..6a814c11b322 100644 --- a/cmd/uplink/cmd_cp.go +++ b/cmd/uplink/cmd_cp.go @@ -150,7 +150,9 @@ func (c *cmdCp) Setup(params clingy.Parameters) { ).([]ulloc.Location) } -func (c *cmdCp) Execute(ctx context.Context) error { +func (c *cmdCp) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + if len(c.locs) < 2 { return errs.New("must have at least one source and destination path") } @@ -191,7 +193,9 @@ func (c *cmdCp) Execute(ctx context.Context) error { return combineErrs(eg) } -func (c *cmdCp) dispatchCopy(ctx context.Context, fs ulfs.Filesystem, source, dest ulloc.Location) error { +func (c *cmdCp) dispatchCopy(ctx context.Context, fs ulfs.Filesystem, source, dest ulloc.Location) (err error) { + defer mon.Task()(&ctx)(&err) + if !source.Remote() && !dest.Remote() { return errs.New("at least one location must be a remote sj:// location") } @@ -247,7 +251,9 @@ func (c *cmdCp) dispatchCopy(ctx context.Context, fs ulfs.Filesystem, source, de return c.copyFile(ctx, fs, source, dest, bar) } -func (c *cmdCp) copyRecursive(ctx context.Context, fs ulfs.Filesystem, source, dest ulloc.Location) error { +func (c *cmdCp) copyRecursive(ctx context.Context, fs ulfs.Filesystem, source, dest ulloc.Location) (err error) { + defer mon.Task()(&ctx)(&err) + if source.Std() || dest.Std() { return errs.New("cannot recursively copy to stdin/stdout") } @@ -367,6 +373,8 @@ func (c *cmdCp) copyRecursive(ctx context.Context, fs ulfs.Filesystem, source, d } func (c *cmdCp) copyFile(ctx context.Context, fs ulfs.Filesystem, source, dest ulloc.Location, bar *mpb.Bar) (err error) { + defer mon.Task()(&ctx)(&err) + if c.dryrun { return nil } @@ -525,7 +533,8 @@ func (c *cmdCp) parallelCopy( dst ulfs.MultiWriteHandle, p int, chunkSize int64, offset, length int64, - bar *mpb.Bar) error { + bar *mpb.Bar) (err error) { + defer mon.Task()(&ctx)(&err) if offset != 0 { if err := src.SetOffset(offset); err != nil { diff --git a/cmd/uplink/cmd_ls.go b/cmd/uplink/cmd_ls.go index 5d35344d9f66..c35c1a0bcc99 100644 --- a/cmd/uplink/cmd_ls.go +++ b/cmd/uplink/cmd_ls.go @@ -64,14 +64,18 @@ func (c *cmdLs) Setup(params clingy.Parameters) { ).(*ulloc.Location) } -func (c *cmdLs) Execute(ctx context.Context) error { +func (c *cmdLs) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + if c.prefix == nil { return c.listBuckets(ctx) } return c.listLocation(ctx, *c.prefix) } -func (c *cmdLs) listBuckets(ctx context.Context) error { +func (c *cmdLs) listBuckets(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + project, err := c.ex.OpenProject(ctx, c.access) if err != nil { return err @@ -90,7 +94,9 @@ func (c *cmdLs) listBuckets(ctx context.Context) error { } } -func (c *cmdLs) listLocation(ctx context.Context, prefix ulloc.Location) error { +func (c *cmdLs) listLocation(ctx context.Context, prefix ulloc.Location) (err error) { + defer mon.Task()(&ctx)(&err) + fs, err := c.ex.OpenFilesystem(ctx, c.access, ulext.BypassEncryption(c.encrypted)) if err != nil { return err diff --git a/cmd/uplink/cmd_mb.go b/cmd/uplink/cmd_mb.go index e647bc9007e4..f78558aa4577 100644 --- a/cmd/uplink/cmd_mb.go +++ b/cmd/uplink/cmd_mb.go @@ -38,7 +38,9 @@ func (c *cmdMb) Setup(params clingy.Parameters) { ).(string) } -func (c *cmdMb) Execute(ctx context.Context) error { +func (c *cmdMb) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + project, err := c.ex.OpenProject(ctx, c.access) if err != nil { return errs.Wrap(err) diff --git a/cmd/uplink/cmd_meta_get.go b/cmd/uplink/cmd_meta_get.go index 93db9d20f9e7..a97c7a4260ba 100644 --- a/cmd/uplink/cmd_meta_get.go +++ b/cmd/uplink/cmd_meta_get.go @@ -42,7 +42,9 @@ func (c *cmdMetaGet) Setup(params clingy.Parameters) { c.entry = params.Arg("entry", "Metadata entry to get", clingy.Optional).(*string) } -func (c *cmdMetaGet) Execute(ctx context.Context) error { +func (c *cmdMetaGet) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + project, err := c.ex.OpenProject(ctx, c.access, ulext.BypassEncryption(c.encrypted)) if err != nil { return err diff --git a/cmd/uplink/cmd_mv.go b/cmd/uplink/cmd_mv.go index f30d1b326924..3c46071b1cc8 100644 --- a/cmd/uplink/cmd_mv.go +++ b/cmd/uplink/cmd_mv.go @@ -63,7 +63,9 @@ func (c *cmdMv) Setup(params clingy.Parameters) { c.dest = params.Arg("dest", "Destination to move", clingy.Transform(ulloc.Parse)).(ulloc.Location) } -func (c *cmdMv) Execute(ctx context.Context) error { +func (c *cmdMv) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + fs, err := c.ex.OpenFilesystem(ctx, c.access) if err != nil { return err @@ -114,7 +116,9 @@ func (c *cmdMv) Execute(ctx context.Context) error { return c.moveFile(ctx, fs, c.source, c.dest) } -func (c *cmdMv) moveRecursive(ctx context.Context, fs ulfs.Filesystem) error { +func (c *cmdMv) moveRecursive(ctx context.Context, fs ulfs.Filesystem) (err error) { + defer mon.Task()(&ctx)(&err) + iter, err := fs.List(ctx, c.source, &ulfs.ListOptions{ Recursive: true, }) @@ -188,7 +192,9 @@ func (c *cmdMv) moveRecursive(ctx context.Context, fs ulfs.Filesystem) error { return nil } -func (c *cmdMv) moveFile(ctx context.Context, fs ulfs.Filesystem, source, dest ulloc.Location) error { +func (c *cmdMv) moveFile(ctx context.Context, fs ulfs.Filesystem, source, dest ulloc.Location) (err error) { + defer mon.Task()(&ctx)(&err) + if c.dryrun { return nil } diff --git a/cmd/uplink/cmd_rb.go b/cmd/uplink/cmd_rb.go index fc02a5d00a14..14b3f4779e8c 100644 --- a/cmd/uplink/cmd_rb.go +++ b/cmd/uplink/cmd_rb.go @@ -39,7 +39,9 @@ func (c *cmdRb) Setup(params clingy.Parameters) { ).(ulloc.Location) } -func (c *cmdRb) Execute(ctx context.Context) error { +func (c *cmdRb) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + project, err := c.ex.OpenProject(ctx, c.access) if err != nil { return err diff --git a/cmd/uplink/cmd_rm.go b/cmd/uplink/cmd_rm.go index f89b442f01a4..2a3b522cca91 100644 --- a/cmd/uplink/cmd_rm.go +++ b/cmd/uplink/cmd_rm.go @@ -63,7 +63,9 @@ func (c *cmdRm) Setup(params clingy.Parameters) { ).(ulloc.Location) } -func (c *cmdRm) Execute(ctx context.Context) error { +func (c *cmdRm) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + if c.location.Local() { return errs.New("remove %v skipped: local delete", c.location) } diff --git a/cmd/uplink/cmd_share.go b/cmd/uplink/cmd_share.go index 35f6c4d48d0c..3f065d2a2b93 100644 --- a/cmd/uplink/cmd_share.go +++ b/cmd/uplink/cmd_share.go @@ -66,7 +66,9 @@ func (c *cmdShare) Setup(params clingy.Parameters) { c.ap.Setup(params, false) } -func (c *cmdShare) Execute(ctx context.Context) error { +func (c *cmdShare) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + if len(c.ap.prefixes) == 0 { return errs.New("you must specify at least one prefix to share. Use the access restrict command to restrict with no prefixes") } diff --git a/cmd/uplink/cmd_version.go b/cmd/uplink/cmd_version.go index 3dcfbd756db3..3b404314e50f 100644 --- a/cmd/uplink/cmd_version.go +++ b/cmd/uplink/cmd_version.go @@ -33,7 +33,9 @@ func (c *cmdVersion) Setup(params clingy.Parameters) { ).(bool) } -func (c *cmdVersion) Execute(ctx context.Context) error { +func (c *cmdVersion) Execute(ctx context.Context) (err error) { + defer mon.Task()(&ctx)(&err) + if version.Build.Release { fmt.Fprintln(clingy.Stdout(ctx), "Release build") } else { diff --git a/cmd/uplink/external.go b/cmd/uplink/external.go index 44bc2447350a..3db416fc2557 100644 --- a/cmd/uplink/external.go +++ b/cmd/uplink/external.go @@ -74,6 +74,7 @@ type external struct { pprofFile string traceFile string monkitTraceFile string + monkitStatsFile string } events struct { @@ -160,6 +161,11 @@ func (ex *external) Setup(f clingy.Flags) { clingy.Advanced, ).(string) + ex.debug.monkitStatsFile = f.Flag( + "debug-monkit-stats", "File to collect Monkit stats", "", + clingy.Advanced, + ).(string) + ex.analytics = f.Flag( "analytics", "Whether to send usage information to Storj", nil, clingy.Transform(strconv.ParseBool), clingy.Optional, clingy.Boolean, @@ -249,6 +255,8 @@ func (ex *external) analyticsEnabled() bool { // Wrap is called by clingy with the command to be executed. func (ex *external) Wrap(ctx context.Context, cmd clingy.Command) (err error) { + defer mon.Task()(&ctx)(&err) + if err = ex.migrate(); err != nil { return err } @@ -300,6 +308,17 @@ func (ex *external) Wrap(ctx context.Context, cmd clingy.Command) (err error) { defer trace.Stop() } + if ex.debug.monkitStatsFile != "" { + fh, err := os.Create(ex.debug.monkitStatsFile) + if err != nil { + return errs.Wrap(err) + } + defer func() { _ = fh.Close() }() + defer monkit.Default.Stats(func(key monkit.SeriesKey, field string, val float64) { + fmt.Fprintf(fh, "%v\t%v\t%v\n", key, field, val) + }) + } + // N.B.: Tracing is currently disabled by default (sample == 0, traceID == 0) and is // something a user can only opt into. as a result, we don't check ex.analyticsEnabled() // in this if statement. If we do ever start turning on trace samples by default, we diff --git a/cmd/uplink/external_access.go b/cmd/uplink/external_access.go index 9034981c8a99..b6263cd6df16 100644 --- a/cmd/uplink/external_access.go +++ b/cmd/uplink/external_access.go @@ -156,7 +156,9 @@ func (ex *external) SaveAccessInfo(defaultName string, accesses map[string]strin return nil } -func (ex *external) RequestAccess(ctx context.Context, satelliteAddr, apiKey, passphrase string, unencryptedObjectKeys bool) (*uplink.Access, error) { +func (ex *external) RequestAccess(ctx context.Context, satelliteAddr, apiKey, passphrase string, unencryptedObjectKeys bool) (_ *uplink.Access, err error) { + defer mon.Task()(&ctx)(&err) + config := uplink.Config{} if unencryptedObjectKeys { access.DisableObjectKeyEncryption(&config) @@ -168,7 +170,9 @@ func (ex *external) RequestAccess(ctx context.Context, satelliteAddr, apiKey, pa return access, nil } -func (ex *external) ExportAccess(ctx context.Context, access *uplink.Access, filename string) error { +func (ex *external) ExportAccess(ctx context.Context, access *uplink.Access, filename string) (err error) { + defer mon.Task()(&ctx)(&err) + serialized, err := access.Serialize() if err != nil { return errs.Wrap(err) diff --git a/cmd/uplink/external_project.go b/cmd/uplink/external_project.go index 7a5127938bc3..a238d761ad7b 100644 --- a/cmd/uplink/external_project.go +++ b/cmd/uplink/external_project.go @@ -17,7 +17,9 @@ import ( const uplinkCLIUserAgent = "uplink-cli" -func (ex *external) OpenFilesystem(ctx context.Context, accessName string, options ...ulext.Option) (ulfs.Filesystem, error) { +func (ex *external) OpenFilesystem(ctx context.Context, accessName string, options ...ulext.Option) (_ ulfs.Filesystem, err error) { + defer mon.Task()(&ctx)(&err) + project, err := ex.OpenProject(ctx, accessName, options...) if err != nil { return nil, err @@ -25,7 +27,9 @@ func (ex *external) OpenFilesystem(ctx context.Context, accessName string, optio return ulfs.NewMixed(ulfs.NewLocal(ulfs.NewLocalBackendOS()), ulfs.NewRemote(project)), nil } -func (ex *external) OpenProject(ctx context.Context, accessName string, options ...ulext.Option) (*uplink.Project, error) { +func (ex *external) OpenProject(ctx context.Context, accessName string, options ...ulext.Option) (_ *uplink.Project, err error) { + defer mon.Task()(&ctx)(&err) + opts := ulext.LoadOptions(options...) access, err := ex.OpenAccess(accessName)