From ba69cc6adef277f4bd5d29671ba56482035fae96 Mon Sep 17 00:00:00 2001 From: Martin C Drohmann Date: Thu, 29 Apr 2021 10:20:30 -0700 Subject: [PATCH 1/2] facilitate incremental download of packages --- .../implementations/alternative/runtime.go | 15 ++++--- .../setup/implementations/camel/runtime.go | 4 +- pkg/platform/runtime/setup/setup.go | 42 +++++++++++++------ 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/pkg/platform/runtime/setup/implementations/alternative/runtime.go b/pkg/platform/runtime/setup/implementations/alternative/runtime.go index 8b373d7059..7505d1d480 100644 --- a/pkg/platform/runtime/setup/implementations/alternative/runtime.go +++ b/pkg/platform/runtime/setup/implementations/alternative/runtime.go @@ -25,7 +25,9 @@ func NewSetup(store *store.Store, artifacts artifact.ArtifactRecipeMap) *Setup { return &Setup{store: store, artifacts: artifacts} } -func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, storedArtifacted store.StoredArtifactMap) error { +func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, storedArtifacted store.StoredArtifactMap) ([]artifact.ArtifactID, error) { + var alreadyInstalled []artifact.ArtifactID + // copy storedArtifactMap keep := make(map[artifact.ArtifactID]store.StoredArtifact) for k, v := range storedArtifacted { @@ -60,7 +62,7 @@ func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, st continue } if err := os.Remove(file); err != nil { - return locale.WrapError(err, "err_rm_artf", "Could not remove old package file at {{.V0}}.", file) + return nil, locale.WrapError(err, "err_rm_artf", "Could not remove old package file at {{.V0}}.", file) } } @@ -85,16 +87,19 @@ func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, st err = os.RemoveAll(dir) if err != nil { - return locale.WrapError(err, "err_rm_artf_dir", "Could not remove empty artifact directory at {{.V0}}", dir) + return alreadyInstalled, locale.WrapError(err, "err_rm_artf_dir", "Could not remove empty artifact directory at {{.V0}}", dir) } } if err := s.store.DeleteArtifactStore(artf.ArtifactID); err != nil { - return errs.Wrap(err, "Could not delete artifact store") + return alreadyInstalled, errs.Wrap(err, "Could not delete artifact store") } } - return nil + for k := range keep { + alreadyInstalled = append(alreadyInstalled, k) + } + return alreadyInstalled, nil } // dirCanBeDeleted checks if the given directory is empty - ignoring files and sub-directories that diff --git a/pkg/platform/runtime/setup/implementations/camel/runtime.go b/pkg/platform/runtime/setup/implementations/camel/runtime.go index 686685b897..ff0025c199 100644 --- a/pkg/platform/runtime/setup/implementations/camel/runtime.go +++ b/pkg/platform/runtime/setup/implementations/camel/runtime.go @@ -18,10 +18,10 @@ func NewSetup(s *store.Store) *Setup { return &Setup{s} } -func (s *Setup) DeleteOutdatedArtifacts(_ artifact.ArtifactChangeset, _ store.StoredArtifactMap) error { +func (s *Setup) DeleteOutdatedArtifacts(_ artifact.ArtifactChangeset, _ store.StoredArtifactMap) ([]artifact.ArtifactID, error) { err := os.RemoveAll(s.store.InstallPath()) logging.Error("Error removing previous camel installation: %v", err) - return nil + return []artifact.ArtifactID{}, nil } func (s *Setup) ResolveArtifactName(_ artifact.ArtifactID) string { diff --git a/pkg/platform/runtime/setup/setup.go b/pkg/platform/runtime/setup/setup.go index ea2b97f3c3..8483d7c509 100644 --- a/pkg/platform/runtime/setup/setup.go +++ b/pkg/platform/runtime/setup/setup.go @@ -11,6 +11,7 @@ import ( "github.com/ActiveState/cli/pkg/platform/runtime/executor" "github.com/gammazero/workerpool" "github.com/go-openapi/strfmt" + "github.com/thoas/go-funk" "github.com/ActiveState/cli/internal/analytics" "github.com/ActiveState/cli/internal/constants" @@ -114,7 +115,8 @@ type ModelProvider interface { } type Setuper interface { - DeleteOutdatedArtifacts(artifact.ArtifactChangeset, store.StoredArtifactMap) error + // DeleteOutdatedArtifacts deletes outdated artifact as best as it can, and returns the IDs of those that are already installed + DeleteOutdatedArtifacts(artifact.ArtifactChangeset, store.StoredArtifactMap) ([]artifact.ArtifactID, error) ResolveArtifactName(artifact.ArtifactID) string DownloadsFromBuild(buildStatus *headchef_models.BuildStatusResponse) ([]artifact.ArtifactDownload, error) } @@ -184,12 +186,22 @@ func (s *Setup) update() error { return locale.WrapError(err, "err_stored_artifacts", "Could not unmarshal stored artifacts, your install may be corrupted.") } - setup.DeleteOutdatedArtifacts(changedArtifacts, storedArtifacts) + alreadyInstalled, err := setup.DeleteOutdatedArtifacts(changedArtifacts, storedArtifacts) + if err != nil { + logging.Error("Could not delete outdated artifacts: %v, falling back to removing everything", err) + err = os.RemoveAll(s.store.InstallPath()) + if err != nil { + return locale.WrapError(err, "Failed to clean installation path") + } + } - // if we get here, we dowload artifacts - analytics.Event(analytics.CatRuntime, analytics.ActRuntimeDownload) + // only send the download analytics event, if we have to install artifacts that are not yet installed + if len(artifacts) != len(alreadyInstalled) { + // if we get here, we dowload artifacts + analytics.Event(analytics.CatRuntime, analytics.ActRuntimeDownload) + } - err = s.installArtifacts(buildResult, artifacts, setup) + err = s.installArtifacts(buildResult, artifacts, alreadyInstalled, setup) if err != nil { return err } @@ -260,7 +272,7 @@ func aggregateErrors() (chan<- error, <-chan error) { return bgErrs, aggErr } -func (s *Setup) installArtifacts(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, setup Setuper) error { +func (s *Setup) installArtifacts(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, alreadyInstalled []artifact.ArtifactID, setup Setuper) error { if !buildResult.BuildReady && buildResult.BuildEngine == model.Camel { return locale.NewInputError("build_status_in_progress", "", apimodel.ProjectURL(s.target.Owner(), s.target.Name(), s.target.CommitUUID().String())) } @@ -270,9 +282,9 @@ func (s *Setup) installArtifacts(buildResult *model.BuildResult, artifacts artif var err error if buildResult.BuildReady { - err = s.installFromBuildResult(buildResult, setup) + err = s.installFromBuildResult(buildResult, alreadyInstalled, setup) } else { - err = s.installFromBuildLog(buildResult, artifacts, setup) + err = s.installFromBuildLog(buildResult, artifacts, alreadyInstalled, setup) } return err @@ -291,18 +303,21 @@ func (s *Setup) setupArtifactSubmitFunction(a artifact.ArtifactDownload, buildRe } } -func (s *Setup) installFromBuildResult(buildResult *model.BuildResult, setup Setuper) error { +func (s *Setup) installFromBuildResult(buildResult *model.BuildResult, alreadyInstalled []artifact.ArtifactID, setup Setuper) error { downloads, err := setup.DownloadsFromBuild(buildResult.BuildStatusResponse) if err != nil { return errs.Wrap(err, "Could not fetch artifacts to download.") } - s.events.TotalArtifacts(len(downloads)) + s.events.TotalArtifacts(len(downloads) - len(alreadyInstalled)) errs, aggregatedErr := aggregateErrors() mainthread.Run(func() { defer close(errs) wp := workerpool.New(MaxConcurrency) for _, a := range downloads { + if funk.Contains(alreadyInstalled, a.ArtifactID) { + continue + } wp.Submit(s.setupArtifactSubmitFunction(a, buildResult, setup, errs)) } @@ -312,8 +327,8 @@ func (s *Setup) installFromBuildResult(buildResult *model.BuildResult, setup Set return <-aggregatedErr } -func (s *Setup) installFromBuildLog(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, setup Setuper) error { - s.events.TotalArtifacts(len(artifacts)) +func (s *Setup) installFromBuildLog(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, alreadyInstalled []artifact.ArtifactID, setup Setuper) error { + s.events.TotalArtifacts(len(artifacts) - len(alreadyInstalled)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -341,6 +356,9 @@ func (s *Setup) installFromBuildLog(buildResult *model.BuildResult, artifacts ar defer wp.StopWait() for a := range buildLog.BuiltArtifactsChannel() { + if funk.Contains(alreadyInstalled, a.ArtifactID) { + continue + } wp.Submit(s.setupArtifactSubmitFunction(a, buildResult, setup, errs)) } }() From e7c77e8594f43d15be34bc6906a9dcc67b77848f Mon Sep 17 00:00:00 2001 From: Martin C Drohmann Date: Thu, 29 Apr 2021 11:26:38 -0700 Subject: [PATCH 2/2] split up DeleteOutdatedArtifacts --- .../implementations/alternative/runtime.go | 40 +++++++++++-------- .../setup/implementations/camel/runtime.go | 9 ++++- pkg/platform/runtime/setup/setup.go | 21 +++++----- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/pkg/platform/runtime/setup/implementations/alternative/runtime.go b/pkg/platform/runtime/setup/implementations/alternative/runtime.go index 7505d1d480..a2385179d8 100644 --- a/pkg/platform/runtime/setup/implementations/alternative/runtime.go +++ b/pkg/platform/runtime/setup/implementations/alternative/runtime.go @@ -25,26 +25,35 @@ func NewSetup(store *store.Store, artifacts artifact.ArtifactRecipeMap) *Setup { return &Setup{store: store, artifacts: artifacts} } -func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, storedArtifacted store.StoredArtifactMap) ([]artifact.ArtifactID, error) { - var alreadyInstalled []artifact.ArtifactID - - // copy storedArtifactMap - keep := make(map[artifact.ArtifactID]store.StoredArtifact) +func (s *Setup) ReusableArtifacts(changeset artifact.ArtifactChangeset, storedArtifacted store.StoredArtifactMap) store.StoredArtifactMap { + keep := make(store.StoredArtifactMap) + // copy store for k, v := range storedArtifacted { keep[k] = v } - del := map[artifact.ArtifactID]struct{}{} + + // remove all updated and removed artifacts for _, upd := range changeset.Updated { delete(keep, upd.FromID) - del[upd.FromID] = struct{}{} } for _, id := range changeset.Removed { delete(keep, id) + } + + return keep +} + +func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, storedArtifacted, alreadyInstalled store.StoredArtifactMap) error { + del := map[artifact.ArtifactID]struct{}{} + for _, upd := range changeset.Updated { + del[upd.FromID] = struct{}{} + } + for _, id := range changeset.Removed { del[id] = struct{}{} } // sort files and dirs in keep for faster look-up - for _, artf := range keep { + for _, artf := range alreadyInstalled { sort.Strings(artf.Dirs) sort.Strings(artf.Files) } @@ -58,11 +67,11 @@ func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, st if !fileutils.TargetExists(file) { continue // don't care it's already deleted (might have been deleted by another artifact that supplied the same file) } - if artifactsContainFile(file, keep) { + if artifactsContainFile(file, alreadyInstalled) { continue } if err := os.Remove(file); err != nil { - return nil, locale.WrapError(err, "err_rm_artf", "Could not remove old package file at {{.V0}}.", file) + return locale.WrapError(err, "err_rm_artf", "Could not remove old package file at {{.V0}}.", file) } } @@ -76,7 +85,7 @@ func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, st continue } - deleteOk, err := dirCanBeDeleted(dir, keep) + deleteOk, err := dirCanBeDeleted(dir, alreadyInstalled) if err != nil { logging.Error("Could not determine if directory %s could be deleted: %v", dir, err) continue @@ -87,19 +96,16 @@ func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, st err = os.RemoveAll(dir) if err != nil { - return alreadyInstalled, locale.WrapError(err, "err_rm_artf_dir", "Could not remove empty artifact directory at {{.V0}}", dir) + return locale.WrapError(err, "err_rm_artf_dir", "Could not remove empty artifact directory at {{.V0}}", dir) } } if err := s.store.DeleteArtifactStore(artf.ArtifactID); err != nil { - return alreadyInstalled, errs.Wrap(err, "Could not delete artifact store") + return errs.Wrap(err, "Could not delete artifact store") } } - for k := range keep { - alreadyInstalled = append(alreadyInstalled, k) - } - return alreadyInstalled, nil + return nil } // dirCanBeDeleted checks if the given directory is empty - ignoring files and sub-directories that diff --git a/pkg/platform/runtime/setup/implementations/camel/runtime.go b/pkg/platform/runtime/setup/implementations/camel/runtime.go index ff0025c199..e99fa85a3a 100644 --- a/pkg/platform/runtime/setup/implementations/camel/runtime.go +++ b/pkg/platform/runtime/setup/implementations/camel/runtime.go @@ -18,10 +18,15 @@ func NewSetup(s *store.Store) *Setup { return &Setup{s} } -func (s *Setup) DeleteOutdatedArtifacts(_ artifact.ArtifactChangeset, _ store.StoredArtifactMap) ([]artifact.ArtifactID, error) { +// ReusableArtifacts returns an empty, because camel installations cannot re-use and artifacts from previous installations +func (s *Setup) ReusableArtifacts(_ artifact.ArtifactChangeset, _ store.StoredArtifactMap) store.StoredArtifactMap { + return make(store.StoredArtifactMap) +} + +func (s *Setup) DeleteOutdatedArtifacts(_ artifact.ArtifactChangeset, _, _ store.StoredArtifactMap) error { err := os.RemoveAll(s.store.InstallPath()) logging.Error("Error removing previous camel installation: %v", err) - return []artifact.ArtifactID{}, nil + return nil } func (s *Setup) ResolveArtifactName(_ artifact.ArtifactID) string { diff --git a/pkg/platform/runtime/setup/setup.go b/pkg/platform/runtime/setup/setup.go index 8483d7c509..a21fdb52c5 100644 --- a/pkg/platform/runtime/setup/setup.go +++ b/pkg/platform/runtime/setup/setup.go @@ -11,7 +11,6 @@ import ( "github.com/ActiveState/cli/pkg/platform/runtime/executor" "github.com/gammazero/workerpool" "github.com/go-openapi/strfmt" - "github.com/thoas/go-funk" "github.com/ActiveState/cli/internal/analytics" "github.com/ActiveState/cli/internal/constants" @@ -115,8 +114,10 @@ type ModelProvider interface { } type Setuper interface { - // DeleteOutdatedArtifacts deletes outdated artifact as best as it can, and returns the IDs of those that are already installed - DeleteOutdatedArtifacts(artifact.ArtifactChangeset, store.StoredArtifactMap) ([]artifact.ArtifactID, error) + // ReusableArtifact returns artifact stores for the artifacts that are already installed and can be re-used for this setup. + ReusableArtifacts(artifact.ArtifactChangeset, store.StoredArtifactMap) store.StoredArtifactMap + // DeleteOutdatedArtifacts deletes outdated artifact as best as it can + DeleteOutdatedArtifacts(artifact.ArtifactChangeset, store.StoredArtifactMap, store.StoredArtifactMap) error ResolveArtifactName(artifact.ArtifactID) string DownloadsFromBuild(buildStatus *headchef_models.BuildStatusResponse) ([]artifact.ArtifactDownload, error) } @@ -186,7 +187,9 @@ func (s *Setup) update() error { return locale.WrapError(err, "err_stored_artifacts", "Could not unmarshal stored artifacts, your install may be corrupted.") } - alreadyInstalled, err := setup.DeleteOutdatedArtifacts(changedArtifacts, storedArtifacts) + alreadyInstalled := setup.ReusableArtifacts(changedArtifacts, storedArtifacts) + + err = setup.DeleteOutdatedArtifacts(changedArtifacts, storedArtifacts, alreadyInstalled) if err != nil { logging.Error("Could not delete outdated artifacts: %v, falling back to removing everything", err) err = os.RemoveAll(s.store.InstallPath()) @@ -272,7 +275,7 @@ func aggregateErrors() (chan<- error, <-chan error) { return bgErrs, aggErr } -func (s *Setup) installArtifacts(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, alreadyInstalled []artifact.ArtifactID, setup Setuper) error { +func (s *Setup) installArtifacts(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, alreadyInstalled store.StoredArtifactMap, setup Setuper) error { if !buildResult.BuildReady && buildResult.BuildEngine == model.Camel { return locale.NewInputError("build_status_in_progress", "", apimodel.ProjectURL(s.target.Owner(), s.target.Name(), s.target.CommitUUID().String())) } @@ -303,7 +306,7 @@ func (s *Setup) setupArtifactSubmitFunction(a artifact.ArtifactDownload, buildRe } } -func (s *Setup) installFromBuildResult(buildResult *model.BuildResult, alreadyInstalled []artifact.ArtifactID, setup Setuper) error { +func (s *Setup) installFromBuildResult(buildResult *model.BuildResult, alreadyInstalled store.StoredArtifactMap, setup Setuper) error { downloads, err := setup.DownloadsFromBuild(buildResult.BuildStatusResponse) if err != nil { return errs.Wrap(err, "Could not fetch artifacts to download.") @@ -315,7 +318,7 @@ func (s *Setup) installFromBuildResult(buildResult *model.BuildResult, alreadyIn defer close(errs) wp := workerpool.New(MaxConcurrency) for _, a := range downloads { - if funk.Contains(alreadyInstalled, a.ArtifactID) { + if _, ok := alreadyInstalled[a.ArtifactID]; ok { continue } wp.Submit(s.setupArtifactSubmitFunction(a, buildResult, setup, errs)) @@ -327,7 +330,7 @@ func (s *Setup) installFromBuildResult(buildResult *model.BuildResult, alreadyIn return <-aggregatedErr } -func (s *Setup) installFromBuildLog(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, alreadyInstalled []artifact.ArtifactID, setup Setuper) error { +func (s *Setup) installFromBuildLog(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, alreadyInstalled store.StoredArtifactMap, setup Setuper) error { s.events.TotalArtifacts(len(artifacts) - len(alreadyInstalled)) ctx, cancel := context.WithCancel(context.Background()) @@ -356,7 +359,7 @@ func (s *Setup) installFromBuildLog(buildResult *model.BuildResult, artifacts ar defer wp.StopWait() for a := range buildLog.BuiltArtifactsChannel() { - if funk.Contains(alreadyInstalled, a.ArtifactID) { + if _, ok := alreadyInstalled[a.ArtifactID]; ok { continue } wp.Submit(s.setupArtifactSubmitFunction(a, buildResult, setup, errs))