Facilitate incremental download of packages

pkg/platform/runtime/setup/implementations/alternative/runtime.go

@@ -25,24 +25,35 @@ func NewSetup(store *store.Store, artifacts artifact.ArtifactRecipeMap) *Setup {
 	return &Setup{store: store, artifacts: artifacts}
 }
 
-func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, storedArtifacted store.StoredArtifactMap) error {
-	// copy storedArtifactMap
-	keep := make(map[artifact.ArtifactID]store.StoredArtifact)
+func (s *Setup) ReusableArtifacts(changeset artifact.ArtifactChangeset, storedArtifacted store.StoredArtifactMap) store.StoredArtifactMap {
+	keep := make(store.StoredArtifactMap)
+	// copy store
 	for k, v := range storedArtifacted {
 		keep[k] = v
 	}
-	del := map[artifact.ArtifactID]struct{}{}
+
+	// remove all updated and removed artifacts
 	for _, upd := range changeset.Updated {
 		delete(keep, upd.FromID)
-		del[upd.FromID] = struct{}{}
 	}
 	for _, id := range changeset.Removed {
 		delete(keep, id)
+	}
+
+	return keep
+}
+
+func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, storedArtifacted, alreadyInstalled store.StoredArtifactMap) error {
+	del := map[artifact.ArtifactID]struct{}{}
+	for _, upd := range changeset.Updated {
+		del[upd.FromID] = struct{}{}
+	}
+	for _, id := range changeset.Removed {
 		del[id] = struct{}{}
 	}
 
 	// sort files and dirs in keep for faster look-up
-	for _, artf := range keep {
+	for _, artf := range alreadyInstalled {
 		sort.Strings(artf.Dirs)
 		sort.Strings(artf.Files)
 	}
@@ -56,7 +67,7 @@ func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, st
 			if !fileutils.TargetExists(file) {
 				continue // don't care it's already deleted (might have been deleted by another artifact that supplied the same file)
 			}
-			if artifactsContainFile(file, keep) {
+			if artifactsContainFile(file, alreadyInstalled) {
 				continue
 			}
 			if err := os.Remove(file); err != nil {
@@ -74,7 +85,7 @@ func (s *Setup) DeleteOutdatedArtifacts(changeset artifact.ArtifactChangeset, st
 				continue
 			}
 
-			deleteOk, err := dirCanBeDeleted(dir, keep)
+			deleteOk, err := dirCanBeDeleted(dir, alreadyInstalled)
 			if err != nil {
 				logging.Error("Could not determine if directory %s could be deleted: %v", dir, err)
 				continue

pkg/platform/runtime/setup/implementations/camel/runtime.go

@@ -18,7 +18,12 @@ func NewSetup(s *store.Store) *Setup {
 	return &Setup{s}
 }
 
-func (s *Setup) DeleteOutdatedArtifacts(_ artifact.ArtifactChangeset, _ store.StoredArtifactMap) error {
+// ReusableArtifacts returns an empty, because camel installations cannot re-use and artifacts from previous installations
+func (s *Setup) ReusableArtifacts(_ artifact.ArtifactChangeset, _ store.StoredArtifactMap) store.StoredArtifactMap {
+	return make(store.StoredArtifactMap)
+}
+
+func (s *Setup) DeleteOutdatedArtifacts(_ artifact.ArtifactChangeset, _, _ store.StoredArtifactMap) error {
 	err := os.RemoveAll(s.store.InstallPath())
 	logging.Error("Error removing previous camel installation: %v", err)
 	return nil

pkg/platform/runtime/setup/setup.go

@@ -114,7 +114,10 @@ type ModelProvider interface {
 }
 
 type Setuper interface {
-	DeleteOutdatedArtifacts(artifact.ArtifactChangeset, store.StoredArtifactMap) error
+	// ReusableArtifact returns artifact stores for the artifacts that are already installed and can be re-used for this setup.
+	ReusableArtifacts(artifact.ArtifactChangeset, store.StoredArtifactMap) store.StoredArtifactMap
+	// DeleteOutdatedArtifacts deletes outdated artifact as best as it can
+	DeleteOutdatedArtifacts(artifact.ArtifactChangeset, store.StoredArtifactMap, store.StoredArtifactMap) error
 	ResolveArtifactName(artifact.ArtifactID) string
 	DownloadsFromBuild(buildStatus *headchef_models.BuildStatusResponse) ([]artifact.ArtifactDownload, error)
 }
@@ -184,12 +187,24 @@ func (s *Setup) update() error {
 		return locale.WrapError(err, "err_stored_artifacts", "Could not unmarshal stored artifacts, your install may be corrupted.")
 	}
 
-	setup.DeleteOutdatedArtifacts(changedArtifacts, storedArtifacts)
+	alreadyInstalled := setup.ReusableArtifacts(changedArtifacts, storedArtifacts)
 
-	// if we get here, we dowload artifacts
-	analytics.Event(analytics.CatRuntime, analytics.ActRuntimeDownload)
+	err = setup.DeleteOutdatedArtifacts(changedArtifacts, storedArtifacts, alreadyInstalled)
+	if err != nil {
+		logging.Error("Could not delete outdated artifacts: %v, falling back to removing everything", err)
+		err = os.RemoveAll(s.store.InstallPath())
+		if err != nil {
+			return locale.WrapError(err, "Failed to clean installation path")
+		}
+	}
 
-	err = s.installArtifacts(buildResult, artifacts, setup)
+	// only send the download analytics event, if we have to install artifacts that are not yet installed
+	if len(artifacts) != len(alreadyInstalled) {
+		// if we get here, we dowload artifacts
+		analytics.Event(analytics.CatRuntime, analytics.ActRuntimeDownload)
+	}
+
+	err = s.installArtifacts(buildResult, artifacts, alreadyInstalled, setup)
 	if err != nil {
 		return err
 	}
@@ -260,7 +275,7 @@ func aggregateErrors() (chan<- error, <-chan error) {
 	return bgErrs, aggErr
 }
 
-func (s *Setup) installArtifacts(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, setup Setuper) error {
+func (s *Setup) installArtifacts(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, alreadyInstalled store.StoredArtifactMap, setup Setuper) error {
 	if !buildResult.BuildReady && buildResult.BuildEngine == model.Camel {
 		return locale.NewInputError("build_status_in_progress", "", apimodel.ProjectURL(s.target.Owner(), s.target.Name(), s.target.CommitUUID().String()))
 	}
@@ -270,9 +285,9 @@ func (s *Setup) installArtifacts(buildResult *model.BuildResult, artifacts artif
 
 	var err error
 	if buildResult.BuildReady {
-		err = s.installFromBuildResult(buildResult, setup)
+		err = s.installFromBuildResult(buildResult, alreadyInstalled, setup)
 	} else {
-		err = s.installFromBuildLog(buildResult, artifacts, setup)
+		err = s.installFromBuildLog(buildResult, artifacts, alreadyInstalled, setup)
 	}
 
 	return err
@@ -291,18 +306,21 @@ func (s *Setup) setupArtifactSubmitFunction(a artifact.ArtifactDownload, buildRe
 	}
 }
 
-func (s *Setup) installFromBuildResult(buildResult *model.BuildResult, setup Setuper) error {
+func (s *Setup) installFromBuildResult(buildResult *model.BuildResult, alreadyInstalled store.StoredArtifactMap, setup Setuper) error {
 	downloads, err := setup.DownloadsFromBuild(buildResult.BuildStatusResponse)
 	if err != nil {
 		return errs.Wrap(err, "Could not fetch artifacts to download.")
 	}
-	s.events.TotalArtifacts(len(downloads))
+	s.events.TotalArtifacts(len(downloads) - len(alreadyInstalled))
 
 	errs, aggregatedErr := aggregateErrors()
 	mainthread.Run(func() {
 		defer close(errs)
 		wp := workerpool.New(MaxConcurrency)
 		for _, a := range downloads {
+			if _, ok := alreadyInstalled[a.ArtifactID]; ok {
+				continue
+			}
 			wp.Submit(s.setupArtifactSubmitFunction(a, buildResult, setup, errs))
 		}
 
@@ -312,8 +330,8 @@ func (s *Setup) installFromBuildResult(buildResult *model.BuildResult, setup Set
 	return <-aggregatedErr
 }
 
-func (s *Setup) installFromBuildLog(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, setup Setuper) error {
-	s.events.TotalArtifacts(len(artifacts))
+func (s *Setup) installFromBuildLog(buildResult *model.BuildResult, artifacts artifact.ArtifactRecipeMap, alreadyInstalled store.StoredArtifactMap, setup Setuper) error {
+	s.events.TotalArtifacts(len(artifacts) - len(alreadyInstalled))
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -341,6 +359,9 @@ func (s *Setup) installFromBuildLog(buildResult *model.BuildResult, artifacts ar
 			defer wp.StopWait()
 
 			for a := range buildLog.BuiltArtifactsChannel() {
+				if _, ok := alreadyInstalled[a.ArtifactID]; ok {
+					continue
+				}
 				wp.Submit(s.setupArtifactSubmitFunction(a, buildResult, setup, errs))
 			}
 		}()