Skip to content

Commit

Permalink
Garbage collect unused images
Browse files Browse the repository at this point in the history
  • Loading branch information
divolgin committed Jul 28, 2021
1 parent c849aff commit 6e728e6
Show file tree
Hide file tree
Showing 32 changed files with 847 additions and 72 deletions.
1 change: 1 addition & 0 deletions cmd/kots/cli/admin-console.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ func AdminConsoleCmd() *cobra.Command {

cmd.AddCommand(AdminConsoleUpgradeCmd())
cmd.AddCommand(AdminPushImagesCmd())
cmd.AddCommand(GarbageCollectImagesCmd())

return cmd
}
126 changes: 126 additions & 0 deletions cmd/kots/cli/garbage-collect-images.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package cli

import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"

"github.com/pkg/errors"
"github.com/replicatedhq/kots/pkg/auth"
"github.com/replicatedhq/kots/pkg/k8sutil"
"github.com/replicatedhq/kots/pkg/logger"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)

func GarbageCollectImagesCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "garbage-collect-images [namespace]",
Short: "Run image garbage collection",
Long: `Triggers image garbage collection for all apps`,
SilenceUsage: true,
SilenceErrors: false,
PreRun: func(cmd *cobra.Command, args []string) {
viper.BindPFlags(cmd.Flags())
},
RunE: func(cmd *cobra.Command, args []string) error {
v := viper.GetViper()

log := logger.NewCLILogger()

// use namespace-as-arg if provided, else use namespace from -n/--namespace
namespace := v.GetString("namespace")
if len(args) == 1 {
namespace = args[0]
} else if len(args) > 1 {
fmt.Printf("more than one argument supplied: %+v\n", args)
os.Exit(1)
}

if err := validateNamespace(namespace); err != nil {
return errors.Wrap(err, "failed to validate namespace")
}

stopCh := make(chan struct{})
defer close(stopCh)

clientset, err := k8sutil.GetClientset()
if err != nil {
return errors.Wrap(err, "failed to get clientset")
}

podName, err := k8sutil.FindKotsadm(clientset, namespace)
if err != nil {
return errors.Wrap(err, "failed to find kotsadm pod")
}

localPort, errChan, err := k8sutil.PortForward(0, 3000, namespace, podName, false, stopCh, log)
if err != nil {
return errors.Wrap(err, "failed to start port forwarding")
}

go func() {
select {
case err := <-errChan:
if err != nil {
log.Error(err)
}
case <-stopCh:
}
}()

url := fmt.Sprintf("http://localhost:%d/api/v1/garbage-collect-images", localPort)

authSlug, err := auth.GetOrCreateAuthSlug(clientset, namespace)
if err != nil {
log.Info("Unable to authenticate to the Admin Console running in the %s namespace. Ensure you have read access to secrets in this namespace and try again.", namespace)
if v.GetBool("debug") {
return errors.Wrap(err, "failed to get kotsadm auth slug")
}
os.Exit(2) // not returning error here as we don't want to show the entire stack trace to normal users
}

newReq, err := http.NewRequest("POST", url, nil)
if err != nil {
return errors.Wrap(err, "failed to create request")
}
newReq.Header.Add("Content-Type", "application/json")
newReq.Header.Add("Authorization", authSlug)

resp, err := http.DefaultClient.Do(newReq)
if err != nil {
return errors.Wrap(err, "failed to check for updates")
}
defer resp.Body.Close()

b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "failed to read")
}

type Response struct {
Error string `json:"error"`
}
response := Response{}
if err = json.Unmarshal(b, &response); err != nil {
return errors.Wrapf(err, "failed to unmarshal server response: %s", b)
}

if response.Error != "" {
return errors.New(response.Error)
}

if resp.StatusCode != http.StatusOK {
return errors.Errorf("unexpected response from server %v: %s", resp.StatusCode, b)
}

log.ActionWithoutSpinner("Garbage collection has been triggered")

return nil
},
}

return cmd
}
1 change: 1 addition & 0 deletions pkg/api/version/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

type AppVersion struct {
KOTSKinds *kotsutil.KotsKinds `json:"kotsKinds"`
AppID string `json:"appId"`
Sequence int64 `json:"sequence"`
Status string `json:"status"`
CreatedOn time.Time `json:"createdOn"`
Expand Down
2 changes: 1 addition & 1 deletion pkg/handlers/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -516,7 +516,7 @@ func updateAppConfig(updateApp *apptypes.App, sequence int64, configGroups []kot
}
}

err = render.RenderDir(archiveDir, app, downstreams, registrySettings)
err = render.RenderDir(archiveDir, app, downstreams, registrySettings, createNewVersion)
if err != nil {
updateAppConfigResponse.Error = "failed to render archive directory"
return updateAppConfigResponse, err
Expand Down
147 changes: 141 additions & 6 deletions pkg/handlers/deploy.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package handlers

import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"
Expand All @@ -11,11 +13,16 @@ import (
"github.com/gorilla/mux"
"github.com/pkg/errors"
downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types"
versiontypes "github.com/replicatedhq/kots/pkg/api/version/types"
"github.com/replicatedhq/kots/pkg/app"
apptypes "github.com/replicatedhq/kots/pkg/app/types"
"github.com/replicatedhq/kots/pkg/kotsadm"
kotsadmobjects "github.com/replicatedhq/kots/pkg/kotsadm/objects"
kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types"
"github.com/replicatedhq/kots/pkg/kotsutil"
"github.com/replicatedhq/kots/pkg/logger"
"github.com/replicatedhq/kots/pkg/redact"
"github.com/replicatedhq/kots/pkg/registry"
"github.com/replicatedhq/kots/pkg/reporting"
"github.com/replicatedhq/kots/pkg/store"
storetypes "github.com/replicatedhq/kots/pkg/store/types"
Expand Down Expand Up @@ -124,30 +131,30 @@ func (h *Handler) DeployAppVersion(w http.ResponseWriter, r *http.Request) {
func (h *Handler) UpdateDeployResult(w http.ResponseWriter, r *http.Request) {
auth, err := parseClusterAuthorization(r.Header.Get("Authorization"))
if err != nil {
logger.Error(err)
logger.Error(errors.Wrap(err, "failed to parse cluster auth"))
w.WriteHeader(http.StatusForbidden)
return
}

clusterID, err := store.GetStore().GetClusterIDFromDeployToken(auth.Password)
if err != nil {
logger.Error(err)
logger.Error(errors.Wrap(err, "failed to get cluster ID"))
w.WriteHeader(http.StatusForbidden)
return
}

updateDeployResultRequest := UpdateDeployResultRequest{}
err = json.NewDecoder(r.Body).Decode(&updateDeployResultRequest)
if err != nil {
logger.Error(err)
logger.Error(errors.Wrap(err, "failed to decode deploy result"))
w.WriteHeader(http.StatusInternalServerError)
return
}

// sequence really should be passed down to operator and returned from it
currentSequence, err := store.GetStore().GetCurrentSequence(updateDeployResultRequest.AppID, clusterID)
if err != nil {
logger.Error(err)
logger.Error(errors.Wrap(err, "failed to get current sequence"))
w.WriteHeader(http.StatusInternalServerError)
return
}
Expand All @@ -159,7 +166,7 @@ func (h *Handler) UpdateDeployResult(w http.ResponseWriter, r *http.Request) {

alreadySuccessful, err := store.GetStore().IsDownstreamDeploySuccessful(updateDeployResultRequest.AppID, clusterID, currentSequence)
if err != nil {
logger.Error(err)
logger.Error(errors.Wrap(err, "failed to check deploy successful"))
w.WriteHeader(http.StatusInternalServerError)
return
}
Expand All @@ -180,15 +187,143 @@ func (h *Handler) UpdateDeployResult(w http.ResponseWriter, r *http.Request) {
}
err = store.GetStore().UpdateDownstreamDeployStatus(updateDeployResultRequest.AppID, clusterID, currentSequence, updateDeployResultRequest.IsError, downstreamOutput)
if err != nil {
logger.Error(err)
logger.Error(errors.Wrap(err, "failed to update downstream deploy status"))
w.WriteHeader(http.StatusInternalServerError)
return
}

if !updateDeployResultRequest.IsError {
go func() {
err := deleteUnusedImages(updateDeployResultRequest.AppID)
if err != nil {
if _, ok := err.(appRollbackError); ok {
logger.Infof("not garbage collecting images because version allows rollbacks: %v", err)
} else {
logger.Infof("failed to delete unused images: %v", err)
}
}
}()
}

w.WriteHeader(http.StatusOK)
return
}

type appRollbackError struct {
AppID string
Sequence int64
}

func (e appRollbackError) Error() string {
return fmt.Sprintf("app:%s, version:%d", e.AppID, e.Sequence)
}

func deleteUnusedImages(appID string) error {
installParams, err := kotsutil.GetInstallationParams(kotsadmtypes.KotsadmConfigMap)
if err != nil {
return errors.Wrap(err, "failed to get app registry info")
}
if !installParams.EnableImageDeletion {
return nil
}

registrySettings, err := store.GetStore().GetRegistryDetailsForApp(appID)
if err != nil {
return errors.Wrap(err, "failed to get app registry info")
}

if registrySettings.IsReadOnly {
return nil
}

isKurl, err := kotsadm.IsKurl()
if err != nil {
return errors.Wrap(err, "failed to check kURL")
}

if !isKurl {
return nil
}

appIDs, err := store.GetStore().GetAppIDsFromRegistry(registrySettings.Hostname)
if err != nil {
return errors.Wrap(err, "failed to get apps with registry")
}

activeVersions := []*versiontypes.AppVersion{}
for _, appID := range appIDs {
downstreams, err := store.GetStore().ListDownstreamsForApp(appID)
if err != nil {
return errors.Wrap(err, "failed to list downstreams for app")
}

for _, d := range downstreams {
curSequence, err := store.GetStore().GetCurrentParentSequence(appID, d.ClusterID)
if err != nil {
return errors.Wrap(err, "failed to get current parent sequence")
}

curVersion, err := store.GetStore().GetAppVersion(appID, curSequence)
if err != nil {
return errors.Wrap(err, "failed to get app version")
}

activeVersions = append(activeVersions, curVersion)

laterVersions, err := store.GetStore().GetAppVersionsAfter(appID, curSequence)
if err != nil {
return errors.Wrapf(err, "failed to get versions after %d", curVersion.Sequence)
}
activeVersions = append(activeVersions, laterVersions...)
}
}

imagesDedup := map[string]struct{}{}
for _, version := range activeVersions {
if version == nil {
continue
}
if version.KOTSKinds == nil {
continue
}
if version.KOTSKinds.KotsApplication.Spec.AllowRollback {
return appRollbackError{AppID: version.AppID, Sequence: version.Sequence}
}
for _, i := range version.KOTSKinds.Installation.Spec.KnownImages {
imagesDedup[i.Image] = struct{}{}
}
}

usedImages := []string{}
for i, _ := range imagesDedup {
usedImages = append(usedImages, i)
}

if installParams.KotsadmRegistry != "" {
deployOptions := kotsadmtypes.DeployOptions{
// Minimal info needed to get the right image names
KotsadmOptions: kotsadmtypes.KotsadmOptions{
// TODO: OverrideVersion
OverrideRegistry: registrySettings.Hostname,
OverrideNamespace: registrySettings.Namespace,
Username: registrySettings.Username,
Password: registrySettings.Password,
},
}
kotsadmImages := kotsadmobjects.GetAdminConsoleImages(deployOptions)
for _, i := range kotsadmImages {
usedImages = append(usedImages, i)
}
}

err = registry.DeleteUnusedImages(context.Background(), registrySettings, usedImages)
if err != nil {
return errors.Wrap(err, "failed to delete unused images")
}

return nil
}

func createSupportBundleSpec(appID string, sequence int64, origin string, inCluster bool) error {
archivePath, err := ioutil.TempDir("", "kotsadm")
if err != nil {
Expand Down

0 comments on commit 6e728e6

Please sign in to comment.