Skip to content
Permalink
Browse files

add missing prune files

not sure what happened there wth... weird merge fail
  • Loading branch information...
tj committed Apr 11, 2018
1 parent eb6a3fb commit c699301be3ead83529f575dbef96f3c4b9eda070
Showing with 124 additions and 0 deletions.
  1. +33 −0 internal/cli/prune/prune.go
  2. +91 −0 platform/lambda/prune.go
@@ -0,0 +1,33 @@
package prune

import (
"github.com/pkg/errors"
"github.com/tj/kingpin"

"github.com/apex/up/internal/cli/root"
"github.com/apex/up/internal/stats"
)

func init() {
cmd := root.Command("prune", "Prune old S3 deployments.")

cmd.Example(`up prune`, "Prune and retain the most recent 60 versions.")
cmd.Example(`up prune --retain 15`, "Prune and retain the most recent 15 versions.")

versions := cmd.Flag("retain", "Number of versions to retain.").Short('r').Default("60").Int()

cmd.Action(func(_ *kingpin.ParseContext) error {
c, p, err := root.Init()
if err != nil {
return errors.Wrap(err, "initializing")
}

region := c.Regions[0]

stats.Track("Prune", map[string]interface{}{
"versions": *versions,
})

return p.Prune(region, *versions)
})
}
@@ -0,0 +1,91 @@
package lambda

import (
"sort"
"time"

"github.com/apex/log"
"github.com/apex/up/platform/event"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/pkg/errors"
)

// Prune implementation.
func (p *Platform) Prune(region string, versions int) error {
p.events.Emit("prune", nil)

if err := p.createRole(); err != nil {
return errors.Wrap(err, "creating iam role")
}

s := s3.New(session.New(aws.NewConfig().WithRegion(region)))
b := aws.String(p.getS3BucketName(region))
prefix := p.config.Name + "/"

params := &s3.ListObjectsInput{
Bucket: b,
Prefix: &prefix,
}

start := time.Now()
var objects []*s3.Object
var count int
var size int64

// we have to fetch them all here, because they were using random unsorted keys
err := s.ListObjectsPages(params, func(page *s3.ListObjectsOutput, lastPage bool) bool {
for _, o := range page.Contents {
objects = append(objects, o)
}
return *page.IsTruncated
})

if err != nil {
return errors.Wrap(err, "listing s3 objects")
}

// sort by time descending
sort.Slice(objects, func(i int, j int) bool {
a := objects[i]
b := objects[j]
return (*b).LastModified.Before(*a.LastModified)
})

// remove old versions
for i, o := range objects {
ctx := log.WithFields(log.Fields{
"index": i,
"key": *o.Key,
"size": *o.Size,
"last_modified": *o.LastModified,
})

if i < versions {
ctx.Debug("retain")
continue
}

ctx.Debug("remove")
size += *o.Size
count++

_, err := s.DeleteObject(&s3.DeleteObjectInput{
Bucket: b,
Key: o.Key,
})

if err != nil {
return errors.Wrap(err, "removing object")
}
}

p.events.Emit("prune.complete", event.Fields{
"duration": time.Since(start),
"size": size,
"count": count,
})

return nil
}

0 comments on commit c699301

Please sign in to comment.
You can’t perform that action at this time.