From 45701dbb31e274cb2b3063884b1f6de64f924b4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Tue, 19 Jun 2018 18:18:35 +0200 Subject: [PATCH] Add CloudFront cache invalidation Fixes #31 --- README.md | 40 +++++--- go.mod | 17 ++++ go.sum | 35 +++++++ lib/cloudfront.go | 201 +++++++++++++++++++++++++++++++++++++++++ lib/cloudfront_test.go | 91 +++++++++++++++++++ lib/config.go | 4 + lib/config_test.go | 2 + lib/deployer.go | 54 +++++++++-- lib/deployer_test.go | 4 + lib/files.go | 19 +++- lib/files_test.go | 2 +- lib/s3.go | 64 ++++--------- lib/session.go | 63 +++++++++++++ lib/store.go | 44 ++++++++- 14 files changed, 564 insertions(+), 76 deletions(-) create mode 100644 lib/cloudfront.go create mode 100644 lib/cloudfront_test.go create mode 100644 lib/session.go diff --git a/README.md b/README.md index a7f800b..f28182d 100644 --- a/README.md +++ b/README.md @@ -25,33 +25,35 @@ Note that `s3deploy` is a perfect tool to use with a continuous integration tool ```bash Usage of s3deploy: - -V print version and exit + -V print version and exit -bucket string - destination bucket name on AWS + destination bucket name on AWS -config string - optional config file (default ".s3deploy.yml") + optional config file (default ".s3deploy.yml") + -distribution-id string + optional CDN distribution ID for cache invalidation -force - upload even if the etags match - -h help + upload even if the etags match + -h help -key string - access key ID for AWS + access key ID for AWS -max-delete int - maximum number of files to delete per deploy (default 256) + maximum number of files to delete per deploy (default 256) -path string - optional bucket sub path + optional bucket sub path -quiet - enable silent mode + enable silent mode -region string - name of AWS region + name of AWS region -secret string - secret access key for AWS + secret access key for AWS -source string - path of files to upload (default ".") + path of files to upload (default ".") -try - trial run, no remote updates - -v enable verbose logging + trial run, no remote updates + -v enable verbose logging -workers int - number of workers to upload files (default -1) + number of workers to upload files (default -1) ``` ### Notes @@ -128,6 +130,14 @@ routes: ``` Replace with your own. + +## CloudFront CDN Cache Invalidation + +If you have configured CloudFront CDN in front of your S3 bucket, you can supply the `distribution-id` as a flag. This will make sure to invalidate the cache for the updated files after the deployment to S3. Note that the AWS user must have the needed access rights. + +Note that CloudFront allows [1,000 paths per month at no charge](https://aws.amazon.com/blogs/aws/simplified-multiple-object-invalidation-for-amazon-cloudfront/), so S3deploy tries to be smart about the invalidation strategy; we try to reduce the number of paths to 8. If that isn't possible, we will fall back to a full invalidation, e.g. "/*". + +TODO(bep) IAM Policy example. ## Background Information diff --git a/go.mod b/go.mod index 341ece8..1fc60b5 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,26 @@ module github.com/bep/s3deploy require ( github.com/aws/aws-sdk-go v1.15.27 + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 + github.com/fsnotify/fsnotify v1.4.7 // indirect + github.com/gopherjs/gopherjs v0.0.0-20181003052733-bf5fc7d38140 // indirect + github.com/jtolds/gls v4.2.1+incompatible // indirect + github.com/kisielk/gotool v1.0.0 // indirect + github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86 // indirect + github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 // indirect + github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect + github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a // indirect + github.com/spf13/cobra v0.0.3 // indirect + github.com/spf13/pflag v1.0.3 // indirect + github.com/stretchr/testify v1.2.2 + golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4 // indirect golang.org/x/net v0.0.0-20180826012351-8a410e7b638d // indirect golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f + golang.org/x/sys v0.0.0-20181003145944-af653ce8b74f // indirect golang.org/x/text v0.3.0 + golang.org/x/tools v0.0.0-20181003164445-c930a8531daf // indirect gopkg.in/yaml.v2 v2.2.1 ) diff --git a/go.sum b/go.sum index 2bfa02e..3272a4d 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,52 @@ github.com/aws/aws-sdk-go v1.15.27 h1:i75BxN4Es/8rTVQbEKAP1WCiIhhz635xTNeDdZJRAXQ= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 h1:tFh1tRc4CA31yP6qDcu+Trax5wW5GuMxvkIba07qVLY= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-ini/ini v1.25.4 h1:Mujh4R/dH6YL8bxuISne3xX2+qcQ9p0IxKAP6ExWoUo= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/gopherjs/gopherjs v0.0.0-20181003052733-bf5fc7d38140 h1:xUNgm/qAe2iOwkqf230y9I1zOlqgq0gjOJ3RcIIJIoQ= +github.com/gopherjs/gopherjs v0.0.0-20181003052733-bf5fc7d38140/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86 h1:D6paGObi5Wud7xg83MaEFyjxQB1W5bz5d0IFppr+ymk= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab h1:eFXv9Nu1lGbrNbj619aWwZfVF5HBrm9Plte8aNptuTI= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 h1:SWV2fHctRpRrp49VXJ6UZja7gU9QLHwRpIPBN89SKEo= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbmil4Ui/dDdFBExb7/cmkNjyX5F97oglmvCDo= +github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4 h1:Vk3wNqEZwyGyei9yq5ekj7frek2u7HUfffJ1/opblzc= +golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20181003145944-af653ce8b74f h1:zAtpFwFDtnvBWPPelq8CSiqRN1wrIzMUk9dwzbpjpNM= +golang.org/x/sys v0.0.0-20181003145944-af653ce8b74f/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20181003164445-c930a8531daf h1:5wIhpNrul45Er8vIy8iqSCUsCEQH7G5YtJ4CfOi4Fk0= +golang.org/x/tools v0.0.0-20181003164445-c930a8531daf/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/lib/cloudfront.go b/lib/cloudfront.go new file mode 100644 index 0000000..7b61f3b --- /dev/null +++ b/lib/cloudfront.go @@ -0,0 +1,201 @@ +// Copyright © 2018 Bjørn Erik Pedersen . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package lib + +import ( + "errors" + "path" + "sort" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudfront" +) + +var _ remoteCDN = (*cloudFrontClient)(nil) + +type cloudFrontClient struct { + // The CloudFront distribution ID + distributionID string + + // Will invalidate the entire cache, e.g. "/*" + force bool + bucketPath string + + logger printer + cf *cloudfront.CloudFront +} + +func newCloudFrontClient( + sess *session.Session, + logger printer, + cfg Config) (*cloudFrontClient, error) { + if cfg.CDNDistributionID == "" { + return nil, errors.New("must provide a distribution ID") + } + return &cloudFrontClient{ + distributionID: cfg.CDNDistributionID, + force: cfg.Force, + bucketPath: cfg.BucketPath, + logger: logger, + cf: cloudfront.New(sess), + }, nil +} + +func (c *cloudFrontClient) InvalidateCDNCache(paths ...string) error { + if len(paths) == 0 { + return nil + } + + dcfg, err := c.cf.GetDistribution(&cloudfront.GetDistributionInput{ + Id: &c.distributionID, + }) + if err != nil { + return err + } + + originPath := *dcfg.Distribution.DistributionConfig.Origins.Items[0].OriginPath + var root string + if originPath != "" || c.bucketPath != "" { + bucket := strings.TrimPrefix(c.bucketPath, "/") + origin := strings.TrimPrefix(originPath, "/") + root = strings.TrimPrefix(bucket, origin) + subPath := strings.TrimPrefix(origin, bucket) + for i, p := range paths { + paths[i] = strings.TrimPrefix(p, subPath) + } + } + + // This will try to reduce the number of invaldation paths to maximum 8. + // If that isn't possible it will fall back to a full invalidation, e.g. "/*". + // CloudFront allows 1000 free invalidations per month. After that they + // cost money, so we want to keep this down. + paths = normalizeInvalidationPaths(root, 8, c.force, paths...) + + if len(paths) > 10 { + c.logger.Printf("Create CloudFront invalidation request for %d paths", len(paths)) + } else { + c.logger.Printf("Create CloudFront invalidation request for %v", paths) + } + + in := &cloudfront.CreateInvalidationInput{ + DistributionId: &c.distributionID, + InvalidationBatch: c.pathsToInvalidationBatch(time.Now().Format("20060102150405"), paths...), + } + + _, err = c.cf.CreateInvalidation( + in, + ) + + return err +} + +func (*cloudFrontClient) pathsToInvalidationBatch(ref string, paths ...string) *cloudfront.InvalidationBatch { + batch := &cloudfront.InvalidationBatch{ + CallerReference: &ref, + } + cfpaths := &cloudfront.Paths{} + for _, p := range paths { + cfpaths.Items = append(cfpaths.Items, aws.String(p)) + } + + qty := int64(len(paths)) + cfpaths.Quantity = &qty + batch.SetPaths(cfpaths) + return batch +} + +// For path rules, see https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html +func normalizeInvalidationPaths( + root string, + threshold int, + force bool, + paths ...string) []string { + + if !strings.HasPrefix(root, "/") { + root = "/" + root + } + + matchAll := path.Join(root, "*/") + clearAll := []string{matchAll} + + if force { + return clearAll + } + + var normalized []string + var maxlevels int + + for _, p := range paths { + p = path.Clean(p) + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + levels := strings.Count(p, "/") + if levels > maxlevels { + maxlevels = levels + } + + if strings.HasSuffix(p, "index.html") { + dir := path.Dir(p) + if !strings.HasSuffix(dir, "/") { + dir += "/" + } + normalized = append(normalized, dir) + } else { + normalized = append(normalized, p) + } + } + + normalized = uniqueStrings(uniqueStrings(normalized)) + sort.Strings(normalized) + + if len(normalized) > threshold { + normalized = uniqueStrings(normalized) + if len(normalized) > threshold { + for k := maxlevels; k > 0; k-- { + for i, p := range normalized { + if strings.Count(p, "/") > k { + parts := strings.Split(strings.TrimPrefix(path.Dir(p), "/"), "/") + normalized[i] = "/" + path.Join(parts[0:len(parts)-k+1]...) + "/*" + } + } + normalized = uniqueStrings(normalized) + if len(normalized) <= threshold { + break + } + } + + if len(normalized) > threshold { + // Give up. + return clearAll + } + } + } + + for _, pattern := range normalized { + if pattern == matchAll { + return clearAll + } + } + + return normalized +} + +func uniqueStrings(s []string) []string { + var unique []string + set := map[string]interface{}{} + for _, val := range s { + if _, ok := set[val]; !ok { + unique = append(unique, val) + set[val] = val + } + } + return unique +} diff --git a/lib/cloudfront_test.go b/lib/cloudfront_test.go new file mode 100644 index 0000000..1a1123b --- /dev/null +++ b/lib/cloudfront_test.go @@ -0,0 +1,91 @@ +// Copyright © 2018 Bjørn Erik Pedersen . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package lib + +import ( + "fmt" + "io/ioutil" + "path" + "testing" + + "github.com/aws/aws-sdk-go/awstesting/mock" + "github.com/stretchr/testify/require" +) + +func TestReduceInvalidationPaths(t *testing.T) { + assert := require.New(t) + + assert.Equal([]string{"/root/"}, normalizeInvalidationPaths("root", 5, false, "/root/index.html")) + assert.Equal([]string{"/"}, normalizeInvalidationPaths("", 5, false, "/index.html")) + assert.Equal([]string{"/*"}, normalizeInvalidationPaths("", 5, true, "/a", "/b")) + assert.Equal([]string{"/root/*"}, normalizeInvalidationPaths("root", 5, true, "/a", "/b")) + + rootPlusMany := append([]string{"/index.html", "/styles.css"}, createFiles("css", false, 20)...) + normalized := normalizeInvalidationPaths("", 5, false, rootPlusMany...) + assert.Equal(3, len(normalized)) + assert.Equal([]string{"/", "/css/*", "/styles.css"}, normalized) + + rootPlusManyInDifferentFolders := append([]string{"/index.html", "/styles.css"}, createFiles("css", true, 20)...) + assert.Equal([]string{"/*"}, normalizeInvalidationPaths("", 5, false, rootPlusManyInDifferentFolders...)) + + rootPlusManyInDifferentFoldersNested := append([]string{"/index.html", "/styles.css"}, createFiles("blog", false, 10)...) + rootPlusManyInDifferentFoldersNested = append(rootPlusManyInDifferentFoldersNested, createFiles("blog/l1", false, 10)...) + rootPlusManyInDifferentFoldersNested = append(rootPlusManyInDifferentFoldersNested, createFiles("blog/l1/l2/l3/l5", false, 10)...) + rootPlusManyInDifferentFoldersNested = append(rootPlusManyInDifferentFoldersNested, createFiles("blog/l1/l3", false, 10)...) + rootPlusManyInDifferentFoldersNested = append(rootPlusManyInDifferentFoldersNested, createFiles("about/l1", true, 10)...) + rootPlusManyInDifferentFoldersNested = append(rootPlusManyInDifferentFoldersNested, createFiles("about/l1/l2/l3", false, 10)...) + + // avoid situations where many changes in some HTML template triggers update in /images and similar + normalized = normalizeInvalidationPaths("", 5, false, rootPlusManyInDifferentFoldersNested...) + assert.Equal(4, len(normalized)) + assert.Equal([]string{"/", "/about/*", "/blog/*", "/styles.css"}, normalized) + + hugoChanges := []string{"/hugoscss/categories/index.html", "/hugoscss/index.html", "/hugoscss/tags/index.html", "/hugoscss/post/index.html", "/hugoscss/post/hello-scss/index.html", "/hugoscss/styles/main.min.36816b22057425f8a5f66b73918446b0cd793c0c6125406c285948f507599d1e.css"} + normalized = normalizeInvalidationPaths("/hugoscss", 3, false, hugoChanges...) + //assert.Equal(1, len(normalized)) + assert.Equal([]string{"/hugoscss/*"}, normalized) +} + +func TestPathsToInvalidationBatch(t *testing.T) { + assert := require.New(t) + + var client *cloudFrontClient + + batch := client.pathsToInvalidationBatch("myref", "/path1/", "/path2/") + + assert.NotNil(batch) + assert.Equal("myref", *batch.CallerReference) + assert.Equal(2, int(*batch.Paths.Quantity)) +} + +func TestNewCloudFrontClient(t *testing.T) { + assert := require.New(t) + s := mock.Session + c, err := newCloudFrontClient(s, newPrinter(ioutil.Discard), Config{ + CDNDistributionID: "12345", + Force: true, + BucketPath: "/mypath", + }) + assert.NoError(err) + assert.NotNil(c) + assert.Equal("12345", c.distributionID) + assert.Equal("/mypath", c.bucketPath) + assert.Equal(true, c.force) +} + +func createFiles(root string, differentFolders bool, num int) []string { + files := make([]string, num) + + for i := 0; i < num; i++ { + nroot := root + if differentFolders { + nroot = fmt.Sprintf("%s-%d", root, i) + } + files[i] = path.Join(nroot, fmt.Sprintf("file%d.css", i+1)) + } + + return files +} diff --git a/lib/config.go b/lib/config.go index 25d6082..93b0d02 100644 --- a/lib/config.go +++ b/lib/config.go @@ -27,6 +27,9 @@ type Config struct { BucketPath string RegionName string + // When set, will invalidate the CDN cache for the updated files. + CDNDistributionID string + // Optional configFile ConfigFile string @@ -62,6 +65,7 @@ func flagsToConfig(f *flag.FlagSet) (*Config, error) { f.StringVar(&cfg.BucketName, "bucket", "", "destination bucket name on AWS") f.StringVar(&cfg.BucketPath, "path", "", "optional bucket sub path") f.StringVar(&cfg.SourcePath, "source", ".", "path of files to upload") + f.StringVar(&cfg.CDNDistributionID, "distribution-id", "", "optional CDN distribution ID for cache invalidation") f.StringVar(&cfg.ConfigFile, "config", ".s3deploy.yml", "optional config file") f.IntVar(&cfg.MaxDelete, "max-delete", 256, "maximum number of files to delete per deploy") f.BoolVar(&cfg.Force, "force", false, "upload even if the etags match") diff --git a/lib/config_test.go b/lib/config_test.go index b8da29e..b2c34f8 100644 --- a/lib/config_test.go +++ b/lib/config_test.go @@ -26,6 +26,7 @@ func TestFlagsToConfig(t *testing.T) { "-quiet=true", "-region=myregion", "-source=mysource", + "-distribution-id=mydistro", "-try=true", } @@ -43,5 +44,6 @@ func TestFlagsToConfig(t *testing.T) { assert.Equal("mysource", cfg.SourcePath) assert.Equal(true, cfg.Try) assert.Equal("myregion", cfg.RegionName) + assert.Equal("mydistro", cfg.CDNDistributionID) } diff --git a/lib/deployer.go b/lib/deployer.go index fadb012..f861d0d 100644 --- a/lib/deployer.go +++ b/lib/deployer.go @@ -40,7 +40,7 @@ type Deployer struct { // Verbose output. outv io.Writer // Regular output. - out io.Writer + printer store remoteStore } @@ -77,7 +77,7 @@ func Deploy(cfg *Config) (DeployStats, error) { var d = &Deployer{ g: g, outv: outv, - out: out, + printer: newPrinter(out), filesToUpload: make(chan *osFile), cfg: cfg, stats: &DeployStats{}} @@ -95,16 +95,16 @@ func Deploy(cfg *Config) (DeployStats, error) { baseStore := d.cfg.baseStore if baseStore == nil { - baseStore, err = newRemoteStore(*d.cfg) + baseStore, err = newRemoteStore(*d.cfg, d) if err != nil { return *d.stats, err } } if d.cfg.Try { baseStore = newNoUpdateStore(baseStore) - fmt.Fprintln(d.out, "This is a trial run, with no remote updates.") + d.Println("This is a trial run, with no remote updates.") } - d.store = newStore(baseStore) + d.store = newStore(*d.cfg, baseStore) for i := 0; i < numberOfWorkers; i++ { g.Go(func() error { @@ -133,11 +133,38 @@ func Deploy(cfg *Config) (DeployStats, error) { withDeleteStats(d.stats), withMaxDelete(d.cfg.MaxDelete)) + if err == nil { + err = d.store.Finalize() + } + return *d.stats, err } -func (d *Deployer) enqueueUpload(ctx context.Context, f *osFile, reason string) { - fmt.Fprintf(d.out, "%s (%s) %s ", f.relPath, reason, up) +type printer interface { + Println(a ...interface{}) (n int, err error) + Printf(format string, a ...interface{}) (n int, err error) +} + +type print struct { + out io.Writer +} + +func newPrinter(out io.Writer) printer { + return print{out: out} +} + +func (p print) Println(a ...interface{}) (n int, err error) { + return fmt.Fprintln(p.out, a...) +} + +func (p print) Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(p.out, format, a...) +} + +func (d *Deployer) enqueueUpload(ctx context.Context, f *osFile, reason uploadReason) { + d.Printf("%s (%s) %s ", f.relPath, reason, up) + // TODO(bep) + f.reason = reason select { case <-ctx.Done(): case d.filesToUpload <- f: @@ -154,6 +181,15 @@ func (d *Deployer) enqueueDelete(key string) { d.filesToDelete = append(d.filesToDelete, key) } +type uploadReason string + +const ( + reasonNotFound uploadReason = "not found" + reasonForce uploadReason = "force" + reasonSize uploadReason = "size" + reasonETag uploadReason = "ETag" +) + // plan figures out which files need to be uploaded. func (d *Deployer) plan(ctx context.Context) error { remoteFiles, err := d.store.FileMap() @@ -170,7 +206,7 @@ func (d *Deployer) plan(ctx context.Context) error { for f := range localFiles { // default: upload because local file not found on remote. up := true - reason := "not found" + reason := reasonNotFound bucketPath := f.relPath if d.cfg.BucketPath != "" { @@ -180,7 +216,7 @@ func (d *Deployer) plan(ctx context.Context) error { if remoteFile, ok := remoteFiles[bucketPath]; ok { if d.cfg.Force { up = true - reason = "force" + reason = reasonForce } else { up, reason = f.shouldThisReplace(remoteFile) } diff --git a/lib/deployer_test.go b/lib/deployer_test.go index a1a4ee7..b8794bf 100644 --- a/lib/deployer_test.go +++ b/lib/deployer_test.go @@ -289,3 +289,7 @@ func (s *testStore) DeleteObjects(ctx context.Context, keys []string, opts ...op } return nil } + +func (s *testStore) Finalize() error { + return nil +} diff --git a/lib/files.go b/lib/files.go index 18e1fd1..7595d6a 100644 --- a/lib/files.go +++ b/lib/files.go @@ -27,6 +27,7 @@ import ( var ( _ file = (*osFile)(nil) _ localFile = (*osFile)(nil) + _ reasoner = (*osFile)(nil) ) type file interface { @@ -36,9 +37,13 @@ type file interface { Size() int64 } +type reasoner interface { + UploadReason() uploadReason +} + type localFile interface { file - shouldThisReplace(other file) (bool, string) + shouldThisReplace(other file) (bool, uploadReason) // Content returns the content to be stored remotely. If this file // configured to be gzipped, then that is what you get. @@ -54,6 +59,8 @@ type osFile struct { // of the target file store. targetRoot string + reason uploadReason + absPath string size int64 @@ -74,6 +81,10 @@ func (f *osFile) Key() string { return f.relPath } +func (f *osFile) UploadReason() uploadReason { + return f.reason +} + func (f *osFile) ETag() string { f.etagInit.Do(func() { var err error @@ -152,13 +163,13 @@ func detectContentTypeFromContent(b []byte) string { } -func (f *osFile) shouldThisReplace(other file) (bool, string) { +func (f *osFile) shouldThisReplace(other file) (bool, uploadReason) { if f.Size() != other.Size() { - return true, "size" + return true, reasonSize } if f.ETag() != other.ETag() { - return true, "ETag" + return true, reasonETag } return false, "" diff --git a/lib/files_test.go b/lib/files_test.go index 3901779..c04ae71 100644 --- a/lib/files_test.go +++ b/lib/files_test.go @@ -51,7 +51,7 @@ func TestShouldThisReplace(t *testing.T) { message := fmt.Sprintf("Test %d", i) b, reason := of.shouldThisReplace(test.testFile) assert.Equal(test.expect, b, message) - assert.Equal(test.expectReason, reason) + assert.Equal(uploadReason(test.expectReason), reason) } } diff --git a/lib/s3.go b/lib/s3.go index 0adc34e..f9bcb9a 100644 --- a/lib/s3.go +++ b/lib/s3.go @@ -7,18 +7,15 @@ package lib import ( "context" - "errors" - "os" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" ) var ( _ remoteStore = (*s3Store)(nil) + _ remoteCDN = (*s3Store)(nil) _ file = (*s3File)(nil) ) @@ -26,6 +23,8 @@ type s3Store struct { bucket string r routes svc *s3.S3 + + cfc *cloudFrontClient } type s3File struct { @@ -44,43 +43,23 @@ func (f *s3File) Size() int64 { return *f.o.Size } -func newRemoteStore(cfg Config) (remoteStore, error) { +func newRemoteStore(cfg Config, logger printer) (*s3Store, error) { var s *s3Store + var cfc *cloudFrontClient - creds, err := s.createCredentials(cfg) + sess, err := newSession(cfg) if err != nil { return nil, err } - region := new(string) - - if cfg.RegionName != "" { - region = &cfg.RegionName - } - - sess, err := session.NewSessionWithOptions(session.Options{ - Config: aws.Config{ - // The region may be set in global config. See SharedConfigState. - Region: region, - - // The credentials object to use when signing requests. - // Uses -key and -secret from command line if provided. - // Defaults to a chain of credential providers to search for - // credentials in environment variables, shared credential file, - // and EC2 Instance Roles. - Credentials: creds, - }, - // This is the default in session.NewSession, but let us be explicit. - // The end user can override this with AWS_SDK_LOAD_CONFIG=1. - // See https://docs.aws.amazon.com/sdk-for-go/api/aws/session/#hdr-Sessions_from_Shared_Config - SharedConfigState: session.SharedConfigStateFromEnv, - }) - - if err != nil { - return nil, err + if cfg.CDNDistributionID != "" { + cfc, err = newCloudFrontClient(sess, logger, cfg) + if err != nil { + return nil, err + } } - s = &s3Store{svc: s3.New(sess), bucket: cfg.BucketName, r: cfg.conf.Routes} + s = &s3Store{svc: s3.New(sess), cfc: cfc, bucket: cfg.BucketName, r: cfg.conf.Routes} return s, nil @@ -140,18 +119,13 @@ func (s *s3Store) DeleteObjects(ctx context.Context, keys []string, opts ...opOp return err } -func (s *s3Store) createCredentials(cfg Config) (*credentials.Credentials, error) { - accessKey, secretKey := cfg.AccessKey, cfg.SecretKey - - if accessKey != "" && secretKey != "" { - return credentials.NewStaticCredentials(accessKey, secretKey, os.Getenv("AWS_SESSION_TOKEN")), nil - } +func (s *s3Store) Finalize() error { + return nil +} - if accessKey != "" || secretKey != "" { - // provided one but not both - return nil, errors.New("AWS key and secret are required") +func (s *s3Store) InvalidateCDNCache(paths ...string) error { + if s.cfc == nil { + return nil } - - // Use AWS default - return nil, nil + return s.cfc.InvalidateCDNCache(paths...) } diff --git a/lib/session.go b/lib/session.go new file mode 100644 index 0000000..3749e7c --- /dev/null +++ b/lib/session.go @@ -0,0 +1,63 @@ +// Copyright © 2018 Bjørn Erik Pedersen . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package lib + +import ( + "errors" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" +) + +func newSession(cfg Config) (*session.Session, error) { + creds, err := createCredentials(cfg) + if err != nil { + return nil, err + } + + region := new(string) + + if cfg.RegionName != "" { + region = &cfg.RegionName + } + + return session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + // The region may be set in global config. See SharedConfigState. + Region: region, + + // The credentials object to use when signing requests. + // Uses -key and -secret from command line if provided. + // Defaults to a chain of credential providers to search for + // credentials in environment variables, shared credential file, + // and EC2 Instance Roles. + Credentials: creds, + }, + // This is the default in session.NewSession, but let us be explicit. + // The end user can override this with AWS_SDK_LOAD_CONFIG=1. + // See https://docs.aws.amazon.com/sdk-for-go/api/aws/session/#hdr-Sessions_from_Shared_Config + SharedConfigState: session.SharedConfigStateFromEnv, + }) + +} + +func createCredentials(cfg Config) (*credentials.Credentials, error) { + accessKey, secretKey := cfg.AccessKey, cfg.SecretKey + + if accessKey != "" && secretKey != "" { + return credentials.NewStaticCredentials(accessKey, secretKey, os.Getenv("AWS_SESSION_TOKEN")), nil + } + + if accessKey != "" || secretKey != "" { + // provided one but not both + return nil, errors.New("AWS key and secret are required") + } + + // Use AWS default + return nil, nil +} diff --git a/lib/store.go b/lib/store.go index 1f6fae5..30198d9 100644 --- a/lib/store.go +++ b/lib/store.go @@ -7,31 +7,56 @@ package lib import ( "context" + "fmt" + "sync" "sync/atomic" ) var ( _ remoteStore = (*store)(nil) + _ remoteCDN = (*noUpdateStore)(nil) ) type remoteStore interface { FileMap(opts ...opOption) (map[string]file, error) Put(ctx context.Context, f localFile, opts ...opOption) error DeleteObjects(ctx context.Context, keys []string, opts ...opOption) error + Finalize() error +} + +type remoteCDN interface { + InvalidateCDNCache(paths ...string) error } type store struct { + cfg Config delegate remoteStore + + changedKeys []string + changedMu sync.Mutex +} + +func newStore(cfg Config, s remoteStore) remoteStore { + return &store{cfg: cfg, delegate: s} } -func newStore(s remoteStore) remoteStore { - return &store{delegate: s} +func (s *store) trackChanged(keys ...string) { + s.changedMu.Lock() + defer s.changedMu.Unlock() + s.changedKeys = append(s.changedKeys, keys...) } func (s *store) FileMap(opts ...opOption) (map[string]file, error) { return s.delegate.FileMap(opts...) } +func (s *store) Finalize() error { + if cdn, ok := s.delegate.(remoteCDN); ok { + return cdn.InvalidateCDNCache(s.changedKeys...) + } + return nil +} + func (s *store) Put(ctx context.Context, f localFile, opts ...opOption) error { conf, err := optsToConfig(opts...) if err != nil { @@ -41,6 +66,7 @@ func (s *store) Put(ctx context.Context, f localFile, opts ...opOption) error { err = s.delegate.Put(ctx, f, opts...) if err == nil { + s.trackChanged(f.Key()) conf.statsCollector(1, 0) } @@ -79,6 +105,7 @@ func (s *store) DeleteObjects(ctx context.Context, keys []string, opts ...opOpti return err } + s.trackChanged(keyChunk...) deleted += len(keyChunk) conf.statsCollector(deleted, 0) if deleted >= conf.maxDelete { @@ -113,6 +140,19 @@ func (s *noUpdateStore) DeleteObjects(ctx context.Context, keys []string, opts . return nil } +func (s *noUpdateStore) Finalize() error { + if s.readOps != nil { + return s.readOps.Finalize() + } + return nil +} + +func (s *noUpdateStore) InvalidateCDNCache(paths ...string) error { + fmt.Println("\nInvalidate CDN:", paths) + return nil + +} + type opConfig struct { maxDelete int statsCollector func(handled, skipped int)