diff --git a/README.md b/README.md index 42de899a..f651090d 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,20 @@ To work with this repo by it's name, first you need to add it using native helm $ helm repo add mynewrepo s3://bucket-name/charts +To store the repository in S3, but to publish it at a different URI you can specify the +URI when initializing. This will rewrite the URIs in the repository index so that they are +downloadable via the published URI. This is useful in case you want to keep the S3 +bucket private and expose the repository over HTTP(S) via CloudFront or a web server. +The delete and push commands will automatically take into account this publish URI when +updating the index. + + $ helm s3 init s3://bucket-name/charts --publish https://charts.my-company.tld + +The repository owner can continue to use the S3 URIs while the repository user can +consume the repository via its published URI: + + $ helm repo add publishedrepo https://charts.my-company.tld + ### Push Now you can push your chart to this repo: @@ -166,6 +180,10 @@ the index in accordance with the charts in the repository. $ helm s3 reindex mynewrepo +Alternatively, you can specify `--publish uri` to reindex an existing repository with a +different published URI. If you don't specify a publish URI, it will reset the repository +back to using S3 URIs. + ## Uninstall $ helm plugin remove s3 diff --git a/cmd/helms3/delete.go b/cmd/helms3/delete.go index beb435ca..48165186 100644 --- a/cmd/helms3/delete.go +++ b/cmd/helms3/delete.go @@ -3,8 +3,10 @@ package main import ( "context" "fmt" + "strings" "github.com/pkg/errors" + "k8s.io/helm/pkg/repo" "github.com/hypnoglow/helm-s3/internal/awss3" "github.com/hypnoglow/helm-s3/internal/awsutil" @@ -29,7 +31,8 @@ func (act deleteAction) Run(ctx context.Context) error { storage := awss3.New(sess) // Fetch current index. - b, err := storage.FetchRaw(ctx, repoEntry.URL+"/index.yaml") + indexURI := repoEntry.URL + "/index.yaml" + b, err := storage.FetchRaw(ctx, indexURI) if err != nil { return errors.WithMessage(err, "fetch current repo index") } @@ -56,16 +59,35 @@ func (act deleteAction) Run(ctx context.Context) error { if len(chartVersion.URLs) < 1 { return fmt.Errorf("chart version index record has no urls") } - uri := chartVersion.URLs[0] + + metadata, err := storage.GetMetadata(ctx, indexURI) + if err != nil { + return err + } + + publishURI := metadata[strings.Title(awss3.MetaPublishURI)] + uri := fmt.Sprintf("%s/%s-%s.tgz", repoEntry.URL, chartVersion.Metadata.Name, chartVersion.Metadata.Version) if err := storage.Delete(ctx, uri); err != nil { return errors.WithMessage(err, "delete chart file from s3") } - if err := storage.PutIndex(ctx, repoEntry.URL, act.acl, idxReader); err != nil { + if err := storage.PutIndex(ctx, repoEntry.URL, publishURI, act.acl, idxReader); err != nil { return errors.WithMessage(err, "upload new index to s3") } - if err := idx.WriteFile(repoEntry.Cache, 0644); err != nil { + localIndexFile, err := repo.LoadIndexFile(repoEntry.Cache) + if err != nil { + return err + } + + localIndex := &index.Index{IndexFile: localIndexFile} + + _, err = localIndex.Delete(act.name, act.version) + if err != nil { + return errors.WithMessage(err, "delete chart from local index") + } + + if err := localIndex.WriteFile(repoEntry.Cache, 0644); err != nil { return errors.WithMessage(err, "update local index") } diff --git a/cmd/helms3/init.go b/cmd/helms3/init.go index a433de43..c9171444 100644 --- a/cmd/helms3/init.go +++ b/cmd/helms3/init.go @@ -11,8 +11,9 @@ import ( ) type initAction struct { - uri string - acl string + uri string + publishURI string + acl string } func (act initAction) Run(ctx context.Context) error { @@ -27,7 +28,7 @@ func (act initAction) Run(ctx context.Context) error { } storage := awss3.New(sess) - if err := storage.PutIndex(ctx, act.uri, act.acl, r); err != nil { + if err := storage.PutIndex(ctx, act.uri, act.publishURI, act.acl, r); err != nil { return errors.WithMessage(err, "upload index to s3") } diff --git a/cmd/helms3/main.go b/cmd/helms3/main.go index a4f2bbd7..bbdcbd34 100644 --- a/cmd/helms3/main.go +++ b/cmd/helms3/main.go @@ -7,7 +7,7 @@ import ( "os" "time" - "gopkg.in/alecthomas/kingpin.v2" + kingpin "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -76,6 +76,9 @@ func main() { initURI := initCmd.Arg("uri", "URI of repository, e.g. s3://awesome-bucket/charts"). Required(). String() + initPublish := initCmd.Flag("publish", "The URI where the S3 bucket should be published"). + Default(""). + String() pushCmd := cli.Command(actionPush, "Push chart to the repository.") pushChartPath := pushCmd.Arg("chartPath", "Path to a chart, e.g. ./epicservice-0.5.1.tgz"). @@ -99,6 +102,9 @@ func main() { reindexTargetRepository := reindexCmd.Arg("repo", "Target repository to reindex"). Required(). String() + reindexPublish := reindexCmd.Flag("publish", "The URI where the S3 bucket should be published"). + Default(""). + String() deleteCmd := cli.Command(actionDelete, "Delete chart from the repository.").Alias("del") deleteChartName := deleteCmd.Arg("chartName", "Name of chart to delete"). @@ -125,8 +131,9 @@ func main() { case actionInit: act = initAction{ - uri: *initURI, - acl: *acl, + uri: *initURI, + publishURI: *initPublish, + acl: *acl, } defer fmt.Printf("Initialized empty repository at %s\n", *initURI) @@ -143,8 +150,9 @@ func main() { case actionReindex: act = reindexAction{ - repoName: *reindexTargetRepository, - acl: *acl, + repoName: *reindexTargetRepository, + publishURI: *reindexPublish, + acl: *acl, } defer fmt.Printf("Repository %s was successfully reindexed.\n", *reindexTargetRepository) diff --git a/cmd/helms3/push.go b/cmd/helms3/push.go index 556e9eb6..8d4948bb 100644 --- a/cmd/helms3/push.go +++ b/cmd/helms3/push.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/pkg/errors" "k8s.io/helm/pkg/chartutil" @@ -80,7 +81,8 @@ func (act pushAction) Run(ctx context.Context) error { return err } - if cachedIndex, err := repo.LoadIndexFile(repoEntry.Cache); err == nil { + cachedIndex, err := repo.LoadIndexFile(repoEntry.Cache) + if err == nil { // if cached index exists, check if the same chart version exists in it. if cachedIndex.Has(chart.Metadata.Name, chart.Metadata.Version) { if act.ignoreIfExists { @@ -137,7 +139,8 @@ func (act pushAction) Run(ctx context.Context) error { // Fetch current index, update it and upload it back. - b, err := storage.FetchRaw(ctx, repoEntry.URL+"/index.yaml") + indexURI := repoEntry.URL + "/index.yaml" + b, err := storage.FetchRaw(ctx, indexURI) if err != nil { return errors.WithMessage(err, "fetch current repo index") } @@ -147,7 +150,18 @@ func (act pushAction) Run(ctx context.Context) error { return errors.WithMessage(err, "load index from downloaded file") } - if err := idx.AddOrReplace(chart.GetMetadata(), fname, repoEntry.URL, hash); err != nil { + metadata, err := storage.GetMetadata(ctx, indexURI) + if err != nil { + return err + } + + publishURI := metadata[strings.Title(awss3.MetaPublishURI)] + uri := repoEntry.URL + if publishURI != "" { + uri = publishURI + } + + if err := idx.AddOrReplace(chart.GetMetadata(), fname, uri, hash); err != nil { return errors.WithMessage(err, "add/replace chart in the index") } idx.SortEntries() @@ -158,11 +172,17 @@ func (act pushAction) Run(ctx context.Context) error { } if !act.dryRun { - if err := storage.PutIndex(ctx, repoEntry.URL, act.acl, idxReader); err != nil { + if err := storage.PutIndex(ctx, repoEntry.URL, publishURI, act.acl, idxReader); err != nil { return errors.WithMessage(err, "upload index to s3") } - if err := idx.WriteFile(repoEntry.Cache, 0644); err != nil { + localIndex := &index.Index{IndexFile: cachedIndex} + if err := localIndex.AddOrReplace(chart.GetMetadata(), fname, repoEntry.URL, hash); err != nil { + return errors.WithMessage(err, "add/replace chart in the index") + } + localIndex.SortEntries() + + if err := localIndex.WriteFile(repoEntry.Cache, 0644); err != nil { return errors.WithMessage(err, "update local index") } } diff --git a/cmd/helms3/reindex.go b/cmd/helms3/reindex.go index e1acd62c..79289fa9 100644 --- a/cmd/helms3/reindex.go +++ b/cmd/helms3/reindex.go @@ -12,8 +12,9 @@ import ( ) type reindexAction struct { - repoName string - acl string + repoName string + publishURI string + acl string } func (act reindexAction) Run(ctx context.Context) error { @@ -30,11 +31,16 @@ func (act reindexAction) Run(ctx context.Context) error { items, errs := storage.Traverse(ctx, repoEntry.URL) + uri := repoEntry.URL + if act.publishURI != "" { + uri = act.publishURI + } + builtIndex := make(chan *index.Index, 1) go func() { idx := index.New() for item := range items { - idx.Add(item.Meta, item.Filename, repoEntry.URL, item.Hash) + idx.Add(item.Meta, item.Filename, uri, item.Hash) } idx.SortEntries() @@ -52,7 +58,7 @@ func (act reindexAction) Run(ctx context.Context) error { return errors.Wrap(err, "get index reader") } - if err := storage.PutIndex(ctx, repoEntry.URL, act.acl, r); err != nil { + if err := storage.PutIndex(ctx, repoEntry.URL, act.publishURI, act.acl, r); err != nil { return errors.Wrap(err, "upload index to the repository") } diff --git a/hack/integration-tests-local.sh b/hack/integration-tests-local.sh index e1275688..75994df7 100755 --- a/hack/integration-tests-local.sh +++ b/hack/integration-tests-local.sh @@ -30,9 +30,6 @@ go build -o bin/helms3 ./cmd/helms3 ## Test bash "$(dirname ${BASH_SOURCE[0]})/integration-tests.sh" -if [ $? -eq 0 ] ; then - echo -e "\nAll tests passed!" -fi ## Tear down diff --git a/hack/integration-tests.sh b/hack/integration-tests.sh index b2bde5b1..a8c2e918 100755 --- a/hack/integration-tests.sh +++ b/hack/integration-tests.sh @@ -2,145 +2,96 @@ set -euo pipefail set -x -# # Set up -# +BUCKET="test-bucket/charts" +CONTENT_TYPE="application/x-gzip" +MINIO="helm-s3-minio/${BUCKET}" +PUBLISH_URI="http://example.com/charts" +REPO="test-repo" +S3_URI="s3://${BUCKET}" +TEST_CASE="" + +function cleanup() { + rc=$? + set +x + rm -f postgresql-0.8.3.tgz + helm repo remove "${REPO}" &>/dev/null + + if [[ ${rc} -eq 0 ]]; then + echo -e "\nAll tests passed!" + else + echo -e "\nTest failed: ${TEST_CASE}" + fi +} + +trap cleanup EXIT # Prepare chart to play with. helm fetch stable/postgresql --version 0.8.3 - -# -# Test: init repo -# - -helm s3 init s3://test-bucket/charts -if [ $? -ne 0 ]; then - echo "Failed to initialize repo" - exit 1 -fi - -mc ls helm-s3-minio/test-bucket/charts/index.yaml -if [ $? -ne 0 ]; then - echo "Repository was not actually initialized" - exit 1 -fi - -helm repo add test-repo s3://test-bucket/charts -if [ $? -ne 0 ]; then - echo "Failed to add repo" - exit 1 -fi - -# -# Test: push chart -# - -helm s3 push postgresql-0.8.3.tgz test-repo -if [ $? -ne 0 ]; then - echo "Failed to push chart to repo" - exit 1 -fi - -mc ls helm-s3-minio/test-bucket/charts/postgresql-0.8.3.tgz -if [ $? -ne 0 ]; then - echo "Chart was not actually uploaded" - exit 1 -fi - -helm search test-repo/postgres | grep -q 0.8.3 -if [ $? -ne 0 ]; then - echo "Failed to find uploaded chart" - exit 1 -fi - -# -# Test: push the same chart again -# - -set +e # next command should return non-zero status - -helm s3 push postgresql-0.8.3.tgz test-repo -if [ $? -eq 0 ]; then - echo "The same chart must not be pushed again" - exit 1 -fi - -set -e - -helm s3 push --force postgresql-0.8.3.tgz test-repo -if [ $? -ne 0 ]; then - echo "The same chart must be pushed again using --force" - exit 1 -fi - -# -# Test: fetch chart -# - -helm fetch test-repo/postgresql --version 0.8.3 -if [ $? -ne 0 ]; then - echo "Failed to fetch chart from repo" - exit 1 -fi - -# -# Test: delete chart -# - -helm s3 delete postgresql --version 0.8.3 test-repo -if [ $? -ne 0 ]; then - echo "Failed to delete chart from repo" - exit 1 -fi - -if mc ls -q helm-s3-minio/test-bucket/charts/postgresql-0.8.3.tgz 2>/dev/null ; then - echo "Chart was not actually deleted" - exit 1 -fi - -if helm search test-repo/postgres | grep -q 0.8.3 ; then - echo "Failed to delete chart from index" - exit 1 -fi - -# -# Test: push with content-type -# -expected_content_type='application/gzip' -helm s3 push --content-type=${expected_content_type} postgresql-0.8.3.tgz test-repo -if [ $? -ne 0 ]; then - echo "Failed to push chart to repo" - exit 1 -fi - -helm search test-repo/postgres | grep -q 0.8.3 -if [ $? -ne 0 ]; then - echo "Failed to find uploaded chart" - exit 1 -fi - -mc ls helm-s3-minio/test-bucket/charts/postgresql-0.8.3.tgz -if [ $? -ne 0 ]; then - echo "Chart was not actually uploaded" - exit 1 -fi - -actual_content_type=$(mc stat helm-s3-minio/test-bucket/charts/postgresql-0.8.3.tgz | awk '/Content-Type/{print $NF}') -if [ $? -ne 0 ]; then - echo "failed to stat uploaded chart" - exit 1 -fi - -if [ "${expected_content_type}" != "${actual_content_type}" ]; then - echo "content-type, expected '${expected_content_type}', actual '${actual_content_type}'" - exit 1 -fi - -# -# Tear down -# - -rm postgresql-0.8.3.tgz -helm repo remove test-repo -set +x - +helm repo remove "${REPO}" &>/dev/null || true + +TEST_CASE="helm s3 init" +helm s3 init "${S3_URI}" +mc ls "${MINIO}/index.yaml" &>/dev/null +helm repo add "${REPO}" "${S3_URI}" + +TEST_CASE="helm s3 push" +helm s3 push postgresql-0.8.3.tgz "${REPO}" +mc ls "${MINIO}/postgresql-0.8.3.tgz" &>/dev/null +helm search "${REPO}/postgres" | grep -q 0.8.3 + +TEST_CASE="helm s3 push fails" +! helm s3 push postgresql-0.8.3.tgz "${REPO}" 2>/dev/null + +TEST_CASE="helm s3 push --force" +helm s3 push --force postgresql-0.8.3.tgz "${REPO}" + +TEST_CASE="helm fetch" +helm fetch "${REPO}/postgresql" --version 0.8.3 + +TEST_CASE="helm s3 reindex --publish " +helm s3 reindex "${REPO}" --publish "${PUBLISH_URI}" +mc cat "${MINIO}/index.yaml" | grep -Fqw "${PUBLISH_URI}/postgresql-0.8.3.tgz" +mc stat "${MINIO}/index.yaml" | grep "X-Amz-Meta-Helm-S3-Publish-Uri" | grep -Fqw "${PUBLISH_URI}" + +TEST_CASE="helm s3 reindex" +helm s3 reindex "${REPO}" +mc cat "${MINIO}/index.yaml" | grep -Fqw "${S3_URI}/postgresql-0.8.3.tgz" +mc stat "${MINIO}/index.yaml" | grep -w "X-Amz-Meta-Helm-S3-Publish-Uri\s*:\s*$" + +TEST_CASE="helm s3 delete" +helm s3 delete postgresql --version 0.8.3 "${REPO}" +! mc ls -q "${MINIO}/postgresql-0.8.3.tgz" 2>/dev/null +! helm search "${REPO}/postgres" | grep -Fq 0.8.3 + +TEST_CASE="helm s3 push --content-type " +helm s3 push --content-type=${CONTENT_TYPE} postgresql-0.8.3.tgz "${REPO}" +helm search "${REPO}/postgres" | grep -Fq 0.8.3 +mc ls "${MINIO}/postgresql-0.8.3.tgz" &>/dev/null +mc stat "${MINIO}/postgresql-0.8.3.tgz" | grep "Content-Type" | grep -Fqw "${CONTENT_TYPE}" + +# Cleanup to test published repo +helm repo remove "${REPO}" +mc rm --recursive --force "${MINIO}" + +TEST_CASE="helm s3 init --publish " +helm s3 init "${S3_URI}" --publish "${PUBLISH_URI}" +mc ls "${MINIO}/index.yaml" &>/dev/null +mc stat "${MINIO}/index.yaml" | grep "X-Amz-Meta-Helm-S3-Publish-Uri" | grep -Fqw "${PUBLISH_URI}" +helm repo add "${REPO}" "${S3_URI}" + +TEST_CASE="helm s3 push (publish)" +helm s3 push postgresql-0.8.3.tgz "${REPO}" +mc ls "${MINIO}/postgresql-0.8.3.tgz" &>/dev/null +mc cat "${MINIO}/index.yaml" | grep -Fqw "${PUBLISH_URI}/postgresql-0.8.3.tgz" +mc stat "${MINIO}/index.yaml" | grep "X-Amz-Meta-Helm-S3-Publish-Uri" | grep -Fqw "${PUBLISH_URI}" +helm search "${REPO}/postgres" | grep -Fq 0.8.3 + +TEST_CASE="helm fetch (publish)" +helm fetch "${REPO}/postgresql" --version 0.8.3 + +TEST_CASE="helm s3 delete (publish)" +helm s3 delete postgresql --version 0.8.3 "${REPO}" +mc stat "${MINIO}/index.yaml" | grep "X-Amz-Meta-Helm-S3-Publish-Uri" | grep -Fqw "${PUBLISH_URI}" +! mc ls -q "${MINIO}/postgresql-0.8.3.tgz" 2>/dev/null +! helm search "${REPO}/postgres" | grep -Fq 0.8.3 diff --git a/internal/awss3/storage.go b/internal/awss3/storage.go index 40808de4..a9c0b2b1 100644 --- a/internal/awss3/storage.go +++ b/internal/awss3/storage.go @@ -218,24 +218,34 @@ func (s *Storage) FetchRaw(ctx context.Context, uri string) ([]byte, error) { // Exists returns true if an object exists in the storage. func (s *Storage) Exists(ctx context.Context, uri string) (bool, error) { + if _, err := s.GetMetadata(ctx, uri); err != nil { + // That's weird that there is no NotFound constant in aws sdk. + if ae, ok := err.(awserr.Error); ok && ae.Code() == "NotFound" { + return false, nil + } + return false, errors.Wrap(err, "head s3 object") + } + + return true, nil +} + +// GetMetadata returns metadata associated with the object in storage +func (s *Storage) GetMetadata(ctx context.Context, uri string) (map[string]string, error) { bucket, key, err := parseURI(uri) if err != nil { - return false, err + return nil, err } - _, err = s3.New(s.session).HeadObject(&s3.HeadObjectInput{ + result, err := s3.New(s.session).HeadObject(&s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), }) + if err != nil { - // That's weird that there is no NotFound constant in aws sdk. - if ae, ok := err.(awserr.Error); ok && ae.Code() == "NotFound" { - return false, nil - } - return false, errors.Wrap(err, "head s3 object") + return nil, err } - return true, nil + return aws.StringValueMap(result.Metadata), nil } // PutChart puts the chart file to the storage. @@ -268,7 +278,7 @@ func (s *Storage) PutChart(ctx context.Context, uri string, r io.Reader, chartMe // PutIndex puts the index file to the storage. // uri must be in the form of s3 protocol: s3://bucket-name/key[...]. -func (s *Storage) PutIndex(ctx context.Context, uri string, acl string, r io.Reader) error { +func (s *Storage) PutIndex(ctx context.Context, uri string, publishURI string, acl string, r io.Reader) error { if strings.HasPrefix(uri, "index.yaml") { return errors.New("uri must not contain \"index.yaml\" suffix, it appends automatically") } @@ -286,6 +296,9 @@ func (s *Storage) PutIndex(ctx context.Context, uri string, acl string, r io.Rea ACL: aws.String(acl), ServerSideEncryption: getSSE(), Body: r, + Metadata: map[string]*string{ + MetaPublishURI: aws.String(publishURI), + }, }) if err != nil { return errors.Wrap(err, "upload index to S3 bucket") @@ -339,4 +352,7 @@ const ( // metaChartDigest is a s3 object metadata key that represents chart digest. metaChartDigest = "chart-digest" + + // MetaPublishURI s3 object metadata key that stores the non-s3 URI to publish + MetaPublishURI = "helm-s3-publish-uri" )