From 30ded459141356c88b239e27a94aa47ff97316bc Mon Sep 17 00:00:00 2001 From: emosbaugh Date: Thu, 14 Nov 2019 10:13:02 -0800 Subject: [PATCH] List new releases (#141) * List new releases * never return null * Channel sequence not release sequence --- ffi/airgap.go | 10 +- ffi/archive.go | 72 ++++++++++ ffi/main.go | 64 ++------- ffi/online.go | 10 +- ffi/updatedownload.go | 111 ++++++++++++++ ffi/updateslist.go | 75 ++++++++++ .../v1beta1tests/license_types_test.go | 2 - pkg/base/replicated.go | 3 - pkg/config/config.go | 2 - pkg/pull/peek.go | 57 ++++++++ pkg/pull/pull.go | 5 +- pkg/upload/archive.go | 2 - pkg/upstream/fetch.go | 19 +-- pkg/upstream/helm.go | 136 +++++++++++------- pkg/upstream/peek.go | 48 +++++++ pkg/upstream/replicated.go | 126 +++++++++++++--- pkg/upstream/upstream.go | 9 ++ pkg/upstream/write.go | 5 - 18 files changed, 591 insertions(+), 165 deletions(-) create mode 100644 ffi/archive.go create mode 100644 ffi/updatedownload.go create mode 100644 ffi/updateslist.go create mode 100644 pkg/pull/peek.go create mode 100644 pkg/upstream/peek.go diff --git a/ffi/airgap.go b/ffi/airgap.go index 3cafde5235..c52c166f09 100644 --- a/ffi/airgap.go +++ b/ffi/airgap.go @@ -14,10 +14,7 @@ import ( "github.com/mholt/archiver" "github.com/pkg/errors" - kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" - kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "github.com/replicatedhq/kots/pkg/pull" - "k8s.io/client-go/kubernetes/scheme" ) //export PullFromAirgap @@ -50,15 +47,12 @@ func PullFromAirgap(socket, licenseData, airgapDir, downstream, outputFile, regi return } - kotsscheme.AddToScheme(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - obj, _, err := decode([]byte(licenseData), nil, nil) + license, err := loadLicense(licenseData) if err != nil { - fmt.Printf("failed to decode license data: %s\n", err.Error()) + fmt.Printf("failed to load license: %s\n", err.Error()) ffiResult = NewFFIResult(1).WithError(err) return } - license := obj.(*kotsv1beta1.License) licenseFile, err := ioutil.TempFile("", "kots") if err != nil { diff --git a/ffi/archive.go b/ffi/archive.go new file mode 100644 index 0000000000..1395b25ba1 --- /dev/null +++ b/ffi/archive.go @@ -0,0 +1,72 @@ +package main + +import ( + "io/ioutil" + "os" + + "github.com/mholt/archiver" + "github.com/pkg/errors" + kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" + "k8s.io/client-go/kubernetes/scheme" +) + +func extractArchive(rootPath, fromArchivePath string) (*archiver.TarGz, error) { + // extract the current archive to this root + tarGz := &archiver.TarGz{ + Tar: &archiver.Tar{ + ImplicitTopLevelFolder: false, + }, + } + if err := tarGz.Unarchive(fromArchivePath, rootPath); err != nil { + return nil, err + } + + return tarGz, nil +} + +func readCursorFromPath(installationFilePath string) (string, error) { + _, err := os.Stat(installationFilePath) + if os.IsNotExist(err) { + return "", nil + } + if err != nil { + return "", errors.Wrap(err, "failed to open file") + } + + installationData, err := ioutil.ReadFile(installationFilePath) + if err != nil { + return "", errors.Wrap(err, "failed to read update installation file") + } + + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode([]byte(installationData), nil, nil) + if err != nil { + return "", errors.Wrap(err, "failed to devode installation data") + } + + installation := obj.(*kotsv1beta1.Installation) + return installation.Spec.UpdateCursor, nil +} + +func loadLicenseFromPath(expectedLicenseFile string) (*kotsv1beta1.License, error) { + _, err := os.Stat(expectedLicenseFile) + if err != nil { + return nil, errors.New("find license file in archive") + } + licenseData, err := ioutil.ReadFile(expectedLicenseFile) + if err != nil { + return nil, errors.Wrap(err, "read license file") + } + + return loadLicense(string(licenseData)) +} + +func loadLicense(licenseData string) (*kotsv1beta1.License, error) { + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode([]byte(licenseData), nil, nil) + if err != nil { + return nil, errors.Wrap(err, "decode license data") + } + + return obj.(*kotsv1beta1.License), nil +} diff --git a/ffi/main.go b/ffi/main.go index 9ce3b068d4..670905179e 100644 --- a/ffi/main.go +++ b/ffi/main.go @@ -10,8 +10,6 @@ import ( "os" "path/filepath" - "github.com/mholt/archiver" - "github.com/pkg/errors" kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "github.com/replicatedhq/kots/pkg/pull" @@ -41,19 +39,15 @@ func UpdateCheck(socket, fromArchivePath string) { } defer os.RemoveAll(tmpRoot) - // extract the current archive to this root - tarGz := archiver.TarGz{ - Tar: &archiver.Tar{ - ImplicitTopLevelFolder: false, - }, - } - if err := tarGz.Unarchive(fromArchivePath, tmpRoot); err != nil { - fmt.Printf("failed to unarchive: %s\n", err.Error()) + tarGz, err := extractArchive(tmpRoot, fromArchivePath) + if err != nil { + fmt.Printf("failed to extract archive: %s\n", err.Error()) ffiResult = NewFFIResult(-1).WithError(err) return } - beforeCursor, err := readCursorFromPath(tmpRoot) + installationFilePath := filepath.Join(tmpRoot, "upstream", "userdata", "installation.yaml") + beforeCursor, err := readCursorFromPath(installationFilePath) if err != nil { fmt.Printf("failed to read cursor file: %s\n", err.Error()) ffiResult = NewFFIResult(-1).WithError(err) @@ -61,29 +55,13 @@ func UpdateCheck(socket, fromArchivePath string) { } expectedLicenseFile := filepath.Join(tmpRoot, "upstream", "userdata", "license.yaml") - _, err = os.Stat(expectedLicenseFile) + license, err := loadLicenseFromPath(expectedLicenseFile) if err != nil { - fmt.Printf("failed to find license file in archive\n") - ffiResult = NewFFIResult(-1).WithError(err) - return - } - licenseData, err := ioutil.ReadFile(expectedLicenseFile) - if err != nil { - fmt.Printf("failed to read license file: %s\n", err.Error()) + fmt.Printf("failed to load license: %s\n", err.Error()) ffiResult = NewFFIResult(-1).WithError(err) return } - kotsscheme.AddToScheme(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - obj, _, err := decode([]byte(licenseData), nil, nil) - if err != nil { - fmt.Printf("failed to decode license data: %s\n", err.Error()) - ffiResult = NewFFIResult(-1).WithError(err) - return - } - license := obj.(*kotsv1beta1.License) - pullOptions := pull.PullOptions{ LicenseFile: expectedLicenseFile, ConfigFile: filepath.Join(tmpRoot, "upstream", "userdata", "config.yaml"), @@ -99,7 +77,7 @@ func UpdateCheck(socket, fromArchivePath string) { return } - afterCursor, err := readCursorFromPath(tmpRoot) + afterCursor, err := readCursorFromPath(installationFilePath) if err != nil { fmt.Printf("failed to read cursor file after update: %s\n", err.Error()) ffiResult = NewFFIResult(-1).WithError(err) @@ -151,7 +129,6 @@ func GetLatestLicense(socket, licenseData string) { statusClient.end(ffiResult) }() - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode obj, _, err := decode([]byte(licenseData), nil, nil) if err != nil { @@ -208,7 +185,6 @@ func GetLatestLicense(socket, licenseData string) { //export VerifyAirgapLicense func VerifyAirgapLicense(licenseData string) *C.char { - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode obj, _, err := decode([]byte(licenseData), nil, nil) if err != nil { @@ -225,30 +201,8 @@ func VerifyAirgapLicense(licenseData string) *C.char { return C.CString("verified") } -func readCursorFromPath(rootPath string) (string, error) { - installationFilePath := filepath.Join(rootPath, "upstream", "userdata", "installation.yaml") - _, err := os.Stat(installationFilePath) - if os.IsNotExist(err) { - return "", nil - } - if err != nil { - return "", errors.Wrap(err, "failed to open file") - } - - installationData, err := ioutil.ReadFile(installationFilePath) - if err != nil { - return "", errors.Wrap(err, "failed to read update installation file") - } - +func init() { kotsscheme.AddToScheme(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - obj, _, err := decode([]byte(installationData), nil, nil) - if err != nil { - return "", errors.Wrap(err, "failed to devode installation data") - } - - installation := obj.(*kotsv1beta1.Installation) - return installation.Spec.UpdateCursor, nil } func main() {} diff --git a/ffi/online.go b/ffi/online.go index 55d209dbc6..f59e28bdf9 100644 --- a/ffi/online.go +++ b/ffi/online.go @@ -9,10 +9,7 @@ import ( "path" "github.com/mholt/archiver" - kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" - kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "github.com/replicatedhq/kots/pkg/pull" - "k8s.io/client-go/kubernetes/scheme" ) //export PullFromLicense @@ -29,15 +26,12 @@ func PullFromLicense(socket string, licenseData string, downstream string, outpu statusClient.end(ffiResult) }() - kotsscheme.AddToScheme(scheme.Scheme) - decode := scheme.Codecs.UniversalDeserializer().Decode - obj, _, err := decode([]byte(licenseData), nil, nil) + license, err := loadLicense(licenseData) if err != nil { - fmt.Printf("failed to decode license data: %s\n", err.Error()) + fmt.Printf("failed to load license: %s\n", err.Error()) ffiResult = NewFFIResult(1).WithError(err) return } - license := obj.(*kotsv1beta1.License) licenseFile, err := ioutil.TempFile("", "kots") if err != nil { diff --git a/ffi/updatedownload.go b/ffi/updatedownload.go new file mode 100644 index 0000000000..55ff715d2d --- /dev/null +++ b/ffi/updatedownload.go @@ -0,0 +1,111 @@ +package main + +import "C" + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/replicatedhq/kots/pkg/pull" +) + +//export UpdateDownload +func UpdateDownload(socket, fromArchivePath, cursor string) { + go func() { + var ffiResult *FFIResult + + statusClient, err := connectToStatusServer(socket) + if err != nil { + fmt.Printf("failed to connect to status server: %s\n", err) + return + } + defer func() { + statusClient.end(ffiResult) + }() + + tmpRoot, err := ioutil.TempDir("", "kots") + if err != nil { + fmt.Printf("failed to create temp path: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + defer os.RemoveAll(tmpRoot) + + tarGz, err := extractArchive(tmpRoot, fromArchivePath) + if err != nil { + fmt.Printf("failed to extract archive: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + + installationFilePath := filepath.Join(tmpRoot, "upstream", "userdata", "installation.yaml") + beforeCursor, err := readCursorFromPath(installationFilePath) + if err != nil { + fmt.Printf("failed to read cursor file: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + + expectedLicenseFile := filepath.Join(tmpRoot, "upstream", "userdata", "license.yaml") + license, err := loadLicenseFromPath(expectedLicenseFile) + if err != nil { + fmt.Printf("failed to load license: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + + pullOptions := pull.PullOptions{ + LicenseFile: expectedLicenseFile, + ConfigFile: filepath.Join(tmpRoot, "upstream", "userdata", "config.yaml"), + UpdateCursor: cursor, + RootDir: tmpRoot, + ExcludeKotsKinds: true, + ExcludeAdminConsole: true, + CreateAppDir: false, + } + + if _, err := pull.Pull(fmt.Sprintf("replicated://%s", license.Spec.AppSlug), pullOptions); err != nil { + fmt.Printf("failed to pull upstream: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + + afterCursor, err := readCursorFromPath(installationFilePath) + if err != nil { + fmt.Printf("failed to read cursor file after update: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + + fmt.Printf("Result of checking for updates for %s: Before: %s, After %s\n", license.Spec.AppSlug, beforeCursor, afterCursor) + + isUpdateAvailable := string(beforeCursor) != string(afterCursor) + if !isUpdateAvailable { + ffiResult = NewFFIResult(0) + return + } + + paths := []string{ + filepath.Join(tmpRoot, "upstream"), + filepath.Join(tmpRoot, "base"), + filepath.Join(tmpRoot, "overlays"), + } + + err = os.Remove(fromArchivePath) + if err != nil { + fmt.Printf("failed to delete archive to replace: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + + if err := tarGz.Archive(paths, fromArchivePath); err != nil { + fmt.Printf("failed to write archive: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + + ffiResult = NewFFIResult(1) + }() +} diff --git a/ffi/updateslist.go b/ffi/updateslist.go new file mode 100644 index 0000000000..c6a0801030 --- /dev/null +++ b/ffi/updateslist.go @@ -0,0 +1,75 @@ +package main + +import "C" + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/replicatedhq/kots/pkg/pull" + "github.com/replicatedhq/kots/pkg/upstream" +) + +//export ListUpdates +func ListUpdates(socket, fromArchivePath, currentCursor string) { + go func() { + var ffiResult *FFIResult + + statusClient, err := connectToStatusServer(socket) + if err != nil { + fmt.Printf("failed to connect to status server: %s\n", err) + return + } + defer func() { + statusClient.end(ffiResult) + }() + + tmpRoot, err := ioutil.TempDir("", "kots") + if err != nil { + fmt.Printf("failed to create temp path: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + defer os.RemoveAll(tmpRoot) + + if _, err := extractArchive(tmpRoot, fromArchivePath); err != nil { + fmt.Printf("failed to extract archive: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + + expectedLicenseFile := filepath.Join(tmpRoot, "upstream", "userdata", "license.yaml") + license, err := loadLicenseFromPath(expectedLicenseFile) + if err != nil { + fmt.Printf("failed to load license: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + + peekOptions := pull.PeekOptions{ + LicenseFile: expectedLicenseFile, + CurrentCursor: currentCursor, + } + + updates, err := pull.Peek(fmt.Sprintf("replicated://%s", license.Spec.AppSlug), peekOptions) + if err != nil { + fmt.Printf("failed to peek upstream: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + if updates == nil { + updates = []upstream.Update{} + } + + b, err := json.Marshal(updates) + if err != nil { + fmt.Printf("failed to marshal updates: %s\n", err.Error()) + ffiResult = NewFFIResult(-1).WithError(err) + return + } + ffiResult = NewFFIResult(0).WithData(string(b)) + }() +} diff --git a/kotskinds/apis/kots/v1beta1/v1beta1tests/license_types_test.go b/kotskinds/apis/kots/v1beta1/v1beta1tests/license_types_test.go index 0bf86479cf..bb8ea27777 100644 --- a/kotskinds/apis/kots/v1beta1/v1beta1tests/license_types_test.go +++ b/kotskinds/apis/kots/v1beta1/v1beta1tests/license_types_test.go @@ -4,7 +4,6 @@ import ( "testing" kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" - kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/client-go/kubernetes/scheme" @@ -41,7 +40,6 @@ spec: value: "123asd" signature: IA==` - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode([]byte(data), nil, nil) require.NoError(t, err) diff --git a/pkg/base/replicated.go b/pkg/base/replicated.go index 46937c762a..30c2e428c5 100644 --- a/pkg/base/replicated.go +++ b/pkg/base/replicated.go @@ -3,7 +3,6 @@ package base import ( "github.com/pkg/errors" kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" - kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "github.com/replicatedhq/kots/pkg/template" "github.com/replicatedhq/kots/pkg/upstream" "k8s.io/client-go/kubernetes/scheme" @@ -68,7 +67,6 @@ func renderReplicated(u *upstream.Upstream, renderOptions *RenderOptions) (*Base } func UnmarshalConfigValuesContent(content []byte) (map[string]template.ItemValue, error) { - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode(content, nil, nil) if err != nil { @@ -93,7 +91,6 @@ func UnmarshalConfigValuesContent(content []byte) (map[string]template.ItemValue } func tryGetConfigFromFileContent(content []byte) *kotsv1beta1.Config { - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode(content, nil, nil) if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index ef4f56b386..b5b24c4585 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -5,7 +5,6 @@ import ( "github.com/pkg/errors" kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" - kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "github.com/replicatedhq/kots/pkg/base" "github.com/replicatedhq/kots/pkg/logger" "github.com/replicatedhq/kots/pkg/template" @@ -21,7 +20,6 @@ func TemplateConfig(log *logger.Logger, configPath string, configData string, co // 4. put new config yaml through templating engine // This process will re-order items and discard comments, so it should not be saved. - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode obj, _, err := decode([]byte(configData), nil, nil) if err != nil { diff --git a/pkg/pull/peek.go b/pkg/pull/peek.go new file mode 100644 index 0000000000..d7afbdcf41 --- /dev/null +++ b/pkg/pull/peek.go @@ -0,0 +1,57 @@ +package pull + +import ( + "github.com/pkg/errors" + "github.com/replicatedhq/kots/pkg/logger" + "github.com/replicatedhq/kots/pkg/upstream" +) + +type PeekOptions struct { + HelmRepoURI string + Namespace string + LocalPath string + LicenseFile string + CurrentCursor string + Silent bool +} + +// Peek will retrieve all later versions of the application specified in upstreamURI +// using the options specified in peekOptions. It returns a list of versions. +func Peek(upstreamURI string, peekOptions PeekOptions) ([]upstream.Update, error) { + log := logger.NewLogger() + + if peekOptions.Silent { + log.Silence() + } + + log.Initialize() + + fetchOptions := upstream.FetchOptions{} + fetchOptions.HelmRepoURI = peekOptions.HelmRepoURI + fetchOptions.LocalPath = peekOptions.LocalPath + fetchOptions.CurrentCursor = peekOptions.CurrentCursor + + if peekOptions.LicenseFile != "" { + license, err := parseLicenseFromFile(peekOptions.LicenseFile) + if err != nil { + if errors.Cause(err) == ErrSignatureInvalid { + return nil, ErrSignatureInvalid + } + if errors.Cause(err) == ErrSignatureMissing { + return nil, ErrSignatureMissing + } + return nil, errors.Wrap(err, "failed to parse license from file") + } + + fetchOptions.License = license + } + + log.ActionWithSpinner("Listing releases") + v, err := upstream.PeekUpstream(upstreamURI, &fetchOptions) + if err != nil { + log.FinishSpinnerWithError() + return nil, errors.Wrap(err, "failed to fetch upstream") + } + + return v, nil +} diff --git a/pkg/pull/pull.go b/pkg/pull/pull.go index 0e8b12c012..34bf7a5d88 100644 --- a/pkg/pull/pull.go +++ b/pkg/pull/pull.go @@ -12,7 +12,6 @@ import ( "github.com/pkg/errors" kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" - kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "github.com/replicatedhq/kots/pkg/base" "github.com/replicatedhq/kots/pkg/docker/registry" "github.com/replicatedhq/kots/pkg/downstream" @@ -34,6 +33,7 @@ type PullOptions struct { LocalPath string LicenseFile string ConfigFile string + UpdateCursor string ExcludeKotsKinds bool ExcludeAdminConsole bool SharedPassword string @@ -112,6 +112,7 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) { fetchOptions.RootDir = pullOptions.RootDir fetchOptions.UseAppDir = pullOptions.CreateAppDir fetchOptions.LocalPath = pullOptions.LocalPath + fetchOptions.CurrentCursor = pullOptions.UpdateCursor if pullOptions.LicenseFile != "" { license, err := parseLicenseFromFile(pullOptions.LicenseFile) @@ -351,7 +352,6 @@ func parseLicenseFromFile(filename string) (*kotsv1beta1.License, error) { return nil, errors.Wrap(err, "failed to read license file") } - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode decoded, gvk, err := decode(contents, nil, nil) if err != nil { @@ -379,7 +379,6 @@ func parseConfigValuesFromFile(filename string) (*kotsv1beta1.ConfigValues, erro return nil, errors.Wrap(err, "failed to read config values file") } - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode decoded, gvk, err := decode(contents, nil, nil) if err != nil { diff --git a/pkg/upload/archive.go b/pkg/upload/archive.go index 854ccbcfd4..bc8ddae39e 100644 --- a/pkg/upload/archive.go +++ b/pkg/upload/archive.go @@ -9,7 +9,6 @@ import ( "github.com/mholt/archiver" "github.com/pkg/errors" kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" - kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "k8s.io/client-go/kubernetes/scheme" ) @@ -58,7 +57,6 @@ func findUpdateCursor(rootPath string) (string, error) { return "", errors.Wrap(err, "failed to read update installation file") } - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode obj, _, err := decode([]byte(installationData), nil, nil) if err != nil { diff --git a/pkg/upstream/fetch.go b/pkg/upstream/fetch.go index a7c8068c5f..61b45bd6c9 100644 --- a/pkg/upstream/fetch.go +++ b/pkg/upstream/fetch.go @@ -9,14 +9,15 @@ import ( ) type FetchOptions struct { - RootDir string - UseAppDir bool - HelmRepoName string - HelmRepoURI string - HelmOptions []string - LocalPath string - License *kotsv1beta1.License - ConfigValues *kotsv1beta1.ConfigValues + RootDir string + UseAppDir bool + HelmRepoName string + HelmRepoURI string + HelmOptions []string + LocalPath string + License *kotsv1beta1.License + ConfigValues *kotsv1beta1.ConfigValues + CurrentCursor string } func FetchUpstream(upstreamURI string, fetchOptions *FetchOptions) (*Upstream, error) { @@ -41,7 +42,7 @@ func downloadUpstream(upstreamURI string, fetchOptions *FetchOptions) (*Upstream return downloadHelm(u, fetchOptions.HelmRepoURI) } if u.Scheme == "replicated" { - return downloadReplicated(u, fetchOptions.LocalPath, fetchOptions.RootDir, fetchOptions.UseAppDir, fetchOptions.License, fetchOptions.ConfigValues) + return downloadReplicated(u, fetchOptions.LocalPath, fetchOptions.RootDir, fetchOptions.UseAppDir, fetchOptions.License, fetchOptions.ConfigValues, fetchOptions.CurrentCursor) } if u.Scheme == "git" { return downloadGit(upstreamURI) diff --git a/pkg/upstream/helm.go b/pkg/upstream/helm.go index 318a976e04..9b947e84f3 100644 --- a/pkg/upstream/helm.go +++ b/pkg/upstream/helm.go @@ -24,77 +24,49 @@ import ( "k8s.io/helm/pkg/repo" ) -func downloadHelm(u *url.URL, repoURI string) (*Upstream, error) { - repoName, chartName, chartVersion, err := parseHelmURL(u) +func peekHelm(u *url.URL, repoURI string) ([]Update, error) { + repoName, chartName, _, err := parseHelmURL(u) if err != nil { return nil, errors.Wrap(err, "failed to parse helm uri") } - if repoURI == "" { - repoURI = getKnownHelmRepoURI(repoName) - } - - if repoURI == "" { - return nil, errors.New("unknown helm repo uri, try passing the repo uri") - } - helmHome, err := ioutil.TempDir("", "kots") if err != nil { return nil, errors.Wrap(err, "failed to create temporary helm home") } defer os.RemoveAll(helmHome) - if err := os.MkdirAll(filepath.Join(helmHome, "repository"), 0755); err != nil { - return nil, errors.Wrap(err, "failed to make directory for helm home") - } - reposFile := filepath.Join(helmHome, "repository", "repositories.yaml") - repoIndexFile, err := ioutil.TempFile("", "index") + i, err := helmLoadRepositoriesIndex(helmHome, repoName, repoURI) if err != nil { - return nil, errors.Wrap(err, "failed to create temporary index file") + return nil, errors.Wrap(err, "failed to load helm repositories") } - defer os.Remove(repoIndexFile.Name()) - cacheIndexFile, err := ioutil.TempFile("", "cache") - if err != nil { - return nil, errors.Wrap(err, "failed to create cache index file") - } - defer os.Remove(cacheIndexFile.Name()) + var updates []Update + for _, result := range i.All() { + if result.Chart.GetName() != chartName { + continue + } - repoYAML := `apiVersion: v1 -generated: "2019-05-29T14:31:58.906598702Z" -repositories: []` - if err := ioutil.WriteFile(reposFile, []byte(repoYAML), 0644); err != nil { - return nil, err + updates = append(updates, Update{Cursor: result.Chart.GetVersion()}) } + return updates, nil +} - c := repo.Entry{ - Name: repoName, - Cache: repoIndexFile.Name(), - URL: repoURI, - } - r, err := repo.NewChartRepository(&c, getter.All(environment.EnvSettings{})) +func downloadHelm(u *url.URL, repoURI string) (*Upstream, error) { + repoName, chartName, chartVersion, err := parseHelmURL(u) if err != nil { - return nil, errors.Wrap(err, "failed to create chart repository") - } - if err := r.DownloadIndexFile(cacheIndexFile.Name()); err != nil { - return nil, errors.Wrap(err, "failed to download index file") + return nil, errors.Wrap(err, "failed to parse helm uri") } - rf, err := repo.LoadRepositoriesFile(reposFile) + helmHome, err := ioutil.TempDir("", "kots") if err != nil { - return nil, errors.Wrap(err, "failed to load repositories file") + return nil, errors.Wrap(err, "failed to create temporary helm home") } - rf.Update(&c) - - i := search.NewIndex() - for _, re := range rf.Repositories { - n := re.Name - ind, err := repo.LoadIndexFile(repoIndexFile.Name()) - if err != nil { - return nil, errors.Wrap(err, "failed to load index file") - } + defer os.RemoveAll(helmHome) - i.AddRepo(n, ind, true) + i, err := helmLoadRepositoriesIndex(helmHome, repoName, repoURI) + if err != nil { + return nil, errors.Wrap(err, "failed to load helm repositories") } if chartVersion == "" { @@ -168,6 +140,72 @@ repositories: []` return nil, errors.New("chart version not found") } +func helmLoadRepositoriesIndex(helmHome, repoName, repoURI string) (*search.Index, error) { + if repoURI == "" { + repoURI = getKnownHelmRepoURI(repoName) + } + + if repoURI == "" { + return nil, errors.New("unknown helm repo uri, try passing the repo uri") + } + + if err := os.MkdirAll(filepath.Join(helmHome, "repository"), 0755); err != nil { + return nil, errors.Wrap(err, "failed to make directory for helm home") + } + reposFile := filepath.Join(helmHome, "repository", "repositories.yaml") + + repoIndexFile, err := ioutil.TempFile("", "index") + if err != nil { + return nil, errors.Wrap(err, "failed to create temporary index file") + } + defer os.Remove(repoIndexFile.Name()) + + cacheIndexFile, err := ioutil.TempFile("", "cache") + if err != nil { + return nil, errors.Wrap(err, "failed to create cache index file") + } + defer os.Remove(cacheIndexFile.Name()) + + repoYAML := `apiVersion: v1 +generated: "2019-05-29T14:31:58.906598702Z" +repositories: []` + if err := ioutil.WriteFile(reposFile, []byte(repoYAML), 0644); err != nil { + return nil, err + } + + c := repo.Entry{ + Name: repoName, + Cache: repoIndexFile.Name(), + URL: repoURI, + } + r, err := repo.NewChartRepository(&c, getter.All(environment.EnvSettings{})) + if err != nil { + return nil, errors.Wrap(err, "failed to create chart repository") + } + if err := r.DownloadIndexFile(cacheIndexFile.Name()); err != nil { + return nil, errors.Wrap(err, "failed to download index file") + } + + rf, err := repo.LoadRepositoriesFile(reposFile) + if err != nil { + return nil, errors.Wrap(err, "failed to load repositories file") + } + rf.Update(&c) + + i := search.NewIndex() + for _, re := range rf.Repositories { + n := re.Name + ind, err := repo.LoadIndexFile(repoIndexFile.Name()) + if err != nil { + return nil, errors.Wrap(err, "failed to load index file") + } + + i.AddRepo(n, ind, true) + } + + return i, nil +} + func parseHelmURL(u *url.URL) (string, string, string, error) { repo := u.Host chartName := strings.TrimLeft(u.Path, "/") diff --git a/pkg/upstream/peek.go b/pkg/upstream/peek.go new file mode 100644 index 0000000000..c2e5544031 --- /dev/null +++ b/pkg/upstream/peek.go @@ -0,0 +1,48 @@ +package upstream + +import ( + "net/url" + + "github.com/pkg/errors" + "github.com/replicatedhq/kots/pkg/util" +) + +type Update struct { + Cursor string `json:"cursor"` +} + +func PeekUpstream(upstreamURI string, fetchOptions *FetchOptions) ([]Update, error) { + versions, err := peekUpstream(upstreamURI, fetchOptions) + if err != nil { + return nil, errors.Wrap(err, "download upstream failed") + } + + return versions, nil +} + +func peekUpstream(upstreamURI string, fetchOptions *FetchOptions) ([]Update, error) { + if !util.IsURL(upstreamURI) { + return nil, errors.New("not implemented") + } + + u, err := url.ParseRequestURI(upstreamURI) + if err != nil { + return nil, errors.Wrap(err, "parse request uri failed") + } + if u.Scheme == "helm" { + return peekHelm(u, fetchOptions.HelmRepoURI) + } + if u.Scheme == "replicated" { + return peekReplicated(u, fetchOptions.LocalPath, fetchOptions.License, fetchOptions.CurrentCursor) + } + if u.Scheme == "git" { + // return peekGit(upstreamURI) + // TODO + } + if u.Scheme == "http" || u.Scheme == "https" { + // return peekHttp(upstreamURI) + // TODO + } + + return nil, errors.Errorf("unknown protocol scheme %q", u.Scheme) +} diff --git a/pkg/upstream/replicated.go b/pkg/upstream/replicated.go index e523db02d5..21fc5fa0c8 100644 --- a/pkg/upstream/replicated.go +++ b/pkg/upstream/replicated.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/gzip" "encoding/base64" + "encoding/json" "fmt" "io" "io/ioutil" @@ -13,13 +14,13 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" imagedocker "github.com/containers/image/docker" dockerref "github.com/containers/image/docker/reference" "github.com/pkg/errors" kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" - kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "github.com/replicatedhq/kots/pkg/docker/registry" "github.com/replicatedhq/kots/pkg/image" "github.com/replicatedhq/kots/pkg/k8sdoc" @@ -27,7 +28,7 @@ import ( "github.com/replicatedhq/kots/pkg/template" "github.com/replicatedhq/kots/pkg/util" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/serializer/json" + serializer "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/client-go/kubernetes/scheme" kustomizeimage "sigs.k8s.io/kustomize/v3/pkg/image" ) @@ -60,7 +61,51 @@ type Release struct { Manifests map[string][]byte } -func downloadReplicated(u *url.URL, localPath string, rootDir string, useAppDir bool, license *kotsv1beta1.License, existingConfigValues *kotsv1beta1.ConfigValues) (*Upstream, error) { +type ChannelRelease struct { + ChannelSequence int `json:"channelSequence"` + ReleaseSequence int `json:"releaseSequence"` + VersionLabel string `json:"versionLabel"` + CreatedAt string `json:"createdAt"` +} + +func peekReplicated(u *url.URL, localPath string, license *kotsv1beta1.License, channelSequence string) ([]Update, error) { + if localPath != "" { + parsedLocalRelease, err := readReplicatedAppFromLocalPath(localPath) + if err != nil { + return nil, errors.Wrap(err, "failed to read replicated app from local path") + } + + return []Update{{Cursor: parsedLocalRelease.UpdateCursor}}, nil + } + + // A license file is required to be set for this to succeed + if license == nil { + return nil, errors.New("No license was provided") + } + + replicatedUpstream, err := parseReplicatedURL(u) + if err != nil { + return nil, errors.Wrap(err, "failed to parse replicated upstream") + } + + remoteLicense, err := getSuccessfulHeadResponse(replicatedUpstream, license) + if err != nil { + return nil, errors.Wrap(err, "failed to get successful head response") + } + + pendingReleases, err := listPendingChannelReleases(replicatedUpstream, remoteLicense, channelSequence) + if err != nil { + return nil, errors.Wrap(err, "failed to list replicated app releases") + } + + updates := []Update{} + for _, pendingRelease := range pendingReleases { + updates = append(updates, Update{Cursor: strconv.Itoa(pendingRelease.ChannelSequence)}) + } + return updates, nil +} + +func downloadReplicated(u *url.URL, localPath string, rootDir string, useAppDir bool, license *kotsv1beta1.License, existingConfigValues *kotsv1beta1.ConfigValues, updateCursor string) (*Upstream, error) { var release *Release if localPath != "" { @@ -70,6 +115,9 @@ func downloadReplicated(u *url.URL, localPath string, rootDir string, useAppDir } release = parsedLocalRelease + if updateCursor != "" && release.UpdateCursor != updateCursor { + return nil, errors.Wrap(err, "release in local path does not match update cursor") + } } else { // A license file is required to be set for this to succeed if license == nil { @@ -81,12 +129,12 @@ func downloadReplicated(u *url.URL, localPath string, rootDir string, useAppDir return nil, errors.Wrap(err, "failed to parse replicated upstream") } - license, err := getSuccessfulHeadResponse(replicatedUpstream, license) + remoteLicense, err := getSuccessfulHeadResponse(replicatedUpstream, license) if err != nil { return nil, errors.Wrap(err, "failed to get successful head response") } - downloadedRelease, err := downloadReplicatedApp(replicatedUpstream, license) + downloadedRelease, err := downloadReplicatedApp(replicatedUpstream, remoteLicense, updateCursor) if err != nil { return nil, errors.Wrap(err, "failed to download replicated app") } @@ -152,7 +200,7 @@ func downloadReplicated(u *url.URL, localPath string, rootDir string, useAppDir return upstream, nil } -func (r *ReplicatedUpstream) getRequest(method string, license *kotsv1beta1.License) (*http.Request, error) { +func (r *ReplicatedUpstream) getRequest(method string, license *kotsv1beta1.License, channelSequence string) (*http.Request, error) { u, err := url.Parse(license.Spec.Endpoint) if err != nil { return nil, errors.Wrap(err, "failed to parse endpoint from license") @@ -163,9 +211,10 @@ func (r *ReplicatedUpstream) getRequest(method string, license *kotsv1beta1.Lice hostname = fmt.Sprintf("%s:%s", u.Hostname(), u.Port()) } - url := fmt.Sprintf("%s://%s/release/%s", u.Scheme, hostname, license.Spec.AppSlug) + url := fmt.Sprintf("%s://%s/release/%s?channelSequence=%s", u.Scheme, hostname, license.Spec.AppSlug, channelSequence) if r.Channel != nil { + // NOTE: I don't believe this is used url = fmt.Sprintf("%s/%s", url, *r.Channel) } @@ -202,7 +251,7 @@ func parseReplicatedURL(u *url.URL) (*ReplicatedUpstream, error) { } func getSuccessfulHeadResponse(replicatedUpstream *ReplicatedUpstream, license *kotsv1beta1.License) (*kotsv1beta1.License, error) { - headReq, err := replicatedUpstream.getRequest("HEAD", license) + headReq, err := replicatedUpstream.getRequest("HEAD", license, "") if err != nil { return nil, errors.Wrap(err, "failed to create http request") } @@ -259,8 +308,8 @@ func readReplicatedAppFromLocalPath(localPath string) (*Release, error) { return &release, nil } -func downloadReplicatedApp(replicatedUpstream *ReplicatedUpstream, license *kotsv1beta1.License) (*Release, error) { - getReq, err := replicatedUpstream.getRequest("GET", license) +func downloadReplicatedApp(replicatedUpstream *ReplicatedUpstream, license *kotsv1beta1.License, channelSequence string) (*Release, error) { + getReq, err := replicatedUpstream.getRequest("GET", license, channelSequence) if err != nil { return nil, errors.Wrap(err, "failed to create http request") } @@ -274,7 +323,7 @@ func downloadReplicatedApp(replicatedUpstream *ReplicatedUpstream, license *kots return nil, errors.Errorf("unexpected result from get request: %d", getResp.StatusCode) } - updateCursor := getResp.Header.Get("X-Replicated-Sequence") + updateCursor := getResp.Header.Get("X-Replicated-ChannelSequence") versionLabel := getResp.Header.Get("X-Replicated-VersionLabel") gzf, err := gzip.NewReader(getResp.Body) @@ -320,10 +369,53 @@ func downloadReplicatedApp(replicatedUpstream *ReplicatedUpstream, license *kots return &release, nil } -func MustMarshalLicense(license *kotsv1beta1.License) []byte { - kotsscheme.AddToScheme(scheme.Scheme) +func listPendingChannelReleases(replicatedUpstream *ReplicatedUpstream, license *kotsv1beta1.License, channelSequence string) ([]ChannelRelease, error) { + u, err := url.Parse(license.Spec.Endpoint) + if err != nil { + return nil, errors.Wrap(err, "failed to parse endpoint from license") + } + + hostname := u.Hostname() + if u.Port() != "" { + hostname = fmt.Sprintf("%s:%s", u.Hostname(), u.Port()) + } + + url := fmt.Sprintf("%s://%s/release/%s/pending?channelSequence=%s", u.Scheme, hostname, license.Spec.AppSlug, channelSequence) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to call newrequest") + } + + req.Header.Set("Authorization", fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", license.Spec.LicenseID, license.Spec.LicenseID))))) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, errors.Wrap(err, "failed to execute get request") + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + return nil, errors.Errorf("unexpected result from get request: %d", resp.StatusCode) + } - s := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "failed to read response body") + } + + var channelReleases struct { + ChannelReleases []ChannelRelease `json:"channelReleases"` + } + if err := json.Unmarshal(body, &channelReleases); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal response") + } + + return channelReleases.ChannelReleases, nil +} + +func MustMarshalLicense(license *kotsv1beta1.License) []byte { + s := serializer.NewYAMLSerializer(serializer.DefaultMetaFactory, scheme.Scheme, scheme.Scheme) var b bytes.Buffer if err := s.Encode(license, &b); err != nil { @@ -334,9 +426,7 @@ func MustMarshalLicense(license *kotsv1beta1.License) []byte { } func mustMarshalConfigValues(configValues *kotsv1beta1.ConfigValues) []byte { - kotsscheme.AddToScheme(scheme.Scheme) - - s := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme) + s := serializer.NewYAMLSerializer(serializer.DefaultMetaFactory, scheme.Scheme, scheme.Scheme) var b bytes.Buffer if err := s.Encode(configValues, &b); err != nil { @@ -447,7 +537,6 @@ func findConfigValuesInFile(filename string) (*kotsv1beta1.ConfigValues, error) } func findConfigInRelease(release *Release) *kotsv1beta1.Config { - kotsscheme.AddToScheme(scheme.Scheme) for _, content := range release.Manifests { decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode(content, nil, nil) @@ -468,7 +557,6 @@ func findConfigInRelease(release *Release) *kotsv1beta1.Config { } func findAppInRelease(release *Release) *kotsv1beta1.Application { - kotsscheme.AddToScheme(scheme.Scheme) for _, content := range release.Manifests { decode := scheme.Codecs.UniversalDeserializer().Decode obj, gvk, err := decode(content, nil, nil) diff --git a/pkg/upstream/upstream.go b/pkg/upstream/upstream.go index d68e21f2ca..76594817a7 100644 --- a/pkg/upstream/upstream.go +++ b/pkg/upstream/upstream.go @@ -1,5 +1,14 @@ package upstream +import ( + kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" + "k8s.io/client-go/kubernetes/scheme" +) + +func init() { + kotsscheme.AddToScheme(scheme.Scheme) +} + type UpstreamFile struct { Path string Content []byte diff --git a/pkg/upstream/write.go b/pkg/upstream/write.go index 6d850c0413..7e4d969ce2 100644 --- a/pkg/upstream/write.go +++ b/pkg/upstream/write.go @@ -8,7 +8,6 @@ import ( "github.com/pkg/errors" kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1" - kotsscheme "github.com/replicatedhq/kots/kotskinds/client/kotsclientset/scheme" "github.com/replicatedhq/kots/pkg/crypto" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/serializer/json" @@ -159,7 +158,6 @@ func getEncryptionKey(previousInstallationContent []byte) (string, error) { return cipher.ToString(), nil } - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode prevObj, _, err := decode(previousInstallationContent, nil, nil) @@ -172,7 +170,6 @@ func getEncryptionKey(previousInstallationContent []byte) (string, error) { } func mergeValues(previousValues []byte, applicationDeliveredValues []byte) ([]byte, error) { - kotsscheme.AddToScheme(scheme.Scheme) decode := scheme.Codecs.UniversalDeserializer().Decode prevObj, _, err := decode(previousValues, nil, nil) @@ -205,8 +202,6 @@ func mergeValues(previousValues []byte, applicationDeliveredValues []byte) ([]by } func mustMarshalInstallation(installation *kotsv1beta1.Installation) []byte { - kotsscheme.AddToScheme(scheme.Scheme) - s := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme) var b bytes.Buffer