From dfefe8ae2573f763b306027e68145c21bef6b516 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 May 2023 19:32:21 -0500 Subject: [PATCH 01/29] WIP --- README.md | 12 +++++++--- spec.go | 1 + spec_test.go | 2 +- sync.go | 57 +++++++++++++++++++++++++++-------------------- testdata/vdm.json | 21 ++++++++++------- validate.go | 26 +++++++++++++++++---- validate_test.go | 14 +++++++++++- 7 files changed, 92 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index dc7b164..b7f7bc8 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,20 @@ the same reasons, in a more sane way. To get started, you'll need a `vdm` spec file, which is just a JSON array of all -your external dependencies along with their revisions & where you want them to -live in your repo: +your external dependencies along with (usually) their revisions & where you want +them to live in your repo: [ { "remote": "https://github.com/opensourcecorp/go-common", "version": "v0.2.0", // tag; can also be a branch, short or long commit hash, or the word 'latest' - "local_path": "./deps/go-common" + "local_path": "./deps/go-common", + "type": "git" // the default, and so can be omitted + }, + { + "remote": "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto", + "local_path": "./deps/http.proto", + "type": "file" // the 'file' type assumes the version is in the remote field itself, so 'version' can be omitted } ] diff --git a/spec.go b/spec.go index 34fff7a..256d17f 100644 --- a/spec.go +++ b/spec.go @@ -12,6 +12,7 @@ type vdmSpec struct { Remote string `json:"remote"` Version string `json:"version"` LocalPath string `json:"local_path"` + Type string `json:"type"` } func (spec vdmSpec) writeVDMMeta() error { diff --git a/spec_test.go b/spec_test.go index 7c8bccc..e967911 100644 --- a/spec_test.go +++ b/spec_test.go @@ -59,7 +59,7 @@ func TestSpecGetVDMMeta(t *testing.T) { specFilePath := "./testdata/vdm.json" specs := getSpecsFromFile(context.Background(), specFilePath) - assert.Equal(t, 4, len(specs)) + assert.Equal(t, 5, len(specs)) t.Cleanup(func() { os.RemoveAll(testVDMMetaFilePath) diff --git a/sync.go b/sync.go index eecb914..244e506 100644 --- a/sync.go +++ b/sync.go @@ -12,7 +12,7 @@ import ( func sync(ctx context.Context, specs []vdmSpec) { for _, spec := range specs { // Common log line prefix - operationMsg := fmt.Sprintf("%s@%s --> %s", spec.Remote, spec.Version, spec.LocalPath) + opMsg := fmt.Sprintf("%s@%s --> %s", spec.Remote, spec.Version, spec.LocalPath) // process stored VDMMETA so we know what operations to actually perform for existing directories vdmMeta := spec.getVDMMeta() @@ -28,38 +28,47 @@ func sync(ctx context.Context, specs []vdmSpec) { } } - // TODO: pull this up so that it only runs if the version changed or the user requested a wipe - if !shouldKeepGitDir(ctx) { - if isDebug(ctx) { - debugLogger.Printf("removing any old data for '%s'", spec.LocalPath) - } - os.RemoveAll(spec.LocalPath) + switch spec.Type { + case "git", "": + syncGitRemote(ctx, spec, opMsg) + default: + errLogger.Fatalf("") } - gitClone(ctx, spec, operationMsg) - - if spec.Version != "latest" { - infoLogger.Printf("%s -- Setting specified version...", operationMsg) - checkoutCmd := exec.Command("git", "-C", spec.LocalPath, "checkout", spec.Version) - checkoutOutput, err := checkoutCmd.CombinedOutput() - if err != nil { - errLogger.Fatalf("error checking out specified revision: exec error '%v', with output: %s", err, string(checkoutOutput)) - } + err := spec.writeVDMMeta() + if err != nil { + errLogger.Fatalf("Could not write VDMMETA file to disk: %v", err) } - if !shouldKeepGitDir(ctx) { - if isDebug(ctx) { - debugLogger.Printf("removing .git dir for local path '%s'", spec.LocalPath) - } - os.RemoveAll(filepath.Join(spec.LocalPath, ".git")) + infoLogger.Printf("%s -- Done.", opMsg) + } +} + +func syncGitRemote(ctx context.Context, spec vdmSpec, operationMsg string) { + // TODO: pull this up so that it only runs if the version changed or the user requested a wipe + if !shouldKeepGitDir(ctx) { + if isDebug(ctx) { + debugLogger.Printf("removing any old data for '%s'", spec.LocalPath) } + os.RemoveAll(spec.LocalPath) + } - err := spec.writeVDMMeta() + gitClone(ctx, spec, operationMsg) + + if spec.Version != "latest" { + infoLogger.Printf("%s -- Setting specified version...", operationMsg) + checkoutCmd := exec.Command("git", "-C", spec.LocalPath, "checkout", spec.Version) + checkoutOutput, err := checkoutCmd.CombinedOutput() if err != nil { - errLogger.Fatalf("Could not write VDMMETA file to disk: %v", err) + errLogger.Fatalf("error checking out specified revision: exec error '%v', with output: %s", err, string(checkoutOutput)) } + } - infoLogger.Printf("%s -- Done.", operationMsg) + if !shouldKeepGitDir(ctx) { + if isDebug(ctx) { + debugLogger.Printf("removing .git dir for local path '%s'", spec.LocalPath) + } + os.RemoveAll(filepath.Join(spec.LocalPath, ".git")) } } diff --git a/testdata/vdm.json b/testdata/vdm.json index 0739190..ec6fc27 100644 --- a/testdata/vdm.json +++ b/testdata/vdm.json @@ -1,22 +1,27 @@ [ { - "remote": "https://github.com/opensourcecorp/go-common", - "version": "v0.2.0", + "remote": "https://github.com/opensourcecorp/go-common", + "version": "v0.2.0", "local_path": "./deps/go-common-tag" }, { - "remote": "https://github.com/opensourcecorp/go-common", - "version": "latest", + "remote": "https://github.com/opensourcecorp/go-common", + "version": "latest", "local_path": "./deps/go-common-latest" }, { - "remote": "https://github.com/opensourcecorp/go-common", - "version": "main", + "remote": "https://github.com/opensourcecorp/go-common", + "version": "main", "local_path": "./deps/go-common-branch" }, { - "remote": "https://github.com/opensourcecorp/go-common", - "version": "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f", + "remote": "https://github.com/opensourcecorp/go-common", + "version": "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f", "local_path": "./deps/go-common-hash" + }, + { + "remote": "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto", + "local_path": "./deps/http.proto", + "type": "file" } ] diff --git a/validate.go b/validate.go index 6accf84..d2c2ded 100644 --- a/validate.go +++ b/validate.go @@ -10,6 +10,7 @@ import ( func (spec vdmSpec) Validate(ctx context.Context) error { var allErrors []error + // Remote field if isDebug(ctx) { debugLogger.Printf("validating field 'Remote' for %+v", spec) } @@ -20,17 +21,22 @@ func (spec vdmSpec) Validate(ctx context.Context) error { if !protocolRegex.MatchString(spec.Remote) { allErrors = append( allErrors, - fmt.Errorf("remote provided as '%s', but all 'remote' fields must begin with a protocol specifier or other valid prefix (e.g. 'https://', 'git@', etc.)", spec.Remote), + fmt.Errorf("remote provided as '%s', but all 'remote' fields must begin with a protocol specifier or other valid prefix (e.g. 'https://', '(user|git)@', etc.)", spec.Remote), ) } + // Version field if isDebug(ctx) { debugLogger.Printf("validating field 'Version' for %+v", spec) } - if len(spec.Version) == 0 { - allErrors = append(allErrors, errors.New("all 'version' fields must be non-zero length. If you don't care about the version (even though you should), then use 'latest'")) + if spec.Type == "git" && len(spec.Version) == 0 { + allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length. If you don't care about the version (even though you probably should), then use 'latest'")) + } + if spec.Type == "file" && len(spec.Version) > 0 { + infoLogger.Printf("NOTE: Remote %s specified as type '%s' but also specified version as '%s'; ignoring version field", spec.Remote, spec.Type, spec.Version) } + // LocalPath field if isDebug(ctx) { debugLogger.Printf("validating field 'LocalPath' for %+v", spec) } @@ -38,12 +44,24 @@ func (spec vdmSpec) Validate(ctx context.Context) error { allErrors = append(allErrors, errors.New("all 'local_path' fields must be non-zero length")) } + // Type field + if isDebug(ctx) { + debugLogger.Printf("validating field 'Version' for %+v", spec) + } + typeMap := map[string]struct{}{ + "git": {}, + "": {}, // also git + "file": {}, + } + if _, ok := typeMap[spec.Type]; !ok { + allErrors = append(allErrors, fmt.Errorf("unrecognized remote type '%s'", spec.Type)) + } + if len(allErrors) > 0 { for _, err := range allErrors { errLogger.Printf("validation failure: %s", err.Error()) } return fmt.Errorf("%d validation failure(s) found in your vdm spec file", len(allErrors)) } - return nil } diff --git a/validate_test.go b/validate_test.go index 17ba1f3..2a92517 100644 --- a/validate_test.go +++ b/validate_test.go @@ -40,11 +40,23 @@ func TestValidate(t *testing.T) { assert.Error(t, err) }) - t.Run("fails on zero-length version", func(t *testing.T) { + t.Run("fails on zero-length version for git remote type", func(t *testing.T) { spec := vdmSpec{ Remote: "https://some-remote", Version: "", LocalPath: "./deps/some-remote", + Type: "git", + } + err := spec.Validate(ctx) + assert.Error(t, err) + }) + + t.Run("fails on unrecognized remote type", func(t *testing.T) { + spec := vdmSpec{ + Remote: "https://some-remote", + Version: "", + LocalPath: "./deps/some-remote", + Type: "bad", } err := spec.Validate(ctx) assert.Error(t, err) From 8f34b5a6540f29651357320d61a42d317b0213df Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Mon, 3 Jul 2023 17:44:41 -0500 Subject: [PATCH 02/29] Broken WIP for moving between computers --- README.md | 50 +++++++++++++++++++++++++++++---------------- common.go | 59 ++--------------------------------------------------- go.mod | 6 +++++- go.sum | 5 +++++ main.go | 14 ++++++------- spec.go | 28 +++++++++++++------------ sync.go | 30 ++++++++++++++------------- system.go | 6 ++++-- validate.go | 14 +++++++------ 9 files changed, 94 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index b7f7bc8..0157480 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,27 @@ # vdm: Versioned Dependency Manager -`vdm` is an alternative to git submodules for managing external dependencies for -the same reasons, in a more sane way. +`vdm` is an alternative to e.g. git submodules for managing arbitrary external +dependencies for the same reasons, in a more sane way. To get started, you'll need a `vdm` spec file, which is just a JSON array of all your external dependencies along with (usually) their revisions & where you want them to live in your repo: - [ - { - "remote": "https://github.com/opensourcecorp/go-common", - "version": "v0.2.0", // tag; can also be a branch, short or long commit hash, or the word 'latest' - "local_path": "./deps/go-common", - "type": "git" // the default, and so can be omitted - }, - { - "remote": "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto", - "local_path": "./deps/http.proto", - "type": "file" // the 'file' type assumes the version is in the remote field itself, so 'version' can be omitted - } - ] +```jsonc +[ + { + "type": "git", // the default + "remote": "https://github.com/opensourcecorp/go-common", + "local_path": "./deps/go-common", + "version": "v0.2.0", // tag; can also be a branch, short or long commit hash, or the word 'latest' + }, + { + "type": "file", // the 'file' type assumes the version is in the remote field itself, so 'version' can be omitted + "remote": "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto", + "local_path": "./deps/http.proto" + } +] +``` You can have as many dependency specifications in that array as you want. By default, this spec file is called `vdm.json` and lives at the calling location @@ -28,7 +30,9 @@ point to it using the `-spec-file` flag to `vdm`. Once you have a spec file, just run: - vdm sync +```sh +vdm sync +``` and `vdm` will process the spec file, grab your dependencies, put them where they belong, and check out the right versions. By default, `vdm sync` also @@ -36,6 +40,17 @@ removes the local `.git` directories for each remote, so as to not upset your local Git tree. If you want to change the version/revision of a remote, just update your spec file and run `vdm sync` again. +After running `vdm sync` with the above example spec file, your directory tree +would look like this: + +``` +./vdm.json +./deps/ + go-common/ + + http.proto +``` + If for any reason you want all the deps in the spec file to retain their `.git` directories (such as if you're using `vdm` to initialize a new computer with actual repos you'd be working in), you can pass the `-keep-git-dir` flag to `vdm @@ -46,5 +61,4 @@ sync`. - Make the sync mechanism more robust, such that if your spec file changes to remove remotes, they'll get cleaned up automatically. -- Support more than just Git -- but I really don't care that much about this - right now. +- Support more than just Git diff --git a/common.go b/common.go index 62cc3e6..96dae39 100644 --- a/common.go +++ b/common.go @@ -2,21 +2,11 @@ package main import ( - "context" "flag" "fmt" - "log" - "os" "regexp" -) -const ( - // ANSI color codes for log messages - colorErr = "\033[31m" // red - colorDebug = "\033[33m" // yellow - colorInfo = "\033[36m" // cyan - colorHappy = "\033[32m" // green - colorReset = "\033[0m" + "github.com/sirupsen/logrus" ) var ( @@ -33,12 +23,6 @@ var ( // sync CLI flags keepGitDir bool - - // Loggers, which include embedded ANSI color codes - infoLogger = log.New(os.Stderr, fmt.Sprintf("%s%s[vdm]%s ", colorReset, colorInfo, colorReset), 0) - errLogger = log.New(os.Stderr, fmt.Sprintf("%s%s[vdm]%s ", colorReset, colorErr, colorReset), 0) - debugLogger = log.New(os.Stderr, fmt.Sprintf("%s%s[vdm]%s ", colorReset, colorDebug, colorReset), 0) - happyLogger = log.New(os.Stderr, fmt.Sprintf("%s%s[vdm]%s ", colorReset, colorHappy, colorReset), 0) ) // registerFlags assigns values to flags that should belong to each and/or all @@ -54,45 +38,6 @@ func registerFlags() { syncCmd.BoolVar(&keepGitDir, "keep-git-dir", false, "should vdm keep the .git directory within git-sourced directories? Most useful if you're using vdm to initialize groups of actual repositories you intend to work in") } -// Linter is mad about using string keys for context.Context, so define empty -// struct types for each usable key here -type debugKey struct{} -type specFilePathKey struct{} -type keepGitDirKey struct{} - -// registerContextKeys assigns common values to the context that is passed -// around, such as CLI flags -func registerContextKeys() context.Context { - ctx := context.Background() - ctx = context.WithValue(ctx, debugKey{}, debug) - ctx = context.WithValue(ctx, specFilePathKey{}, specFilePath) - ctx = context.WithValue(ctx, keepGitDirKey{}, keepGitDir) - - return ctx -} - -// isDebug checks against the passed context to determine if the debug CLI flag -// was set by the user -func isDebug(ctx context.Context) bool { - debugVal := ctx.Value(debugKey{}) - if debugVal == nil { - return false - } - - return debugVal.(bool) -} - -// shouldKeepGitDir checks against the passed context to determine if the -// keepGitDir CLI flag was set by the user -func shouldKeepGitDir(ctx context.Context) bool { - keepGitDirVal := ctx.Value(keepGitDirKey{}) - if keepGitDirVal == nil { - return false - } - - return keepGitDirVal.(bool) -} - // rootUsage has help text for the root command, so that users don't get an // unhelpful error when forgetting to specify a subcommand func showRootUsage() { @@ -109,6 +54,6 @@ func checkRootUsage(args []string) { helpFlagRegex := regexp.MustCompile(`\-?h(elp)?`) if len(args) == 1 || (len(args) == 2 && helpFlagRegex.MatchString(args[1])) { showRootUsage() - errLogger.Fatal("You must provide a command to vdm") + logrus.Fatal("You must provide a command to vdm") } } diff --git a/go.mod b/go.mod index fc53a8a..23ed57c 100644 --- a/go.mod +++ b/go.mod @@ -2,10 +2,14 @@ module github.com/opensourcecorp/vdm go 1.19 -require github.com/stretchr/testify v1.8.2 +require ( + github.com/sirupsen/logrus v1.9.3 + github.com/stretchr/testify v1.8.2 +) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6a56e69..fbbad56 100644 --- a/go.sum +++ b/go.sum @@ -3,13 +3,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 3977d25..c28167d 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,8 @@ package main import ( "os" + + "github.com/sirupsen/logrus" ) func main() { @@ -9,14 +11,12 @@ func main() { cmd, ok := subcommands[os.Args[1]] // length-guarded already by checkRootUsage() above if !ok { showRootUsage() - errLogger.Fatalf("Unrecognized vdm subcommand '%s'", os.Args[1]) + logrus.Fatalf("Unrecognized vdm subcommand '%s'", os.Args[1]) } registerFlags() cmd.Parse(os.Args[2:]) - ctx := registerContextKeys() - - err := checkGitAvailable(ctx) + err := checkGitAvailable(rf) if err != nil { os.Exit(1) } @@ -26,7 +26,7 @@ func main() { for _, spec := range specs { err := spec.Validate(ctx) if err != nil { - errLogger.Fatalf("Your vdm spec file is malformed: %v", err) + logrus.Fatalf("Your vdm spec file is malformed: %v", err) } } @@ -35,8 +35,8 @@ func main() { sync(ctx, specs) default: // should never get here since we check above, but still showRootUsage() - errLogger.Fatalf("Unrecognized vdm subcommand '%s'", cmd.Name()) + logrus.Fatalf("Unrecognized vdm subcommand '%s'", cmd.Name()) } - happyLogger.Print("All done!") + logrus.Info("All done!") } diff --git a/spec.go b/spec.go index 256d17f..f0ea4a9 100644 --- a/spec.go +++ b/spec.go @@ -6,6 +6,8 @@ import ( "errors" "os" "path/filepath" + + "github.com/sirupsen/logrus" ) type vdmSpec struct { @@ -34,30 +36,30 @@ func (spec vdmSpec) getVDMMeta() vdmSpec { if errors.Is(err, os.ErrNotExist) { return vdmSpec{} } else if err != nil { - errLogger.Fatalf("Couldn't check if VDMMETA exists at '%s': %v", metaFilePath, err) + logrus.Fatalf("Couldn't check if VDMMETA exists at '%s': %v", metaFilePath, err) } vdmMetaFile, err := os.ReadFile(filepath.Join(spec.LocalPath, "VDMMETA")) if err != nil { if debug { - debugLogger.Printf("error reading VMDMMETA from disk: %v", err) + logrus.Debugf("error reading VMDMMETA from disk: %v", err) } - errLogger.Fatalf("There was a problem reading the VDMMETA file from '%s': %v", metaFilePath, err) + logrus.Fatalf("There was a problem reading the VDMMETA file from '%s': %v", metaFilePath, err) } if debug { - debugLogger.Printf("VDMMETA contents read:\n%s", string(vdmMetaFile)) + logrus.Debugf("VDMMETA contents read:\n%s", string(vdmMetaFile)) } var vdmMeta vdmSpec err = json.Unmarshal(vdmMetaFile, &vdmMeta) if err != nil { if debug { - debugLogger.Printf("error during VDMMETA unmarshal: %v", err) + logrus.Debugf("error during VDMMETA unmarshal: %v", err) } - errLogger.Fatalf("There was a problem reading the contents of the VDMMETA file at '%s': %v", metaFilePath, err) + logrus.Fatalf("There was a problem reading the contents of the VDMMETA file at '%s': %v", metaFilePath, err) } if debug { - debugLogger.Printf("VDMMETA unmarshalled: %+v", vdmMeta) + logrus.Debugf("VDMMETA unmarshalled: %+v", vdmMeta) } return vdmMeta @@ -67,24 +69,24 @@ func getSpecsFromFile(ctx context.Context, specFilePath string) []vdmSpec { specFile, err := os.ReadFile(specFilePath) if err != nil { if isDebug(ctx) { - debugLogger.Printf("error reading specFile from disk: %v", err) + logrus.Debugf("error reading specFile from disk: %v", err) } - errLogger.Fatalf("There was a problem reading your vdm file from '%s' -- does it not exist? Either pass the -spec-file flag, or create one in the default location (details in the README)", specFilePath) + logrus.Fatalf("There was a problem reading your vdm file from '%s' -- does it not exist? Either pass the -spec-file flag, or create one in the default location (details in the README)", specFilePath) } if debug { - debugLogger.Printf("specFile contents read:\n%s", string(specFile)) + logrus.Debugf("specFile contents read:\n%s", string(specFile)) } var specs []vdmSpec err = json.Unmarshal(specFile, &specs) if err != nil { if isDebug(ctx) { - debugLogger.Printf("error during specFile unmarshal: %v", err) + logrus.Debugf("error during specFile unmarshal: %v", err) } - errLogger.Fatal("There was a problem reading the contents of your vdm spec file") + logrus.Fatal("There was a problem reading the contents of your vdm spec file") } if isDebug(ctx) { - debugLogger.Printf("vdmSpecs unmarshalled: %+v", specs) + logrus.Debugf("vdmSpecs unmarshalled: %+v", specs) } return specs diff --git a/sync.go b/sync.go index 244e506..d7e7528 100644 --- a/sync.go +++ b/sync.go @@ -6,6 +6,8 @@ import ( "os" "os/exec" "path/filepath" + + "github.com/sirupsen/logrus" ) // sync ensures that the only local dependencies are ones defined in the specfile @@ -17,13 +19,13 @@ func sync(ctx context.Context, specs []vdmSpec) { // process stored VDMMETA so we know what operations to actually perform for existing directories vdmMeta := spec.getVDMMeta() if vdmMeta == (vdmSpec{}) { - infoLogger.Printf("VDMMETA not found under local path '%s' -- will be created", spec.LocalPath) + logrus.Infof("VDMMETA not found under local path '%s' -- will be created", spec.LocalPath) } else { if vdmMeta.Version != spec.Version { - infoLogger.Printf("Changing '%s' from current local version spec '%s' to '%s'...", spec.Remote, vdmMeta.Version, spec.Version) + logrus.Infof("Changing '%s' from current local version spec '%s' to '%s'...", spec.Remote, vdmMeta.Version, spec.Version) } else { if isDebug(ctx) { - debugLogger.Printf("Version unchanged (%s) in spec file for '%s' --> '%s'", spec.Version, spec.Remote, spec.LocalPath) + logrus.Debugf("Version unchanged (%s) in spec file for '%s' --> '%s'", spec.Version, spec.Remote, spec.LocalPath) } } } @@ -32,15 +34,15 @@ func sync(ctx context.Context, specs []vdmSpec) { case "git", "": syncGitRemote(ctx, spec, opMsg) default: - errLogger.Fatalf("") + logrus.Fatalf("") } err := spec.writeVDMMeta() if err != nil { - errLogger.Fatalf("Could not write VDMMETA file to disk: %v", err) + logrus.Fatalf("Could not write VDMMETA file to disk: %v", err) } - infoLogger.Printf("%s -- Done.", opMsg) + logrus.Infof("%s -- Done.", opMsg) } } @@ -48,7 +50,7 @@ func syncGitRemote(ctx context.Context, spec vdmSpec, operationMsg string) { // TODO: pull this up so that it only runs if the version changed or the user requested a wipe if !shouldKeepGitDir(ctx) { if isDebug(ctx) { - debugLogger.Printf("removing any old data for '%s'", spec.LocalPath) + logrus.Debugf("removing any old data for '%s'", spec.LocalPath) } os.RemoveAll(spec.LocalPath) } @@ -56,17 +58,17 @@ func syncGitRemote(ctx context.Context, spec vdmSpec, operationMsg string) { gitClone(ctx, spec, operationMsg) if spec.Version != "latest" { - infoLogger.Printf("%s -- Setting specified version...", operationMsg) + logrus.Infof("%s -- Setting specified version...", operationMsg) checkoutCmd := exec.Command("git", "-C", spec.LocalPath, "checkout", spec.Version) checkoutOutput, err := checkoutCmd.CombinedOutput() if err != nil { - errLogger.Fatalf("error checking out specified revision: exec error '%v', with output: %s", err, string(checkoutOutput)) + logrus.Fatalf("error checking out specified revision: exec error '%v', with output: %s", err, string(checkoutOutput)) } } if !shouldKeepGitDir(ctx) { if isDebug(ctx) { - debugLogger.Printf("removing .git dir for local path '%s'", spec.LocalPath) + logrus.Debugf("removing .git dir for local path '%s'", spec.LocalPath) } os.RemoveAll(filepath.Join(spec.LocalPath, ".git")) } @@ -79,20 +81,20 @@ func gitClone(ctx context.Context, spec vdmSpec, operationMsg string) { var cloneCmdArgs []string if spec.Version == "latest" { if isDebug(ctx) { - debugLogger.Printf("%s -- version specified as 'latest', so making shallow clone and skipping separate checkout operation", operationMsg) + logrus.Debugf("%s -- version specified as 'latest', so making shallow clone and skipping separate checkout operation", operationMsg) } cloneCmdArgs = []string{"clone", "--depth=1", spec.Remote, spec.LocalPath} } else { if isDebug(ctx) { - debugLogger.Printf("%s -- version specified as NOT latest, so making regular clone and will make separate checkout operation", operationMsg) + logrus.Debugf("%s -- version specified as NOT latest, so making regular clone and will make separate checkout operation", operationMsg) } cloneCmdArgs = []string{"clone", spec.Remote, spec.LocalPath} } - infoLogger.Printf("%s -- Retrieving...", operationMsg) + logrus.Infof("%s -- Retrieving...", operationMsg) cloneCmd := exec.Command("git", cloneCmdArgs...) cloneOutput, err := cloneCmd.CombinedOutput() if err != nil { - errLogger.Fatalf("error cloning remote: exec error '%v', with output: %s", err, string(cloneOutput)) + logrus.Fatalf("error cloning remote: exec error '%v', with output: %s", err, string(cloneOutput)) } } diff --git a/system.go b/system.go index 684972a..1adf045 100644 --- a/system.go +++ b/system.go @@ -4,6 +4,8 @@ import ( "context" "errors" "os/exec" + + "github.com/sirupsen/logrus" ) func checkGitAvailable(ctx context.Context) error { @@ -11,12 +13,12 @@ func checkGitAvailable(ctx context.Context) error { sysOutput, err := cmd.CombinedOutput() if err != nil { if isDebug(ctx) { - debugLogger.Printf("%s: %s", err.Error(), string(sysOutput)) + logrus.Debugf("%s: %s", err.Error(), string(sysOutput)) } return errors.New("git does not seem to be available on your PATH, so cannot continue") } if isDebug(ctx) { - debugLogger.Print("git was found on PATH") + logrus.Debug("git was found on PATH") } return nil } diff --git a/validate.go b/validate.go index d2c2ded..dbfb821 100644 --- a/validate.go +++ b/validate.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "regexp" + + "github.com/sirupsen/logrus" ) func (spec vdmSpec) Validate(ctx context.Context) error { @@ -12,7 +14,7 @@ func (spec vdmSpec) Validate(ctx context.Context) error { // Remote field if isDebug(ctx) { - debugLogger.Printf("validating field 'Remote' for %+v", spec) + logrus.Debugf("validating field 'Remote' for %+v", spec) } if len(spec.Remote) == 0 { allErrors = append(allErrors, errors.New("all 'remote' fields must be non-zero length")) @@ -27,18 +29,18 @@ func (spec vdmSpec) Validate(ctx context.Context) error { // Version field if isDebug(ctx) { - debugLogger.Printf("validating field 'Version' for %+v", spec) + logrus.Debugf("validating field 'Version' for %+v", spec) } if spec.Type == "git" && len(spec.Version) == 0 { allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length. If you don't care about the version (even though you probably should), then use 'latest'")) } if spec.Type == "file" && len(spec.Version) > 0 { - infoLogger.Printf("NOTE: Remote %s specified as type '%s' but also specified version as '%s'; ignoring version field", spec.Remote, spec.Type, spec.Version) + logrus.Infof("NOTE: Remote %s specified as type '%s' but also specified version as '%s'; ignoring version field", spec.Remote, spec.Type, spec.Version) } // LocalPath field if isDebug(ctx) { - debugLogger.Printf("validating field 'LocalPath' for %+v", spec) + logrus.Debugf("validating field 'LocalPath' for %+v", spec) } if len(spec.LocalPath) == 0 { allErrors = append(allErrors, errors.New("all 'local_path' fields must be non-zero length")) @@ -46,7 +48,7 @@ func (spec vdmSpec) Validate(ctx context.Context) error { // Type field if isDebug(ctx) { - debugLogger.Printf("validating field 'Version' for %+v", spec) + logrus.Debugf("validating field 'Version' for %+v", spec) } typeMap := map[string]struct{}{ "git": {}, @@ -59,7 +61,7 @@ func (spec vdmSpec) Validate(ctx context.Context) error { if len(allErrors) > 0 { for _, err := range allErrors { - errLogger.Printf("validation failure: %s", err.Error()) + logrus.Errorf("validation failure: %s", err.Error()) } return fmt.Errorf("%d validation failure(s) found in your vdm spec file", len(allErrors)) } From 4fdd01f7664121d5657bf1281f793f3a26abaf62 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Mon, 3 Jul 2023 23:39:12 -0500 Subject: [PATCH 03/29] WIP --- Makefile | 3 +- README.md | 2 +- cmd/doc.go | 4 + cmd/root.go | 41 +++++++ cmd/sync.go | 83 +++++++++++++++ cmd/sync_test.go | 52 +++++++++ common.go | 59 ----------- go.mod | 3 + go.sum | 8 ++ internal/remote/doc.go | 4 + internal/remote/file.go | 1 + internal/remote/git.go | 87 +++++++++++++++ system_test.go => internal/remote/git_test.go | 11 +- internal/vdmspec/doc.go | 4 + internal/vdmspec/spec.go | 91 ++++++++++++++++ internal/vdmspec/spec_test.go | 79 ++++++++++++++ validate.go => internal/vdmspec/validate.go | 21 ++-- .../vdmspec/validate_test.go | 29 +++-- main.go | 36 +------ spec.go | 93 ---------------- spec_test.go | 68 ------------ sync.go | 100 ------------------ sync_test.go | 69 ------------ system.go | 24 ----- 24 files changed, 486 insertions(+), 486 deletions(-) create mode 100644 cmd/doc.go create mode 100644 cmd/root.go create mode 100644 cmd/sync.go create mode 100644 cmd/sync_test.go delete mode 100644 common.go create mode 100644 internal/remote/doc.go create mode 100644 internal/remote/file.go create mode 100644 internal/remote/git.go rename system_test.go => internal/remote/git_test.go (58%) create mode 100644 internal/vdmspec/doc.go create mode 100644 internal/vdmspec/spec.go create mode 100644 internal/vdmspec/spec_test.go rename validate.go => internal/vdmspec/validate.go (80%) rename validate_test.go => internal/vdmspec/validate_test.go (78%) delete mode 100644 spec.go delete mode 100644 spec_test.go delete mode 100644 sync.go delete mode 100644 sync_test.go delete mode 100644 system.go diff --git a/Makefile b/Makefile index 714aaab..d47bff7 100644 --- a/Makefile +++ b/Makefile @@ -8,11 +8,12 @@ all: test clean test: clean go vet ./... - go test -cover -coverprofile=./cover.out ./... + go test -v ./... staticcheck ./... make -s clean test-coverage: test + go test -cover -coverprofile=./cover.out ./... go tool cover -html=./cover.out -o cover.html xdg-open ./cover.html diff --git a/README.md b/README.md index 0157480..874ff1a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ them to live in your repo: { "type": "file", // the 'file' type assumes the version is in the remote field itself, so 'version' can be omitted "remote": "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto", - "local_path": "./deps/http.proto" + "local_path": "./deps/proto/http/http.proto" } ] ``` diff --git a/cmd/doc.go b/cmd/doc.go new file mode 100644 index 0000000..669e3cf --- /dev/null +++ b/cmd/doc.go @@ -0,0 +1,4 @@ +/* +Package cmd calls the implementation logic for vdm. +*/ +package cmd diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..c81edf3 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,41 @@ +package cmd + +import ( + "fmt" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var rootCmd = cobra.Command{ + Use: "vdm", + Short: "vdm -- a Versioned-Dependency Manager", + Long: "vdm is used to manage arbitrary remote depdencies in a code repository", + TraverseChildren: true, +} + +type RootFlags struct { + SpecFilePath string + Debug bool +} + +var RootFlagValues RootFlags + +func init() { + rootCmd.PersistentFlags().StringVar(&RootFlagValues.SpecFilePath, "specfile-path", "./vdm.json", "Path to vdm specfile") + rootCmd.PersistentFlags().BoolVar(&RootFlagValues.Debug, "debug", false, "Show debug logs") + + if RootFlagValues.Debug { + logrus.SetLevel(logrus.DebugLevel) + } + + rootCmd.AddCommand(syncCmd) +} + +func Execute() error { + if err := rootCmd.Execute(); err != nil { + return fmt.Errorf("executing root command: %v", err) + } + + return nil +} diff --git a/cmd/sync.go b/cmd/sync.go new file mode 100644 index 0000000..3c3edf6 --- /dev/null +++ b/cmd/sync.go @@ -0,0 +1,83 @@ +package cmd + +import ( + "fmt" + "path/filepath" + + "github.com/opensourcecorp/vdm/internal/remote" + "github.com/opensourcecorp/vdm/internal/vdmspec" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var syncCmd = &cobra.Command{ + Use: "sync", + Short: "Sync remotes based on specfile", + RunE: syncExecute, +} + +type SyncFlags struct { + KeepGitDir bool +} + +var SyncFlagValues SyncFlags + +func syncExecute(_ *cobra.Command, _ []string) error { + if err := sync(); err != nil { + return fmt.Errorf("executing sync command: %w", err) + } + return nil +} + +// sync ensures that the only local dependencies are ones defined in the specfile +func sync() error { + specs, err := vdmspec.GetSpecsFromFile(RootFlagValues.SpecFilePath) + if err != nil { + return fmt.Errorf("getting specs from spec file: %w", err) + } + + for _, spec := range specs { + err := spec.Validate() + if err != nil { + return fmt.Errorf("your vdm spec file is malformed: %w", err) + } + } + + for _, spec := range specs { + // process stored vdm metafile so we know what operations to actually + // perform for existing directories + vdmMeta, err := spec.GetVDMMeta() + if err != nil { + return fmt.Errorf("getting vdm metadata file for sync: %w", err) + } + + if vdmMeta == (vdmspec.VDMSpec{}) { + logrus.Infof("%s not found at local path '%s' -- will be created", vdmspec.MetaFileName, filepath.Join(spec.LocalPath)) + } else { + if vdmMeta.Version != spec.Version { + logrus.Infof("Changing '%s' from current local version spec '%s' to '%s'...", spec.Remote, vdmMeta.Version, spec.Version) + } else { + logrus.Debugf("Version unchanged (%s) in spec file for '%s' --> '%s'", spec.Version, spec.Remote, spec.LocalPath) + } + } + + switch spec.Type { + case "git", "": + remote.SyncGit(spec) + case "file": + remote.SyncFile(spec) + default: + return fmt.Errorf("unrecognized remote type '%s'", spec.Type) + } + + err = spec.WriteVDMMeta() + if err != nil { + return fmt.Errorf("could not write %s file to disk: %w", vdmspec.MetaFileName, err) + } + + logrus.Infof("%s -- Done.", spec.OpMsg()) + } + + logrus.Info("All done!") + return nil +} diff --git a/cmd/sync_test.go b/cmd/sync_test.go new file mode 100644 index 0000000..e1e5886 --- /dev/null +++ b/cmd/sync_test.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/opensourcecorp/vdm/internal/vdmspec" + "github.com/stretchr/testify/assert" +) + +func TestSync(t *testing.T) { + const testVDMRoot = "./testdata" + + specFilePath := filepath.Join(testVDMRoot, "vdm.json") + + specs, err := vdmspec.GetSpecsFromFile(specFilePath) + assert.NoError(t, err) + + sync() + + t.Run("spec[0] used a tag", func(t *testing.T) { + vdmMeta, err := specs[0].GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, vdmMeta.Version, "v0.2.0") + }) + + t.Run("spec[1] used 'latest'", func(t *testing.T) { + vdmMeta, err := specs[1].GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, vdmMeta.Version, "latest") + }) + + t.Run("spec[2] used a branch", func(t *testing.T) { + vdmMeta, err := specs[2].GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, vdmMeta.Version, "main") + }) + + t.Run("spec[4] used a hash", func(t *testing.T) { + vdmMeta, err := specs[3].GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, vdmMeta.Version, "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f") + }) + + t.Cleanup(func() { + for _, spec := range specs { + err := os.RemoveAll(spec.LocalPath) + assert.NoError(t, err) + } + }) +} diff --git a/common.go b/common.go deleted file mode 100644 index 96dae39..0000000 --- a/common.go +++ /dev/null @@ -1,59 +0,0 @@ -// Common and/or initialization consts, vars, and functions -package main - -import ( - "flag" - "fmt" - "regexp" - - "github.com/sirupsen/logrus" -) - -var ( - // Subcommands - syncCmd = flag.NewFlagSet("sync", flag.ExitOnError) - - subcommands = map[string]*flag.FlagSet{ - syncCmd.Name(): syncCmd, - } - - // CLI args common to each subcommand - debug bool - specFilePath string - - // sync CLI flags - keepGitDir bool -) - -// registerFlags assigns values to flags that should belong to each and/or all -// command(s) -func registerFlags() { - // common - for _, cmd := range subcommands { - cmd.StringVar(&specFilePath, "spec-file", "./vdm.json", "Path to vdm spec file") - cmd.BoolVar(&debug, "debug", false, "Print debug logs") - } - - // sync - syncCmd.BoolVar(&keepGitDir, "keep-git-dir", false, "should vdm keep the .git directory within git-sourced directories? Most useful if you're using vdm to initialize groups of actual repositories you intend to work in") -} - -// rootUsage has help text for the root command, so that users don't get an -// unhelpful error when forgetting to specify a subcommand -func showRootUsage() { - fmt.Printf(`vdm declaratively manages remote dependencies as local directories. - -Subcommands: - sync sync local paths based on your vdm spec file - -`) -} - -// checkRootUsage prints usage information if a user doesn't specify a subcommand -func checkRootUsage(args []string) { - helpFlagRegex := regexp.MustCompile(`\-?h(elp)?`) - if len(args) == 1 || (len(args) == 2 && helpFlagRegex.MatchString(args[1])) { - showRootUsage() - logrus.Fatal("You must provide a command to vdm") - } -} diff --git a/go.mod b/go.mod index 23ed57c..00cee8f 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,15 @@ go 1.19 require ( github.com/sirupsen/logrus v1.9.3 + github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.2 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index fbbad56..348225d 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,18 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/internal/remote/doc.go b/internal/remote/doc.go new file mode 100644 index 0000000..256b804 --- /dev/null +++ b/internal/remote/doc.go @@ -0,0 +1,4 @@ +/* +Package remote defines logic for the various types of remotes that vdm supports. +*/ +package remote diff --git a/internal/remote/file.go b/internal/remote/file.go new file mode 100644 index 0000000..fbe5b64 --- /dev/null +++ b/internal/remote/file.go @@ -0,0 +1 @@ +package remote diff --git a/internal/remote/git.go b/internal/remote/git.go new file mode 100644 index 0000000..ae44931 --- /dev/null +++ b/internal/remote/git.go @@ -0,0 +1,87 @@ +package remote + +import ( + "errors" + "fmt" + "os/exec" + + "github.com/opensourcecorp/vdm/internal/vdmspec" + "github.com/sirupsen/logrus" +) + +func checkGitAvailable() error { + cmd := exec.Command("git", "--version") + sysOutput, err := cmd.CombinedOutput() + if err != nil { + logrus.Debugf("%s: %s", err.Error(), string(sysOutput)) + return errors.New("git does not seem to be available on your PATH, so cannot continue") + } + logrus.Debug("git was found on PATH") + return nil +} + +func gitClone(spec vdmspec.VDMSpec) error { + err := checkGitAvailable() + if err != nil { + return fmt.Errorf("remote '%s' is a git type, but git may not installed/available on PATH: %w", spec.Remote, err) + } + + // If users want "latest", then we can just do a depth-one clone and + // skip the checkout operation. But if they want non-latest, we need the + // full history to be able to find a specified revision + var cloneCmdArgs []string + if spec.Version == "latest" { + logrus.Debugf("%s -- version specified as 'latest', so making shallow clone and skipping separate checkout operation", spec.OpMsg()) + cloneCmdArgs = []string{"clone", "--depth=1", spec.Remote, spec.LocalPath} + } else { + logrus.Debugf("%s -- version specified as NOT latest, so making regular clone and will make separate checkout operation", spec.OpMsg()) + cloneCmdArgs = []string{"clone", spec.Remote, spec.LocalPath} + } + + logrus.Infof("%s -- Retrieving...", spec.OpMsg()) + cloneCmd := exec.Command("git", cloneCmdArgs...) + cloneOutput, err := cloneCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("cloning remote: exec error '%w', with output: %s", err, string(cloneOutput)) + } + + return nil +} + +// SyncGit is the root of the sync operations for "git" remote types. +func SyncGit(spec vdmspec.VDMSpec) error { + // // TODO: pull this up so that it only runs if the version changed or the user requested a wipe + // if !cmd.SyncFlagValues.KeepGitDir { + // logrus.Debugf("removing any old data for '%s'", spec.LocalPath) + // os.RemoveAll(spec.LocalPath) + // } + + err := gitClone(spec) + if err != nil { + return fmt.Errorf("cloing remote: %w", err) + } + + if spec.Version != "latest" { + logrus.Infof("%s -- Setting specified version...", spec.OpMsg()) + checkoutCmd := exec.Command("git", "-C", spec.LocalPath, "checkout", spec.Version) + checkoutOutput, err := checkoutCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error checking out specified revision: exec error '%w', with output: %s", err, string(checkoutOutput)) + } + } + + // if cmd.SyncFlagValues.KeepGitDir { + // logrus.Debugf("removing .git dir for local path '%s'", spec.LocalPath) + // err := os.RemoveAll(filepath.Join(spec.LocalPath, ".git")) + // if err != nil { + // return fmt.Errorf("removing ") + // } + // } + + return nil +} + +// SyncFile is the root of the sync operations for "file" remote types. +func SyncFile(spec vdmspec.VDMSpec) error { + panic("not implemented") +} diff --git a/system_test.go b/internal/remote/git_test.go similarity index 58% rename from system_test.go rename to internal/remote/git_test.go index 47d3edb..281a5c9 100644 --- a/system_test.go +++ b/internal/remote/git_test.go @@ -1,22 +1,19 @@ -package main +package remote import ( - "context" "testing" "github.com/stretchr/testify/assert" ) -func TestCheckAvailable(t *testing.T) { - ctx := context.Background() - +func TestCheckGitAvailable(t *testing.T) { t.Run("git", func(t *testing.T) { // Host of this test better have git available lol - gitAvailable := checkGitAvailable(ctx) + gitAvailable := checkGitAvailable() assert.NoError(t, gitAvailable) t.Setenv("PATH", "") - gitAvailable = checkGitAvailable(ctx) + gitAvailable = checkGitAvailable() assert.Error(t, gitAvailable) }) } diff --git a/internal/vdmspec/doc.go b/internal/vdmspec/doc.go new file mode 100644 index 0000000..f9df9ce --- /dev/null +++ b/internal/vdmspec/doc.go @@ -0,0 +1,4 @@ +/* +Package vdmspec defines the [VDMSpec] struct type, and its associated methods. +*/ +package vdmspec diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go new file mode 100644 index 0000000..d0d58de --- /dev/null +++ b/internal/vdmspec/spec.go @@ -0,0 +1,91 @@ +package vdmspec + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" +) + +type VDMSpec struct { + Remote string `json:"remote"` + Version string `json:"version,omitempty"` + LocalPath string `json:"local_path"` + Type string `json:"type,omitempty"` +} + +const MetaFileName = "VDMMETA" + +func (spec VDMSpec) WriteVDMMeta() error { + metaFilePath := filepath.Join(spec.LocalPath, MetaFileName) + vdmMetaContent, err := json.MarshalIndent(spec, "", " ") + if err != nil { + return fmt.Errorf("writing %s: %w", metaFilePath, err) + } + + vdmMetaContent = append(vdmMetaContent, []byte("\n")...) + + logrus.Debugf("writing metadata file to '%s'", metaFilePath) + err = os.WriteFile(metaFilePath, vdmMetaContent, 0644) + if err != nil { + logrus.Debug("error here") + return fmt.Errorf("writing metadata file: %w", err) + } + + return nil +} + +func (spec VDMSpec) GetVDMMeta() (VDMSpec, error) { + metaFilePath := filepath.Join(spec.LocalPath, MetaFileName) + _, err := os.Stat(metaFilePath) + if errors.Is(err, os.ErrNotExist) { + return VDMSpec{}, nil // this is ok, because it might literally not exist yet + } else if err != nil { + return VDMSpec{}, fmt.Errorf("couldn't check if %s exists at '%s': %w", MetaFileName, metaFilePath, err) + } + + vdmMetaFile, err := os.ReadFile(filepath.Join(spec.LocalPath, MetaFileName)) + if err != nil { + logrus.Debugf("error reading VMDMMETA from disk: %v", err) + return VDMSpec{}, fmt.Errorf("there was a problem reading the %s file from '%s': %w", MetaFileName, metaFilePath, err) + } + logrus.Debugf("%s contents read:\n%s", MetaFileName, string(vdmMetaFile)) + + var vdmMeta VDMSpec + err = json.Unmarshal(vdmMetaFile, &vdmMeta) + if err != nil { + logrus.Debugf("error during %s unmarshal: %v", MetaFileName, err) + return VDMSpec{}, fmt.Errorf("there was a problem reading the contents of the %s file at '%s': %v", MetaFileName, metaFilePath, err) + } + logrus.Debugf("file %s unmarshalled: %+v", MetaFileName, vdmMeta) + + return vdmMeta, nil +} + +func GetSpecsFromFile(specFilePath string) ([]VDMSpec, error) { + specFile, err := os.ReadFile(specFilePath) + if err != nil { + logrus.Debugf("error reading specFile from disk: %v", err) + logrus.Fatalf("There was a problem reading your vdm file from '%s' -- does it not exist? Either pass the -spec-file flag, or create one in the default location (details in the README)", specFilePath) + } + logrus.Debugf("specFile contents read:\n%s", string(specFile)) + + var specs []VDMSpec + err = json.Unmarshal(specFile, &specs) + if err != nil { + logrus.Debugf("error during specFile unmarshal: %v", err) + logrus.Fatal("There was a problem reading the contents of your vdm spec file") + } + logrus.Debugf("vdmSpecs unmarshalled: %+v", specs) + + return specs, nil +} + +// OpMsg constructs a loggable message outlining the specific operation being +// performed at the moment +func (spec VDMSpec) OpMsg() string { + return fmt.Sprintf("%s@%s --> %s", spec.Remote, spec.Version, spec.LocalPath) +} diff --git a/internal/vdmspec/spec_test.go b/internal/vdmspec/spec_test.go new file mode 100644 index 0000000..7fca3ad --- /dev/null +++ b/internal/vdmspec/spec_test.go @@ -0,0 +1,79 @@ +package vdmspec + +import ( + "os" + "path/filepath" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func init() { + logrus.SetLevel(logrus.DebugLevel) +} + +func TestVDMMeta(t *testing.T) { + const testVDMRoot = "./testdata" + testVDMMetaFilePath := filepath.Join(testVDMRoot, MetaFileName) + + t.Run("GetVDMMeta", func(t *testing.T) { + spec := VDMSpec{ + Remote: "https://some-remote", + Version: "v1.0.0", + LocalPath: "./testdata", + } + vdmMetaContents := ` + { + "remote": "https://some-remote", + "version": "v1.0.0", + "local_path": "./testdata" + }` + err := os.WriteFile(testVDMMetaFilePath, []byte(vdmMetaContents), 0644) + if err != nil { + t.Fatal(err) + } + + got, err := spec.GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, spec, got) + + t.Cleanup(func() { + err := os.RemoveAll(testVDMMetaFilePath) + assert.NoError(t, err) + }) + }) + + t.Run("WriteVDMMeta", func(t *testing.T) { + spec := VDMSpec{ + Remote: "https://some-remote", + Version: "v1.0.0", + LocalPath: "./testdata", + } + err := spec.WriteVDMMeta() + require.NoError(t, err) + + got, err := spec.GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, spec, got) + + t.Cleanup(func() { + err := os.RemoveAll(testVDMMetaFilePath) + assert.NoError(t, err) + }) + }) + + t.Run("GetSpecsFromFile", func(t *testing.T) { + specFilePath := "./testdata/vdm.json" + + specs, err := GetSpecsFromFile(specFilePath) + assert.NoError(t, err) + assert.Equal(t, 5, len(specs)) + + t.Cleanup(func() { + err := os.RemoveAll(testVDMMetaFilePath) + assert.NoError(t, err) + }) + }) +} diff --git a/validate.go b/internal/vdmspec/validate.go similarity index 80% rename from validate.go rename to internal/vdmspec/validate.go index dbfb821..6414014 100644 --- a/validate.go +++ b/internal/vdmspec/validate.go @@ -1,7 +1,6 @@ -package main +package vdmspec import ( - "context" "errors" "fmt" "regexp" @@ -9,13 +8,11 @@ import ( "github.com/sirupsen/logrus" ) -func (spec vdmSpec) Validate(ctx context.Context) error { +func (spec VDMSpec) Validate() error { var allErrors []error // Remote field - if isDebug(ctx) { - logrus.Debugf("validating field 'Remote' for %+v", spec) - } + logrus.Debugf("validating field 'Remote' for %+v", spec) if len(spec.Remote) == 0 { allErrors = append(allErrors, errors.New("all 'remote' fields must be non-zero length")) } @@ -28,9 +25,7 @@ func (spec vdmSpec) Validate(ctx context.Context) error { } // Version field - if isDebug(ctx) { - logrus.Debugf("validating field 'Version' for %+v", spec) - } + logrus.Debugf("validating field 'Version' for %+v", spec) if spec.Type == "git" && len(spec.Version) == 0 { allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length. If you don't care about the version (even though you probably should), then use 'latest'")) } @@ -39,17 +34,13 @@ func (spec vdmSpec) Validate(ctx context.Context) error { } // LocalPath field - if isDebug(ctx) { - logrus.Debugf("validating field 'LocalPath' for %+v", spec) - } + logrus.Debugf("validating field 'LocalPath' for %+v", spec) if len(spec.LocalPath) == 0 { allErrors = append(allErrors, errors.New("all 'local_path' fields must be non-zero length")) } // Type field - if isDebug(ctx) { - logrus.Debugf("validating field 'Version' for %+v", spec) - } + logrus.Debugf("validating field 'Version' for %+v", spec) typeMap := map[string]struct{}{ "git": {}, "": {}, // also git diff --git a/validate_test.go b/internal/vdmspec/validate_test.go similarity index 78% rename from validate_test.go rename to internal/vdmspec/validate_test.go index 2a92517..33d42a1 100644 --- a/validate_test.go +++ b/internal/vdmspec/validate_test.go @@ -1,74 +1,71 @@ -package main +package vdmspec import ( - "context" "testing" "github.com/stretchr/testify/assert" ) func TestValidate(t *testing.T) { - ctx := context.Background() - t.Run("passes", func(t *testing.T) { - spec := vdmSpec{ + spec := VDMSpec{ Remote: "https://some-remote", Version: "v1.0.0", LocalPath: "./deps/some-remote", } - err := spec.Validate(ctx) + err := spec.Validate() assert.NoError(t, err) }) t.Run("fails on zero-length remote", func(t *testing.T) { - spec := vdmSpec{ + spec := VDMSpec{ Remote: "", Version: "v1.0.0", LocalPath: "./deps/some-remote", } - err := spec.Validate(ctx) + err := spec.Validate() assert.Error(t, err) }) t.Run("fails on remote without valid protocol", func(t *testing.T) { - spec := vdmSpec{ + spec := VDMSpec{ Remote: "some-remote", Version: "v1.0.0", LocalPath: "./deps/some-remote", } - err := spec.Validate(ctx) + err := spec.Validate() assert.Error(t, err) }) t.Run("fails on zero-length version for git remote type", func(t *testing.T) { - spec := vdmSpec{ + spec := VDMSpec{ Remote: "https://some-remote", Version: "", LocalPath: "./deps/some-remote", Type: "git", } - err := spec.Validate(ctx) + err := spec.Validate() assert.Error(t, err) }) t.Run("fails on unrecognized remote type", func(t *testing.T) { - spec := vdmSpec{ + spec := VDMSpec{ Remote: "https://some-remote", Version: "", LocalPath: "./deps/some-remote", Type: "bad", } - err := spec.Validate(ctx) + err := spec.Validate() assert.Error(t, err) }) t.Run("fails on zero-length local path", func(t *testing.T) { - spec := vdmSpec{ + spec := VDMSpec{ Remote: "https://some-remote", Version: "v1.0.0", LocalPath: "", } - err := spec.Validate(ctx) + err := spec.Validate() assert.Error(t, err) }) } diff --git a/main.go b/main.go index c28167d..1fa94f7 100644 --- a/main.go +++ b/main.go @@ -1,42 +1,12 @@ package main import ( - "os" - + "github.com/opensourcecorp/vdm/cmd" "github.com/sirupsen/logrus" ) func main() { - checkRootUsage(os.Args) - cmd, ok := subcommands[os.Args[1]] // length-guarded already by checkRootUsage() above - if !ok { - showRootUsage() - logrus.Fatalf("Unrecognized vdm subcommand '%s'", os.Args[1]) - } - registerFlags() - cmd.Parse(os.Args[2:]) - - err := checkGitAvailable(rf) - if err != nil { - os.Exit(1) + if err := cmd.Execute(); err != nil { + logrus.Fatalf("running vdm: %v", err) } - - specs := getSpecsFromFile(ctx, specFilePath) - - for _, spec := range specs { - err := spec.Validate(ctx) - if err != nil { - logrus.Fatalf("Your vdm spec file is malformed: %v", err) - } - } - - switch cmd.Name() { - case syncCmd.Name(): - sync(ctx, specs) - default: // should never get here since we check above, but still - showRootUsage() - logrus.Fatalf("Unrecognized vdm subcommand '%s'", cmd.Name()) - } - - logrus.Info("All done!") } diff --git a/spec.go b/spec.go deleted file mode 100644 index f0ea4a9..0000000 --- a/spec.go +++ /dev/null @@ -1,93 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "os" - "path/filepath" - - "github.com/sirupsen/logrus" -) - -type vdmSpec struct { - Remote string `json:"remote"` - Version string `json:"version"` - LocalPath string `json:"local_path"` - Type string `json:"type"` -} - -func (spec vdmSpec) writeVDMMeta() error { - metaFilePath := filepath.Join(spec.LocalPath, "VDMMETA") - vdmMetaContent, err := json.MarshalIndent(spec, "", " ") - if err != nil { - return err - } - - vdmMetaContent = append(vdmMetaContent, []byte("\n")...) - os.WriteFile(metaFilePath, vdmMetaContent, 0644) - - return nil -} - -func (spec vdmSpec) getVDMMeta() vdmSpec { - metaFilePath := filepath.Join(spec.LocalPath, "VDMMETA") - _, err := os.Stat(metaFilePath) - if errors.Is(err, os.ErrNotExist) { - return vdmSpec{} - } else if err != nil { - logrus.Fatalf("Couldn't check if VDMMETA exists at '%s': %v", metaFilePath, err) - } - - vdmMetaFile, err := os.ReadFile(filepath.Join(spec.LocalPath, "VDMMETA")) - if err != nil { - if debug { - logrus.Debugf("error reading VMDMMETA from disk: %v", err) - } - logrus.Fatalf("There was a problem reading the VDMMETA file from '%s': %v", metaFilePath, err) - } - if debug { - logrus.Debugf("VDMMETA contents read:\n%s", string(vdmMetaFile)) - } - - var vdmMeta vdmSpec - err = json.Unmarshal(vdmMetaFile, &vdmMeta) - if err != nil { - if debug { - logrus.Debugf("error during VDMMETA unmarshal: %v", err) - } - logrus.Fatalf("There was a problem reading the contents of the VDMMETA file at '%s': %v", metaFilePath, err) - } - if debug { - logrus.Debugf("VDMMETA unmarshalled: %+v", vdmMeta) - } - - return vdmMeta -} - -func getSpecsFromFile(ctx context.Context, specFilePath string) []vdmSpec { - specFile, err := os.ReadFile(specFilePath) - if err != nil { - if isDebug(ctx) { - logrus.Debugf("error reading specFile from disk: %v", err) - } - logrus.Fatalf("There was a problem reading your vdm file from '%s' -- does it not exist? Either pass the -spec-file flag, or create one in the default location (details in the README)", specFilePath) - } - if debug { - logrus.Debugf("specFile contents read:\n%s", string(specFile)) - } - - var specs []vdmSpec - err = json.Unmarshal(specFile, &specs) - if err != nil { - if isDebug(ctx) { - logrus.Debugf("error during specFile unmarshal: %v", err) - } - logrus.Fatal("There was a problem reading the contents of your vdm spec file") - } - if isDebug(ctx) { - logrus.Debugf("vdmSpecs unmarshalled: %+v", specs) - } - - return specs -} diff --git a/spec_test.go b/spec_test.go deleted file mode 100644 index e967911..0000000 --- a/spec_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package main - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSpecGetVDMMeta(t *testing.T) { - const testVDMRoot = "./testdata" - testVDMMetaFilePath := filepath.Join(testVDMRoot, "VDMMETA") - - t.Run("getVDMMeta", func(t *testing.T) { - spec := vdmSpec{ - Remote: "https://some-remote", - Version: "v1.0.0", - LocalPath: "./testdata", - } - vdmMetaContents := ` - { - "remote": "https://some-remote", - "version": "v1.0.0", - "local_path": "./testdata" - }` - err := os.WriteFile(testVDMMetaFilePath, []byte(vdmMetaContents), 0644) - if err != nil { - t.Fatal(err) - } - - got := spec.getVDMMeta() - assert.Equal(t, spec, got) - - t.Cleanup(func() { - os.RemoveAll(testVDMMetaFilePath) - }) - }) - - t.Run("writeVDMMeta", func(t *testing.T) { - spec := vdmSpec{ - Remote: "https://some-remote", - Version: "v1.0.0", - LocalPath: "./testdata", - } - err := spec.writeVDMMeta() - assert.NoError(t, err) - - got := spec.getVDMMeta() - assert.Equal(t, spec, got) - - t.Cleanup(func() { - os.RemoveAll(testVDMMetaFilePath) - }) - }) - - t.Run("getSpecsFromFile", func(t *testing.T) { - specFilePath := "./testdata/vdm.json" - - specs := getSpecsFromFile(context.Background(), specFilePath) - assert.Equal(t, 5, len(specs)) - - t.Cleanup(func() { - os.RemoveAll(testVDMMetaFilePath) - }) - }) -} diff --git a/sync.go b/sync.go deleted file mode 100644 index d7e7528..0000000 --- a/sync.go +++ /dev/null @@ -1,100 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - - "github.com/sirupsen/logrus" -) - -// sync ensures that the only local dependencies are ones defined in the specfile -func sync(ctx context.Context, specs []vdmSpec) { - for _, spec := range specs { - // Common log line prefix - opMsg := fmt.Sprintf("%s@%s --> %s", spec.Remote, spec.Version, spec.LocalPath) - - // process stored VDMMETA so we know what operations to actually perform for existing directories - vdmMeta := spec.getVDMMeta() - if vdmMeta == (vdmSpec{}) { - logrus.Infof("VDMMETA not found under local path '%s' -- will be created", spec.LocalPath) - } else { - if vdmMeta.Version != spec.Version { - logrus.Infof("Changing '%s' from current local version spec '%s' to '%s'...", spec.Remote, vdmMeta.Version, spec.Version) - } else { - if isDebug(ctx) { - logrus.Debugf("Version unchanged (%s) in spec file for '%s' --> '%s'", spec.Version, spec.Remote, spec.LocalPath) - } - } - } - - switch spec.Type { - case "git", "": - syncGitRemote(ctx, spec, opMsg) - default: - logrus.Fatalf("") - } - - err := spec.writeVDMMeta() - if err != nil { - logrus.Fatalf("Could not write VDMMETA file to disk: %v", err) - } - - logrus.Infof("%s -- Done.", opMsg) - } -} - -func syncGitRemote(ctx context.Context, spec vdmSpec, operationMsg string) { - // TODO: pull this up so that it only runs if the version changed or the user requested a wipe - if !shouldKeepGitDir(ctx) { - if isDebug(ctx) { - logrus.Debugf("removing any old data for '%s'", spec.LocalPath) - } - os.RemoveAll(spec.LocalPath) - } - - gitClone(ctx, spec, operationMsg) - - if spec.Version != "latest" { - logrus.Infof("%s -- Setting specified version...", operationMsg) - checkoutCmd := exec.Command("git", "-C", spec.LocalPath, "checkout", spec.Version) - checkoutOutput, err := checkoutCmd.CombinedOutput() - if err != nil { - logrus.Fatalf("error checking out specified revision: exec error '%v', with output: %s", err, string(checkoutOutput)) - } - } - - if !shouldKeepGitDir(ctx) { - if isDebug(ctx) { - logrus.Debugf("removing .git dir for local path '%s'", spec.LocalPath) - } - os.RemoveAll(filepath.Join(spec.LocalPath, ".git")) - } -} - -func gitClone(ctx context.Context, spec vdmSpec, operationMsg string) { - // If users want "latest", then we can just do a depth-one clone and - // skip the checkout operation. But if they want non-latest, we need the - // full history to be able to find a specified revision - var cloneCmdArgs []string - if spec.Version == "latest" { - if isDebug(ctx) { - logrus.Debugf("%s -- version specified as 'latest', so making shallow clone and skipping separate checkout operation", operationMsg) - } - cloneCmdArgs = []string{"clone", "--depth=1", spec.Remote, spec.LocalPath} - } else { - if isDebug(ctx) { - logrus.Debugf("%s -- version specified as NOT latest, so making regular clone and will make separate checkout operation", operationMsg) - } - cloneCmdArgs = []string{"clone", spec.Remote, spec.LocalPath} - } - - logrus.Infof("%s -- Retrieving...", operationMsg) - cloneCmd := exec.Command("git", cloneCmdArgs...) - cloneOutput, err := cloneCmd.CombinedOutput() - if err != nil { - logrus.Fatalf("error cloning remote: exec error '%v', with output: %s", err, string(cloneOutput)) - } -} diff --git a/sync_test.go b/sync_test.go deleted file mode 100644 index bc5741b..0000000 --- a/sync_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSync(t *testing.T) { - ctx := context.Background() - - const testVDMRoot = "./testdata" - specFilePath := filepath.Join(testVDMRoot, "vdm.json") - - specs := getSpecsFromFile(ctx, specFilePath) - - sync(ctx, specs) - - t.Run("spec[0] used a tag", func(t *testing.T) { - vdmMeta := specs[0].getVDMMeta() - assert.Equal(t, vdmMeta.Version, "v0.2.0") - }) - - t.Run("spec[1] used 'latest'", func(t *testing.T) { - vdmMeta := specs[1].getVDMMeta() - assert.Equal(t, vdmMeta.Version, "latest") - }) - - t.Run("spec[2] used a branch", func(t *testing.T) { - vdmMeta := specs[2].getVDMMeta() - assert.Equal(t, vdmMeta.Version, "main") - }) - - t.Run("spec[4] used a hash", func(t *testing.T) { - vdmMeta := specs[3].getVDMMeta() - assert.Equal(t, vdmMeta.Version, "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f") - }) - - t.Cleanup(func() { - for _, spec := range specs { - os.RemoveAll(spec.LocalPath) - } - }) -} - -func TestShouldKeepGitDir(t *testing.T) { - ctx := context.Background() - ctx = context.WithValue(ctx, keepGitDirKey{}, true) - - const testVDMRoot = "./testdata" - specFilePath := filepath.Join(testVDMRoot, "vdm.json") - - specs := getSpecsFromFile(ctx, specFilePath) - - sync(ctx, specs) - - for _, spec := range specs { - assert.DirExists(t, filepath.Join(spec.LocalPath, ".git")) - } - - t.Cleanup(func() { - for _, spec := range specs { - os.RemoveAll(spec.LocalPath) - } - }) -} diff --git a/system.go b/system.go deleted file mode 100644 index 1adf045..0000000 --- a/system.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "context" - "errors" - "os/exec" - - "github.com/sirupsen/logrus" -) - -func checkGitAvailable(ctx context.Context) error { - cmd := exec.Command("git", "--version") - sysOutput, err := cmd.CombinedOutput() - if err != nil { - if isDebug(ctx) { - logrus.Debugf("%s: %s", err.Error(), string(sysOutput)) - } - return errors.New("git does not seem to be available on your PATH, so cannot continue") - } - if isDebug(ctx) { - logrus.Debug("git was found on PATH") - } - return nil -} From 947b9fd412479d5d64e88f51cfe82b297641a642 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Mon, 3 Jul 2023 23:47:05 -0500 Subject: [PATCH 04/29] Add pre-commit hook --- Makefile | 3 +++ scripts/ci.sh | 4 ++++ 2 files changed, 7 insertions(+) create mode 100755 scripts/ci.sh diff --git a/Makefile b/Makefile index d47bff7..b140060 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,9 @@ clean: ./deps/ \ ./testdata/deps/ +pre-commit-hook: + cp ./scripts/ci.sh ./.git/hooks/pre-commit + # Some targets that help set up local workstations with rhad tooling. Assumes # ~/.local/bin is on $PATH add-local-symlinks: diff --git a/scripts/ci.sh b/scripts/ci.sh new file mode 100755 index 0000000..3fe6905 --- /dev/null +++ b/scripts/ci.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +make test From 7d965efef64f20b58973f20c9055caf1a3f43ab0 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Tue, 4 Jul 2023 00:32:59 -0500 Subject: [PATCH 05/29] Got spec_test working, now I just need to get remote/file working for tests to pass --- Makefile | 6 ++-- cmd/sync_test.go | 26 ++++++++------ internal/remote/git.go | 3 +- internal/vdmspec/spec.go | 21 +++++++---- internal/vdmspec/spec_test.go | 68 +++++++++++++++++------------------ 5 files changed, 68 insertions(+), 56 deletions(-) diff --git a/Makefile b/Makefile index b140060..3292370 100644 --- a/Makefile +++ b/Makefile @@ -52,9 +52,9 @@ clean: *cache* \ .*cache* \ ./build/ \ - ./dist/ \ - ./deps/ \ - ./testdata/deps/ + ./dist/ +# TODO: until I sort out the tests to write test data consistently, these deps/ directories can kind of show up anywhere + @find . -type d -name '*deps*' -exec rm -rf {}+ \; pre-commit-hook: cp ./scripts/ci.sh ./.git/hooks/pre-commit diff --git a/cmd/sync_test.go b/cmd/sync_test.go index e1e5886..76ff626 100644 --- a/cmd/sync_test.go +++ b/cmd/sync_test.go @@ -7,40 +7,46 @@ import ( "github.com/opensourcecorp/vdm/internal/vdmspec" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestSync(t *testing.T) { - const testVDMRoot = "./testdata" +const testVDMRoot = "../testdata" - specFilePath := filepath.Join(testVDMRoot, "vdm.json") +var ( + testSpecFilePath = filepath.Join(testVDMRoot, "vdm.json") +) - specs, err := vdmspec.GetSpecsFromFile(specFilePath) - assert.NoError(t, err) +func TestSync(t *testing.T) { + specs, err := vdmspec.GetSpecsFromFile(testSpecFilePath) + require.NoError(t, err) - sync() + // Need to override for test + RootFlagValues.SpecFilePath = testSpecFilePath + err = sync() + require.NoError(t, err) t.Run("spec[0] used a tag", func(t *testing.T) { vdmMeta, err := specs[0].GetVDMMeta() assert.NoError(t, err) - assert.Equal(t, vdmMeta.Version, "v0.2.0") + assert.Equal(t, "v0.2.0", vdmMeta.Version) }) t.Run("spec[1] used 'latest'", func(t *testing.T) { vdmMeta, err := specs[1].GetVDMMeta() assert.NoError(t, err) - assert.Equal(t, vdmMeta.Version, "latest") + assert.Equal(t, "latest", vdmMeta.Version) }) t.Run("spec[2] used a branch", func(t *testing.T) { vdmMeta, err := specs[2].GetVDMMeta() assert.NoError(t, err) - assert.Equal(t, vdmMeta.Version, "main") + assert.Equal(t, "main", vdmMeta.Version) }) t.Run("spec[4] used a hash", func(t *testing.T) { vdmMeta, err := specs[3].GetVDMMeta() assert.NoError(t, err) - assert.Equal(t, vdmMeta.Version, "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f") + assert.Equal(t, "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f", vdmMeta.Version) }) t.Cleanup(func() { diff --git a/internal/remote/git.go b/internal/remote/git.go index ae44931..e266592 100644 --- a/internal/remote/git.go +++ b/internal/remote/git.go @@ -83,5 +83,6 @@ func SyncGit(spec vdmspec.VDMSpec) error { // SyncFile is the root of the sync operations for "file" remote types. func SyncFile(spec vdmspec.VDMSpec) error { - panic("not implemented") + logrus.Error("the 'file' type is not yet implemented") + return nil } diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index d0d58de..2defbdc 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/sirupsen/logrus" ) @@ -31,7 +32,6 @@ func (spec VDMSpec) WriteVDMMeta() error { logrus.Debugf("writing metadata file to '%s'", metaFilePath) err = os.WriteFile(metaFilePath, vdmMetaContent, 0644) if err != nil { - logrus.Debug("error here") return fmt.Errorf("writing metadata file: %w", err) } @@ -68,16 +68,25 @@ func (spec VDMSpec) GetVDMMeta() (VDMSpec, error) { func GetSpecsFromFile(specFilePath string) ([]VDMSpec, error) { specFile, err := os.ReadFile(specFilePath) if err != nil { - logrus.Debugf("error reading specFile from disk: %v", err) - logrus.Fatalf("There was a problem reading your vdm file from '%s' -- does it not exist? Either pass the -spec-file flag, or create one in the default location (details in the README)", specFilePath) + logrus.Debugf("error reading specfile from disk: %v", err) + return nil, fmt.Errorf( + strings.Join([]string{ + "there was a problem reading your vdm file from '%s' -- does it not exist?", + "Either pass the -spec-file flag, or create one in the default location (details in the README).", + "Error details: %w"}, + " ", + ), + specFilePath, + err, + ) } - logrus.Debugf("specFile contents read:\n%s", string(specFile)) + logrus.Debugf("specfile contents read:\n%s", string(specFile)) var specs []VDMSpec err = json.Unmarshal(specFile, &specs) if err != nil { - logrus.Debugf("error during specFile unmarshal: %v", err) - logrus.Fatal("There was a problem reading the contents of your vdm spec file") + logrus.Debugf("error during specfile unmarshal: %v", err) + return nil, fmt.Errorf("there was a problem reading the contents of your vdm spec file: %w", err) } logrus.Debugf("vdmSpecs unmarshalled: %+v", specs) diff --git a/internal/vdmspec/spec_test.go b/internal/vdmspec/spec_test.go index 7fca3ad..20b557d 100644 --- a/internal/vdmspec/spec_test.go +++ b/internal/vdmspec/spec_test.go @@ -1,43 +1,42 @@ package vdmspec import ( + "fmt" "os" "path/filepath" "testing" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func init() { - logrus.SetLevel(logrus.DebugLevel) -} +const testVDMRoot = "../../testdata" -func TestVDMMeta(t *testing.T) { - const testVDMRoot = "./testdata" - testVDMMetaFilePath := filepath.Join(testVDMRoot, MetaFileName) +var ( + testVDMMetaFilePath = filepath.Join(testVDMRoot, MetaFileName) + + testSpec = VDMSpec{ + Remote: "https://some-remote", + Version: "v1.0.0", + LocalPath: testVDMRoot, + } + + testSpecFilePath = filepath.Join(testVDMRoot, "vdm.json") + + testVDMMetaContents = fmt.Sprintf( + `{"remote": "https://some-remote", "version": "v1.0.0", "local_path": "%s"}`, + testVDMRoot, + ) +) +func TestVDMMeta(t *testing.T) { t.Run("GetVDMMeta", func(t *testing.T) { - spec := VDMSpec{ - Remote: "https://some-remote", - Version: "v1.0.0", - LocalPath: "./testdata", - } - vdmMetaContents := ` - { - "remote": "https://some-remote", - "version": "v1.0.0", - "local_path": "./testdata" - }` - err := os.WriteFile(testVDMMetaFilePath, []byte(vdmMetaContents), 0644) - if err != nil { - t.Fatal(err) - } - - got, err := spec.GetVDMMeta() + err := os.WriteFile(testVDMMetaFilePath, []byte(testVDMMetaContents), 0644) + require.NoError(t, err) + + got, err := testSpec.GetVDMMeta() assert.NoError(t, err) - assert.Equal(t, spec, got) + assert.Equal(t, testSpec, got) t.Cleanup(func() { err := os.RemoveAll(testVDMMetaFilePath) @@ -46,17 +45,16 @@ func TestVDMMeta(t *testing.T) { }) t.Run("WriteVDMMeta", func(t *testing.T) { - spec := VDMSpec{ - Remote: "https://some-remote", - Version: "v1.0.0", - LocalPath: "./testdata", - } - err := spec.WriteVDMMeta() + // Needs to have parent dir(s) exist for write to work + err := os.MkdirAll(testSpec.LocalPath, 0644) require.NoError(t, err) - got, err := spec.GetVDMMeta() + err = testSpec.WriteVDMMeta() + require.NoError(t, err) + + got, err := testSpec.GetVDMMeta() assert.NoError(t, err) - assert.Equal(t, spec, got) + assert.Equal(t, testSpec, got) t.Cleanup(func() { err := os.RemoveAll(testVDMMetaFilePath) @@ -65,9 +63,7 @@ func TestVDMMeta(t *testing.T) { }) t.Run("GetSpecsFromFile", func(t *testing.T) { - specFilePath := "./testdata/vdm.json" - - specs, err := GetSpecsFromFile(specFilePath) + specs, err := GetSpecsFromFile(testSpecFilePath) assert.NoError(t, err) assert.Equal(t, 5, len(specs)) From bd5e8ac0583a44cd7a4a543954c5c1475a17f05f Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Tue, 4 Jul 2023 19:31:35 -0500 Subject: [PATCH 06/29] Got tests passing by building a VDMMETA constructor --- Makefile | 9 +++++---- internal/vdmspec/spec.go | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 3292370..398d36c 100644 --- a/Makefile +++ b/Makefile @@ -8,12 +8,11 @@ all: test clean test: clean go vet ./... - go test -v ./... + go test -cover -coverprofile=./cover.out ./... staticcheck ./... make -s clean test-coverage: test - go test -cover -coverprofile=./cover.out ./... go tool cover -html=./cover.out -o cover.html xdg-open ./cover.html @@ -53,8 +52,10 @@ clean: .*cache* \ ./build/ \ ./dist/ -# TODO: until I sort out the tests to write test data consistently, these deps/ directories can kind of show up anywhere - @find . -type d -name '*deps*' -exec rm -rf {}+ \; +# TODO: until I sort out the tests to write test data consistently, these deps/ +# directories etc. can kind of show up anywhere + @find . -type d -name '*deps*' -exec rm -rf {} + + @find . -type f -name '*VDMMETA*' -delete pre-commit-hook: cp ./scripts/ci.sh ./.git/hooks/pre-commit diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index 2defbdc..642fbe7 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -20,8 +20,19 @@ type VDMSpec struct { const MetaFileName = "VDMMETA" -func (spec VDMSpec) WriteVDMMeta() error { +func (spec VDMSpec) MakeMetaFilePath() string { metaFilePath := filepath.Join(spec.LocalPath, MetaFileName) + // TODO: this is brittle, but it's the best I can think of right now + if spec.Type == "file" { + // converts to e.g. 'VDMMETA_http.proto' + metaFilePath = fmt.Sprintf("%s_%s", MetaFileName, filepath.Base(spec.LocalPath)) + } + + return metaFilePath +} + +func (spec VDMSpec) WriteVDMMeta() error { + metaFilePath := spec.MakeMetaFilePath() vdmMetaContent, err := json.MarshalIndent(spec, "", " ") if err != nil { return fmt.Errorf("writing %s: %w", metaFilePath, err) @@ -39,7 +50,7 @@ func (spec VDMSpec) WriteVDMMeta() error { } func (spec VDMSpec) GetVDMMeta() (VDMSpec, error) { - metaFilePath := filepath.Join(spec.LocalPath, MetaFileName) + metaFilePath := spec.MakeMetaFilePath() _, err := os.Stat(metaFilePath) if errors.Is(err, os.ErrNotExist) { return VDMSpec{}, nil // this is ok, because it might literally not exist yet From c714d4348da27475f9cc334a95b16fe3e039bbff Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Tue, 4 Jul 2023 21:41:10 -0500 Subject: [PATCH 07/29] lots, still need more tests --- Makefile | 7 +-- README.md | 10 ++--- cmd/sync.go | 18 ++++---- cmd/sync_test.go | 46 +++++++++++-------- internal/remote/file.go | 85 ++++++++++++++++++++++++++++++++++++ internal/remote/file_test.go | 1 + internal/remote/git.go | 67 ++++++++++++---------------- internal/remote/git_test.go | 74 +++++++++++++++++++++++++++---- internal/vdmspec/spec.go | 14 ++++-- internal/vdmspec/validate.go | 12 ++--- testdata/vdm.json | 4 +- 11 files changed, 242 insertions(+), 96 deletions(-) create mode 100644 internal/remote/file_test.go diff --git a/Makefile b/Makefile index 398d36c..885d7dc 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ test-coverage: test build: clean @mkdir -p build/$$(go env GOOS)-$$(go env GOARCH) - @go build -o build/$$(go env GOOS)-$$(go env GOARCH)/$(BINNAME) + @go build -o build/$$(go env GOOS)-$$(go env GOARCH)/$(BINNAME) -ldflags "-s -w" xbuild: clean @for target in \ @@ -31,10 +31,11 @@ xbuild: clean do \ GOOS=$$(echo "$${target}" | cut -d'-' -f1) ; \ GOARCH=$$(echo "$${target}" | cut -d'-' -f2) ; \ + export GOOS GOARCH ; \ outdir=build/"$${GOOS}-$${GOARCH}" ; \ mkdir -p "$${outdir}" ; \ printf "Building for %s-%s into build/ ...\n" "$${GOOS}" "$${GOARCH}" ; \ - GOOS="$${GOOS}" GOARCH="$${GOARCH}" go build -o "$${outdir}"/$(BINNAME) ; \ + go build -o "$${outdir}"/$(BINNAME) -ldflags "-s -w" ; \ done package: xbuild @@ -51,7 +52,7 @@ clean: *cache* \ .*cache* \ ./build/ \ - ./dist/ + ./dist/*.gz # TODO: until I sort out the tests to write test data consistently, these deps/ # directories etc. can kind of show up anywhere @find . -type d -name '*deps*' -exec rm -rf {} + diff --git a/README.md b/README.md index 874ff1a..c68c51c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ them to live in your repo: "version": "v0.2.0", // tag; can also be a branch, short or long commit hash, or the word 'latest' }, { - "type": "file", // the 'file' type assumes the version is in the remote field itself, so 'version' can be omitted + "type": "file", // the 'file' type assumes the version is in the remote field itself, so 'version' is omitted "remote": "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto", "local_path": "./deps/proto/http/http.proto" } @@ -51,14 +51,12 @@ would look like this: http.proto ``` -If for any reason you want all the deps in the spec file to retain their `.git` -directories (such as if you're using `vdm` to initialize a new computer with -actual repos you'd be working in), you can pass the `-keep-git-dir` flag to `vdm -sync`. - ## Future work - Make the sync mechanism more robust, such that if your spec file changes to remove remotes, they'll get cleaned up automatically. +- Add `--keep-git-dir` flag so that `git` remote types don't wipe the `.git` + directory at clone-time. + - Support more than just Git diff --git a/cmd/sync.go b/cmd/sync.go index 3c3edf6..386ea57 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -16,12 +16,6 @@ var syncCmd = &cobra.Command{ RunE: syncExecute, } -type SyncFlags struct { - KeepGitDir bool -} - -var SyncFlagValues SyncFlags - func syncExecute(_ *cobra.Command, _ []string) error { if err := sync(); err != nil { return fmt.Errorf("executing sync command: %w", err) @@ -29,7 +23,8 @@ func syncExecute(_ *cobra.Command, _ []string) error { return nil } -// sync ensures that the only local dependencies are ones defined in the specfile +// sync does the heavy lifting to ensure that the local directory tree(s) match +// the desired state as defined in the specfile. func sync() error { specs, err := vdmspec.GetSpecsFromFile(RootFlagValues.SpecFilePath) if err != nil { @@ -43,6 +38,7 @@ func sync() error { } } +SpecLoop: for _, spec := range specs { // process stored vdm metafile so we know what operations to actually // perform for existing directories @@ -54,10 +50,12 @@ func sync() error { if vdmMeta == (vdmspec.VDMSpec{}) { logrus.Infof("%s not found at local path '%s' -- will be created", vdmspec.MetaFileName, filepath.Join(spec.LocalPath)) } else { - if vdmMeta.Version != spec.Version { - logrus.Infof("Changing '%s' from current local version spec '%s' to '%s'...", spec.Remote, vdmMeta.Version, spec.Version) + if vdmMeta.Version != spec.Version && vdmMeta.Remote != spec.Remote { + logrus.Infof("Will change '%s' from current local version spec '%s' to '%s'...", spec.Remote, vdmMeta.Version, spec.Version) + panic("not implemented") } else { - logrus.Debugf("Version unchanged (%s) in spec file for '%s' --> '%s'", spec.Version, spec.Remote, spec.LocalPath) + logrus.Infof("Version unchanged (%s) in spec file for '%s' --> '%s', skipping", spec.Version, spec.Remote, spec.LocalPath) + continue SpecLoop } } diff --git a/cmd/sync_test.go b/cmd/sync_test.go index 76ff626..077dff2 100644 --- a/cmd/sync_test.go +++ b/cmd/sync_test.go @@ -25,28 +25,38 @@ func TestSync(t *testing.T) { err = sync() require.NoError(t, err) - t.Run("spec[0] used a tag", func(t *testing.T) { - vdmMeta, err := specs[0].GetVDMMeta() - assert.NoError(t, err) - assert.Equal(t, "v0.2.0", vdmMeta.Version) - }) + t.Run("SyncGit", func(t *testing.T) { + t.Run("spec[0] used a tag", func(t *testing.T) { + vdmMeta, err := specs[0].GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, "v0.2.0", vdmMeta.Version) + }) - t.Run("spec[1] used 'latest'", func(t *testing.T) { - vdmMeta, err := specs[1].GetVDMMeta() - assert.NoError(t, err) - assert.Equal(t, "latest", vdmMeta.Version) - }) + t.Run("spec[1] used 'latest'", func(t *testing.T) { + vdmMeta, err := specs[1].GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, "latest", vdmMeta.Version) + }) - t.Run("spec[2] used a branch", func(t *testing.T) { - vdmMeta, err := specs[2].GetVDMMeta() - assert.NoError(t, err) - assert.Equal(t, "main", vdmMeta.Version) + t.Run("spec[2] used a branch", func(t *testing.T) { + vdmMeta, err := specs[2].GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, "main", vdmMeta.Version) + }) + + t.Run("spec[3] used a hash", func(t *testing.T) { + vdmMeta, err := specs[3].GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f", vdmMeta.Version) + }) }) - t.Run("spec[4] used a hash", func(t *testing.T) { - vdmMeta, err := specs[3].GetVDMMeta() - assert.NoError(t, err) - assert.Equal(t, "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f", vdmMeta.Version) + t.Run("SyncFile", func(t *testing.T) { + t.Run("spec[4] had an implcit version", func(t *testing.T) { + vdmMeta, err := specs[4].GetVDMMeta() + assert.NoError(t, err) + assert.Equal(t, "", vdmMeta.Version) + }) }) t.Cleanup(func() { diff --git a/internal/remote/file.go b/internal/remote/file.go index fbe5b64..ec1ed6f 100644 --- a/internal/remote/file.go +++ b/internal/remote/file.go @@ -1 +1,86 @@ package remote + +import ( + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + + "github.com/opensourcecorp/vdm/internal/vdmspec" + "github.com/sirupsen/logrus" +) + +// SyncFile is the root of the sync operations for "file" remote types. +func SyncFile(spec vdmspec.VDMSpec) error { + fileExists, err := checkFileExists(spec) + if err != nil { + return fmt.Errorf("checking if file exists locally: %w", err) + } + + if !fileExists { + logrus.Infof("File '%s' does not exist locally, retrieving", spec.LocalPath) + err = retrieveFile(spec) + if err != nil { + return fmt.Errorf("retrieving file: %w", err) + } + } else { + logrus.Infof("File '%s' already exists locally, skipping", spec.LocalPath) + } + + return nil +} + +func checkFileExists(spec vdmspec.VDMSpec) (bool, error) { + fullPath, err := filepath.Abs(spec.LocalPath) + if err != nil { + return false, fmt.Errorf("determining abspath for file '%s': %w", spec.LocalPath, err) + } + + _, err = os.Stat(spec.LocalPath) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } else if err != nil { + return false, fmt.Errorf("couldn't check if %s exists at '%s': %w", spec.LocalPath, fullPath, err) + } + + return true, nil +} + +func retrieveFile(spec vdmspec.VDMSpec) error { + resp, err := http.Get(spec.Remote) + if err != nil { + return fmt.Errorf("retrieving remote file '%s': %w", spec.Remote, err) + } + defer func() { + if closeErr := resp.Body.Close(); closeErr != nil { + logrus.Errorf("closing response body after remote file '%s' retrieval: %v", spec.Remote, err) + } + }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unsuccessful status code '%d' from server when retrieving remote file '%s'", resp.StatusCode, spec.Remote) + } + + // Note: I would normally use os.WriteFile() using the returned bytes + // directly, but the internet says this os.Create()/io.Copy() approach + // appears to be idiomatic + outFile, err := os.Create(spec.LocalPath) + if err != nil { + return fmt.Errorf("creating landing file '%s' for remote file: %w", spec.LocalPath, err) + } + defer func() { + if closeErr := outFile.Close(); closeErr != nil { + logrus.Errorf("closing local file '%s' after remote file '%s' retrieval: %v", spec.LocalPath, spec.Remote, err) + } + }() + + bytesWritten, err := io.Copy(outFile, resp.Body) + if err != nil { + return fmt.Errorf("copying HTTP response to disk: ") + } + logrus.Debugf("wrote %d bytes to '%s'", bytesWritten, spec.LocalPath) + + return nil +} diff --git a/internal/remote/file_test.go b/internal/remote/file_test.go new file mode 100644 index 0000000..fbe5b64 --- /dev/null +++ b/internal/remote/file_test.go @@ -0,0 +1 @@ +package remote diff --git a/internal/remote/git.go b/internal/remote/git.go index e266592..ac38de7 100644 --- a/internal/remote/git.go +++ b/internal/remote/git.go @@ -3,12 +3,40 @@ package remote import ( "errors" "fmt" + "os" "os/exec" + "path/filepath" "github.com/opensourcecorp/vdm/internal/vdmspec" "github.com/sirupsen/logrus" ) +// SyncGit is the root of the sync operations for "git" remote types. +func SyncGit(spec vdmspec.VDMSpec) error { + err := gitClone(spec) + if err != nil { + return fmt.Errorf("cloing remote: %w", err) + } + + if spec.Version != "latest" { + logrus.Infof("%s -- Setting specified version...", spec.OpMsg()) + checkoutCmd := exec.Command("git", "-C", spec.LocalPath, "checkout", spec.Version) + checkoutOutput, err := checkoutCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error checking out specified revision: exec error '%w', with output: %s", err, string(checkoutOutput)) + } + } + + logrus.Debugf("removing .git dir for local path '%s'", spec.LocalPath) + dotGitPath := filepath.Join(spec.LocalPath, ".git") + err = os.RemoveAll(dotGitPath) + if err != nil { + return fmt.Errorf("removing directory %s: %w", dotGitPath, err) + } + + return nil +} + func checkGitAvailable() error { cmd := exec.Command("git", "--version") sysOutput, err := cmd.CombinedOutput() @@ -47,42 +75,3 @@ func gitClone(spec vdmspec.VDMSpec) error { return nil } - -// SyncGit is the root of the sync operations for "git" remote types. -func SyncGit(spec vdmspec.VDMSpec) error { - // // TODO: pull this up so that it only runs if the version changed or the user requested a wipe - // if !cmd.SyncFlagValues.KeepGitDir { - // logrus.Debugf("removing any old data for '%s'", spec.LocalPath) - // os.RemoveAll(spec.LocalPath) - // } - - err := gitClone(spec) - if err != nil { - return fmt.Errorf("cloing remote: %w", err) - } - - if spec.Version != "latest" { - logrus.Infof("%s -- Setting specified version...", spec.OpMsg()) - checkoutCmd := exec.Command("git", "-C", spec.LocalPath, "checkout", spec.Version) - checkoutOutput, err := checkoutCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("error checking out specified revision: exec error '%w', with output: %s", err, string(checkoutOutput)) - } - } - - // if cmd.SyncFlagValues.KeepGitDir { - // logrus.Debugf("removing .git dir for local path '%s'", spec.LocalPath) - // err := os.RemoveAll(filepath.Join(spec.LocalPath, ".git")) - // if err != nil { - // return fmt.Errorf("removing ") - // } - // } - - return nil -} - -// SyncFile is the root of the sync operations for "file" remote types. -func SyncFile(spec vdmspec.VDMSpec) error { - logrus.Error("the 'file' type is not yet implemented") - return nil -} diff --git a/internal/remote/git_test.go b/internal/remote/git_test.go index 281a5c9..502f868 100644 --- a/internal/remote/git_test.go +++ b/internal/remote/git_test.go @@ -1,19 +1,77 @@ package remote import ( + "os" "testing" + "github.com/opensourcecorp/vdm/internal/vdmspec" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func getTestGitSpec() vdmspec.VDMSpec { + specLocalPath := "./deps/go-common" + return vdmspec.VDMSpec{ + Type: "git", + Remote: "https://github.com/opensourcecorp/go-common", + Version: "v0.2.0", + LocalPath: specLocalPath, + } +} + +func TestSyncGit(t *testing.T) { + spec := getTestGitSpec() + SyncGit(spec) + t.Cleanup(func() { + if cleanupErr := os.RemoveAll(spec.LocalPath); cleanupErr != nil { + t.Fatalf("removing specLocalPath: %v", cleanupErr) + } + }) + + t.Run(".git directory was removed", func(t *testing.T) { + _, err := os.Stat("./deps/go-common-tag/.git") + assert.ErrorIs(t, err, os.ErrNotExist) + }) +} + func TestCheckGitAvailable(t *testing.T) { - t.Run("git", func(t *testing.T) { - // Host of this test better have git available lol - gitAvailable := checkGitAvailable() - assert.NoError(t, gitAvailable) - - t.Setenv("PATH", "") - gitAvailable = checkGitAvailable() - assert.Error(t, gitAvailable) + t.Run("checkGitAvailable", func(t *testing.T) { + t.Run("no error when git is available", func(t *testing.T) { + // Host of this test better have git available lol + gitAvailable := checkGitAvailable() + assert.NoError(t, gitAvailable) + }) + + t.Run("error when git is NOT available", func(t *testing.T) { + t.Setenv("PATH", "") + gitAvailable := checkGitAvailable() + assert.Error(t, gitAvailable) + }) + }) +} + +func TestGitClone(t *testing.T) { + spec := getTestGitSpec() + cloneErr := gitClone(spec) + t.Cleanup(func() { + if cleanupErr := os.RemoveAll(spec.LocalPath); cleanupErr != nil { + t.Fatalf("removing specLocalPath: %v", cleanupErr) + } + }) + + t.Run("no error on success", func(t *testing.T) { + require.NoError(t, cloneErr) + }) + + t.Run("LocalPath is a directory, not a file", func(t *testing.T) { + outDir, err := os.Stat("./deps/go-common") + assert.NoError(t, err) + assert.True(t, outDir.IsDir()) + }) + + t.Run("a known file in the remote exists, and is a file", func(t *testing.T) { + sampleFile, err := os.Stat("./deps/go-common/go.mod") + assert.NoError(t, err) + assert.False(t, sampleFile.IsDir()) }) } diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index 642fbe7..4e2ce5b 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -12,10 +12,10 @@ import ( ) type VDMSpec struct { + Type string `json:"type,omitempty"` Remote string `json:"remote"` Version string `json:"version,omitempty"` LocalPath string `json:"local_path"` - Type string `json:"type,omitempty"` } const MetaFileName = "VDMMETA" @@ -24,8 +24,10 @@ func (spec VDMSpec) MakeMetaFilePath() string { metaFilePath := filepath.Join(spec.LocalPath, MetaFileName) // TODO: this is brittle, but it's the best I can think of right now if spec.Type == "file" { + fileDir := filepath.Dir(spec.LocalPath) + fileName := filepath.Base(spec.LocalPath) // converts to e.g. 'VDMMETA_http.proto' - metaFilePath = fmt.Sprintf("%s_%s", MetaFileName, filepath.Base(spec.LocalPath)) + metaFilePath = filepath.Join(fileDir, fmt.Sprintf("%s_%s", MetaFileName, fileName)) } return metaFilePath @@ -58,7 +60,7 @@ func (spec VDMSpec) GetVDMMeta() (VDMSpec, error) { return VDMSpec{}, fmt.Errorf("couldn't check if %s exists at '%s': %w", MetaFileName, metaFilePath, err) } - vdmMetaFile, err := os.ReadFile(filepath.Join(spec.LocalPath, MetaFileName)) + vdmMetaFile, err := os.ReadFile(metaFilePath) if err != nil { logrus.Debugf("error reading VMDMMETA from disk: %v", err) return VDMSpec{}, fmt.Errorf("there was a problem reading the %s file from '%s': %w", MetaFileName, metaFilePath, err) @@ -107,5 +109,9 @@ func GetSpecsFromFile(specFilePath string) ([]VDMSpec, error) { // OpMsg constructs a loggable message outlining the specific operation being // performed at the moment func (spec VDMSpec) OpMsg() string { - return fmt.Sprintf("%s@%s --> %s", spec.Remote, spec.Version, spec.LocalPath) + if spec.Version != "" { + return fmt.Sprintf("%s@%s --> %s", spec.Remote, spec.Version, spec.LocalPath) + } else { + return fmt.Sprintf("%s --> %s", spec.Remote, spec.LocalPath) + } } diff --git a/internal/vdmspec/validate.go b/internal/vdmspec/validate.go index 6414014..a4dd30a 100644 --- a/internal/vdmspec/validate.go +++ b/internal/vdmspec/validate.go @@ -30,7 +30,7 @@ func (spec VDMSpec) Validate() error { allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length. If you don't care about the version (even though you probably should), then use 'latest'")) } if spec.Type == "file" && len(spec.Version) > 0 { - logrus.Infof("NOTE: Remote %s specified as type '%s' but also specified version as '%s'; ignoring version field", spec.Remote, spec.Type, spec.Version) + logrus.Warnf("NOTE: Remote '%s' specified as type '%s', which does not take explicit version info (you provided '%s'); ignoring version field", spec.Remote, spec.Type, spec.Version) } // LocalPath field @@ -40,11 +40,11 @@ func (spec VDMSpec) Validate() error { } // Type field - logrus.Debugf("validating field 'Version' for %+v", spec) - typeMap := map[string]struct{}{ - "git": {}, - "": {}, // also git - "file": {}, + logrus.Debugf("validating field 'Type' for %+v", spec) + typeMap := map[string]int{ + "git": 1, + "": 2, // also git + "file": 3, } if _, ok := typeMap[spec.Type]; !ok { allErrors = append(allErrors, fmt.Errorf("unrecognized remote type '%s'", spec.Type)) diff --git a/testdata/vdm.json b/testdata/vdm.json index ec6fc27..473830e 100644 --- a/testdata/vdm.json +++ b/testdata/vdm.json @@ -20,8 +20,8 @@ "local_path": "./deps/go-common-hash" }, { + "type": "file", "remote": "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto", - "local_path": "./deps/http.proto", - "type": "file" + "local_path": "./deps/http.proto" } ] From a36e6f8810566922e887c8b3a41ffe203a684508 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Wed, 5 Jul 2023 19:36:07 -0500 Subject: [PATCH 08/29] Tweak some asserts to requires --- internal/remote/git_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/remote/git_test.go b/internal/remote/git_test.go index 502f868..4819f10 100644 --- a/internal/remote/git_test.go +++ b/internal/remote/git_test.go @@ -21,7 +21,8 @@ func getTestGitSpec() vdmspec.VDMSpec { func TestSyncGit(t *testing.T) { spec := getTestGitSpec() - SyncGit(spec) + err := SyncGit(spec) + require.NoError(t, err) t.Cleanup(func() { if cleanupErr := os.RemoveAll(spec.LocalPath); cleanupErr != nil { t.Fatalf("removing specLocalPath: %v", cleanupErr) @@ -65,13 +66,13 @@ func TestGitClone(t *testing.T) { t.Run("LocalPath is a directory, not a file", func(t *testing.T) { outDir, err := os.Stat("./deps/go-common") - assert.NoError(t, err) + require.NoError(t, err) assert.True(t, outDir.IsDir()) }) t.Run("a known file in the remote exists, and is a file", func(t *testing.T) { sampleFile, err := os.Stat("./deps/go-common/go.mod") - assert.NoError(t, err) + require.NoError(t, err) assert.False(t, sampleFile.IsDir()) }) } From 975ccbfb058fb2c02066bea52f759b7c8d38cfeb Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Wed, 5 Jul 2023 20:24:04 -0500 Subject: [PATCH 09/29] Add WIP support for Debian packaging --- .gitignore | 1 - CHANGELOG | 0 Makefile | 50 ++++++++++++++++------------------ dist/.gitignore | 9 ++++++ dist/debian/vdm/DEBIAN/control | 6 ++++ dist/man/man.1.md | 12 ++++++++ scripts/package-debian.sh | 26 ++++++++++++++++++ scripts/package.sh | 11 ++++++++ scripts/xbuild.sh | 15 ++++++++++ 9 files changed, 102 insertions(+), 28 deletions(-) create mode 100644 CHANGELOG create mode 100644 dist/.gitignore create mode 100644 dist/debian/vdm/DEBIAN/control create mode 100644 dist/man/man.1.md create mode 100755 scripts/package-debian.sh create mode 100755 scripts/package.sh create mode 100755 scripts/xbuild.sh diff --git a/.gitignore b/.gitignore index da631e4..3d4ec61 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,3 @@ *cache* build/ deps/ -dist/ diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..e69de29 diff --git a/Makefile b/Makefile index 885d7dc..0dbaf98 100644 --- a/Makefile +++ b/Makefile @@ -2,62 +2,58 @@ SHELL = /usr/bin/env bash -euo pipefail BINNAME := vdm -all: test clean +# DO NOT TOUCH -- use `make bump-version oldversion=X newversion=Y`! +VERSION := 0.4.0 -.PHONY: % +all: test +.PHONY: test test: clean go vet ./... go test -cover -coverprofile=./cover.out ./... staticcheck ./... make -s clean +.PHONY: test-coverage test-coverage: test go tool cover -html=./cover.out -o cover.html xdg-open ./cover.html +# builds for the current platform only build: clean - @mkdir -p build/$$(go env GOOS)-$$(go env GOARCH) - @go build -o build/$$(go env GOOS)-$$(go env GOARCH)/$(BINNAME) -ldflags "-s -w" + @go build -buildmode=pie -o build/$$(go env GOOS)-$$(go env GOARCH)/$(BINNAME) -ldflags "-s -w" xbuild: clean - @for target in \ - darwin-amd64 \ - linux-amd64 \ - linux-arm \ - linux-arm64 \ - windows-amd64 \ - ; \ - do \ - GOOS=$$(echo "$${target}" | cut -d'-' -f1) ; \ - GOARCH=$$(echo "$${target}" | cut -d'-' -f2) ; \ - export GOOS GOARCH ; \ - outdir=build/"$${GOOS}-$${GOARCH}" ; \ - mkdir -p "$${outdir}" ; \ - printf "Building for %s-%s into build/ ...\n" "$${GOOS}" "$${GOARCH}" ; \ - go build -o "$${outdir}"/$(BINNAME) -ldflags "-s -w" ; \ - done + @bash ./scripts/xbuild.sh package: xbuild - @mkdir -p dist - @cd build || exit 1; \ - for built in * ; do \ - printf 'Packaging for %s into dist/ ...\n' "$${built}" ; \ - cd $${built} && tar -czf ../../dist/$(BINNAME)_$${built}.tar.gz * && cd - >/dev/null ; \ - done + bash ./scripts/package.sh +package-debian: build + @bash ./scripts/package-debian.sh + +.PHONY: clean clean: @rm -rf \ /tmp/$(BINNAME)-tests \ *cache* \ .*cache* \ ./build/ \ - ./dist/*.gz + ./dist/*.gz \ + ./dist/debian/vdm.deb + @sudo rm -rf ./dist/debian/vdm/usr # TODO: until I sort out the tests to write test data consistently, these deps/ # directories etc. can kind of show up anywhere @find . -type d -name '*deps*' -exec rm -rf {} + @find . -type f -name '*VDMMETA*' -delete +bump-version: clean + @if [[ -z "$(oldversion)" ]] || [[ -z "$(newversion)" ]] ; then printf 'ERROR: "oldversion" and "newversion" must be provided\n' && exit 1 ; fi + find . \ + -type f \ + -not -path './go.*' \ + -exec sed -i 's/$(oldversion)/$(newversion)/g' {} \; + pre-commit-hook: cp ./scripts/ci.sh ./.git/hooks/pre-commit diff --git a/dist/.gitignore b/dist/.gitignore new file mode 100644 index 0000000..a6f5919 --- /dev/null +++ b/dist/.gitignore @@ -0,0 +1,9 @@ +# Packaged binaries for e.g. GitHub +*.gz + +# The possible locations of the app binary itself, and its debfile +vdm +*.deb +debian/vdm/usr/bin/vdm +# ... but NOT its Debian packaging root directory, which must have the same name +!debian/vdm/ diff --git a/dist/debian/vdm/DEBIAN/control b/dist/debian/vdm/DEBIAN/control new file mode 100644 index 0000000..33150b3 --- /dev/null +++ b/dist/debian/vdm/DEBIAN/control @@ -0,0 +1,6 @@ +Package: vdm +Version: 0.4.0 +Architecture: amd64 +Maintainer: Ryan Price +Description: Versioned Dependency Manager + vdm manages arbitrary remote dependencies for your own code. diff --git a/dist/man/man.1.md b/dist/man/man.1.md new file mode 100644 index 0000000..2fb986f --- /dev/null +++ b/dist/man/man.1.md @@ -0,0 +1,12 @@ +% VDM(1) vdm 0.4.0 +% Ryan J. Price +% 2023 + +# NAME +vdm - Manage arbitary remote dependencies for your code + +# SYNOPSIS +**vdm** [*OPTION*] + +# DESCRIPTION +**vdm** does some stuff. diff --git a/scripts/package-debian.sh b/scripts/package-debian.sh new file mode 100755 index 0000000..e2ba507 --- /dev/null +++ b/scripts/package-debian.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +# TODO: figure out cross-arch distribution for Debian at some point +printf 'Note: only supporting Debian builds for amd64 at the moment\n' + +# We need several directories to exist, and lintian says they need to be owned +# by root, so instead of keeping them in the repo, we create them (and populate +# their contents) on the fly +mkdir -p \ + ./dist/debian/vdm/usr/bin \ + ./dist/debian/vdm/usr/share/doc/vdm \ + ./dist/debian/vdm/usr/share/man/man1 +sudo chown -R 0:0 ./dist/debian/vdm/usr + +sudo cp ./build/linux-amd64/vdm ./dist/debian/vdm/usr/bin/vdm +sudo cp ./LICENSE ./dist/debian/vdm/usr/share/doc/vdm/copyright +sudo sh -c 'gzip -9 -n -c ./CHANGELOG > ./dist/debian/vdm/usr/share/doc/vdm/changelog.gz' +sudo sh -c 'pandoc ./dist/man/man.1.md -s -t man | gzip -9 -n -c - > ./dist/debian/vdm/usr/share/man/man1/vdm.1.gz' + +# Actually build the debfile +dpkg-deb --build ./dist/debian/vdm + +# Ask lintian (the Debian package linter) to scream as loudly as it can about +# anything it finds +lintian --info --tag-display-limit=0 ./dist/debian/vdm.deb diff --git a/scripts/package.sh b/scripts/package.sh new file mode 100755 index 0000000..dfc3274 --- /dev/null +++ b/scripts/package.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +# cd-jumps because it makes logs cleaner, not sorry +mkdir -p dist +cd build || exit 1 +for built in * ; do + printf 'Packaging for %s into dist/ ...\n' "${built}" + cd "${built}" && tar -czf ../../dist/vdm_"${built}".tar.gz ./* + cd - > /dev/null +done diff --git a/scripts/xbuild.sh b/scripts/xbuild.sh new file mode 100755 index 0000000..3e0d3c6 --- /dev/null +++ b/scripts/xbuild.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +targets=$(go tool dist list | grep -E 'linux|windows|darwin' | grep -E 'amd64|arm64') +printf 'Will build for:\n%s\n\n' "${targets}" + +for target in ${targets} ; do + GOOS=$(echo "${target}" | cut -d'/' -f1) + GOARCH=$(echo "${target}" | cut -d'/' -f2) + export GOOS GOARCH + outdir=build/"${GOOS}-${GOARCH}" + mkdir -p "${outdir}" + printf "Building for %s-%s into build/ ...\n" "${GOOS}" "${GOARCH}" + go build -buildmode=pie -o "${outdir}"/vdm -ldflags "-s -w" +done From fdf904314e506916d041857fcc95b8fd833bbf22 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 9 Jul 2023 19:20:05 -0500 Subject: [PATCH 10/29] Rename VDMSpec to Spec --- cmd/sync.go | 2 +- dist/man/man.1.md | 2 +- internal/remote/file.go | 6 +++--- internal/remote/git.go | 4 ++-- internal/remote/git_test.go | 4 ++-- internal/vdmspec/doc.go | 2 +- internal/vdmspec/spec.go | 24 ++++++++++++------------ internal/vdmspec/spec_test.go | 2 +- internal/vdmspec/validate.go | 4 ++-- internal/vdmspec/validate_test.go | 12 ++++++------ 10 files changed, 31 insertions(+), 31 deletions(-) diff --git a/cmd/sync.go b/cmd/sync.go index 386ea57..d5822e6 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -47,7 +47,7 @@ SpecLoop: return fmt.Errorf("getting vdm metadata file for sync: %w", err) } - if vdmMeta == (vdmspec.VDMSpec{}) { + if vdmMeta == (vdmspec.Spec{}) { logrus.Infof("%s not found at local path '%s' -- will be created", vdmspec.MetaFileName, filepath.Join(spec.LocalPath)) } else { if vdmMeta.Version != spec.Version && vdmMeta.Remote != spec.Remote { diff --git a/dist/man/man.1.md b/dist/man/man.1.md index 2fb986f..71498ec 100644 --- a/dist/man/man.1.md +++ b/dist/man/man.1.md @@ -6,7 +6,7 @@ vdm - Manage arbitary remote dependencies for your code # SYNOPSIS -**vdm** [*OPTION*] +**vdm** [*OPTIONS*] COMMAND [*OPTIONS*] # DESCRIPTION **vdm** does some stuff. diff --git a/internal/remote/file.go b/internal/remote/file.go index ec1ed6f..4fbbabe 100644 --- a/internal/remote/file.go +++ b/internal/remote/file.go @@ -13,7 +13,7 @@ import ( ) // SyncFile is the root of the sync operations for "file" remote types. -func SyncFile(spec vdmspec.VDMSpec) error { +func SyncFile(spec vdmspec.Spec) error { fileExists, err := checkFileExists(spec) if err != nil { return fmt.Errorf("checking if file exists locally: %w", err) @@ -32,7 +32,7 @@ func SyncFile(spec vdmspec.VDMSpec) error { return nil } -func checkFileExists(spec vdmspec.VDMSpec) (bool, error) { +func checkFileExists(spec vdmspec.Spec) (bool, error) { fullPath, err := filepath.Abs(spec.LocalPath) if err != nil { return false, fmt.Errorf("determining abspath for file '%s': %w", spec.LocalPath, err) @@ -48,7 +48,7 @@ func checkFileExists(spec vdmspec.VDMSpec) (bool, error) { return true, nil } -func retrieveFile(spec vdmspec.VDMSpec) error { +func retrieveFile(spec vdmspec.Spec) error { resp, err := http.Get(spec.Remote) if err != nil { return fmt.Errorf("retrieving remote file '%s': %w", spec.Remote, err) diff --git a/internal/remote/git.go b/internal/remote/git.go index ac38de7..d416649 100644 --- a/internal/remote/git.go +++ b/internal/remote/git.go @@ -12,7 +12,7 @@ import ( ) // SyncGit is the root of the sync operations for "git" remote types. -func SyncGit(spec vdmspec.VDMSpec) error { +func SyncGit(spec vdmspec.Spec) error { err := gitClone(spec) if err != nil { return fmt.Errorf("cloing remote: %w", err) @@ -48,7 +48,7 @@ func checkGitAvailable() error { return nil } -func gitClone(spec vdmspec.VDMSpec) error { +func gitClone(spec vdmspec.Spec) error { err := checkGitAvailable() if err != nil { return fmt.Errorf("remote '%s' is a git type, but git may not installed/available on PATH: %w", spec.Remote, err) diff --git a/internal/remote/git_test.go b/internal/remote/git_test.go index 4819f10..6f821f6 100644 --- a/internal/remote/git_test.go +++ b/internal/remote/git_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/require" ) -func getTestGitSpec() vdmspec.VDMSpec { +func getTestGitSpec() vdmspec.Spec { specLocalPath := "./deps/go-common" - return vdmspec.VDMSpec{ + return vdmspec.Spec{ Type: "git", Remote: "https://github.com/opensourcecorp/go-common", Version: "v0.2.0", diff --git a/internal/vdmspec/doc.go b/internal/vdmspec/doc.go index f9df9ce..051c4bc 100644 --- a/internal/vdmspec/doc.go +++ b/internal/vdmspec/doc.go @@ -1,4 +1,4 @@ /* -Package vdmspec defines the [VDMSpec] struct type, and its associated methods. +Package vdmspec defines the [Spec] struct type, and its associated methods. */ package vdmspec diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index 4e2ce5b..535eb25 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -11,7 +11,7 @@ import ( "github.com/sirupsen/logrus" ) -type VDMSpec struct { +type Spec struct { Type string `json:"type,omitempty"` Remote string `json:"remote"` Version string `json:"version,omitempty"` @@ -20,7 +20,7 @@ type VDMSpec struct { const MetaFileName = "VDMMETA" -func (spec VDMSpec) MakeMetaFilePath() string { +func (spec Spec) MakeMetaFilePath() string { metaFilePath := filepath.Join(spec.LocalPath, MetaFileName) // TODO: this is brittle, but it's the best I can think of right now if spec.Type == "file" { @@ -33,7 +33,7 @@ func (spec VDMSpec) MakeMetaFilePath() string { return metaFilePath } -func (spec VDMSpec) WriteVDMMeta() error { +func (spec Spec) WriteVDMMeta() error { metaFilePath := spec.MakeMetaFilePath() vdmMetaContent, err := json.MarshalIndent(spec, "", " ") if err != nil { @@ -51,34 +51,34 @@ func (spec VDMSpec) WriteVDMMeta() error { return nil } -func (spec VDMSpec) GetVDMMeta() (VDMSpec, error) { +func (spec Spec) GetVDMMeta() (Spec, error) { metaFilePath := spec.MakeMetaFilePath() _, err := os.Stat(metaFilePath) if errors.Is(err, os.ErrNotExist) { - return VDMSpec{}, nil // this is ok, because it might literally not exist yet + return Spec{}, nil // this is ok, because it might literally not exist yet } else if err != nil { - return VDMSpec{}, fmt.Errorf("couldn't check if %s exists at '%s': %w", MetaFileName, metaFilePath, err) + return Spec{}, fmt.Errorf("couldn't check if %s exists at '%s': %w", MetaFileName, metaFilePath, err) } vdmMetaFile, err := os.ReadFile(metaFilePath) if err != nil { logrus.Debugf("error reading VMDMMETA from disk: %v", err) - return VDMSpec{}, fmt.Errorf("there was a problem reading the %s file from '%s': %w", MetaFileName, metaFilePath, err) + return Spec{}, fmt.Errorf("there was a problem reading the %s file from '%s': %w", MetaFileName, metaFilePath, err) } logrus.Debugf("%s contents read:\n%s", MetaFileName, string(vdmMetaFile)) - var vdmMeta VDMSpec + var vdmMeta Spec err = json.Unmarshal(vdmMetaFile, &vdmMeta) if err != nil { logrus.Debugf("error during %s unmarshal: %v", MetaFileName, err) - return VDMSpec{}, fmt.Errorf("there was a problem reading the contents of the %s file at '%s': %v", MetaFileName, metaFilePath, err) + return Spec{}, fmt.Errorf("there was a problem reading the contents of the %s file at '%s': %v", MetaFileName, metaFilePath, err) } logrus.Debugf("file %s unmarshalled: %+v", MetaFileName, vdmMeta) return vdmMeta, nil } -func GetSpecsFromFile(specFilePath string) ([]VDMSpec, error) { +func GetSpecsFromFile(specFilePath string) ([]Spec, error) { specFile, err := os.ReadFile(specFilePath) if err != nil { logrus.Debugf("error reading specfile from disk: %v", err) @@ -95,7 +95,7 @@ func GetSpecsFromFile(specFilePath string) ([]VDMSpec, error) { } logrus.Debugf("specfile contents read:\n%s", string(specFile)) - var specs []VDMSpec + var specs []Spec err = json.Unmarshal(specFile, &specs) if err != nil { logrus.Debugf("error during specfile unmarshal: %v", err) @@ -108,7 +108,7 @@ func GetSpecsFromFile(specFilePath string) ([]VDMSpec, error) { // OpMsg constructs a loggable message outlining the specific operation being // performed at the moment -func (spec VDMSpec) OpMsg() string { +func (spec Spec) OpMsg() string { if spec.Version != "" { return fmt.Sprintf("%s@%s --> %s", spec.Remote, spec.Version, spec.LocalPath) } else { diff --git a/internal/vdmspec/spec_test.go b/internal/vdmspec/spec_test.go index 20b557d..aef86e6 100644 --- a/internal/vdmspec/spec_test.go +++ b/internal/vdmspec/spec_test.go @@ -15,7 +15,7 @@ const testVDMRoot = "../../testdata" var ( testVDMMetaFilePath = filepath.Join(testVDMRoot, MetaFileName) - testSpec = VDMSpec{ + testSpec = Spec{ Remote: "https://some-remote", Version: "v1.0.0", LocalPath: testVDMRoot, diff --git a/internal/vdmspec/validate.go b/internal/vdmspec/validate.go index a4dd30a..0c5abf7 100644 --- a/internal/vdmspec/validate.go +++ b/internal/vdmspec/validate.go @@ -8,7 +8,7 @@ import ( "github.com/sirupsen/logrus" ) -func (spec VDMSpec) Validate() error { +func (spec Spec) Validate() error { var allErrors []error // Remote field @@ -16,7 +16,7 @@ func (spec VDMSpec) Validate() error { if len(spec.Remote) == 0 { allErrors = append(allErrors, errors.New("all 'remote' fields must be non-zero length")) } - protocolRegex := regexp.MustCompile(`(https://|git://|git@)`) + protocolRegex := regexp.MustCompile(`(http(s?)://|git://|git@)`) if !protocolRegex.MatchString(spec.Remote) { allErrors = append( allErrors, diff --git a/internal/vdmspec/validate_test.go b/internal/vdmspec/validate_test.go index 33d42a1..38ed830 100644 --- a/internal/vdmspec/validate_test.go +++ b/internal/vdmspec/validate_test.go @@ -8,7 +8,7 @@ import ( func TestValidate(t *testing.T) { t.Run("passes", func(t *testing.T) { - spec := VDMSpec{ + spec := Spec{ Remote: "https://some-remote", Version: "v1.0.0", LocalPath: "./deps/some-remote", @@ -18,7 +18,7 @@ func TestValidate(t *testing.T) { }) t.Run("fails on zero-length remote", func(t *testing.T) { - spec := VDMSpec{ + spec := Spec{ Remote: "", Version: "v1.0.0", LocalPath: "./deps/some-remote", @@ -28,7 +28,7 @@ func TestValidate(t *testing.T) { }) t.Run("fails on remote without valid protocol", func(t *testing.T) { - spec := VDMSpec{ + spec := Spec{ Remote: "some-remote", Version: "v1.0.0", LocalPath: "./deps/some-remote", @@ -38,7 +38,7 @@ func TestValidate(t *testing.T) { }) t.Run("fails on zero-length version for git remote type", func(t *testing.T) { - spec := VDMSpec{ + spec := Spec{ Remote: "https://some-remote", Version: "", LocalPath: "./deps/some-remote", @@ -49,7 +49,7 @@ func TestValidate(t *testing.T) { }) t.Run("fails on unrecognized remote type", func(t *testing.T) { - spec := VDMSpec{ + spec := Spec{ Remote: "https://some-remote", Version: "", LocalPath: "./deps/some-remote", @@ -60,7 +60,7 @@ func TestValidate(t *testing.T) { }) t.Run("fails on zero-length local path", func(t *testing.T) { - spec := VDMSpec{ + spec := Spec{ Remote: "https://some-remote", Version: "v1.0.0", LocalPath: "", From c99f5c5414b21f626198381f802a2d14a72ec02f Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 21 Jul 2024 19:43:47 -0500 Subject: [PATCH 11/29] Tweaks --- Makefile | 8 ++++---- cmd/root.go | 2 +- scripts/xbuild.sh | 5 ++++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 0dbaf98..2a89918 100644 --- a/Makefile +++ b/Makefile @@ -5,14 +5,13 @@ BINNAME := vdm # DO NOT TOUCH -- use `make bump-version oldversion=X newversion=Y`! VERSION := 0.4.0 -all: test +all: test package package-debian .PHONY: test test: clean go vet ./... go test -cover -coverprofile=./cover.out ./... staticcheck ./... - make -s clean .PHONY: test-coverage test-coverage: test @@ -27,7 +26,7 @@ xbuild: clean @bash ./scripts/xbuild.sh package: xbuild - bash ./scripts/package.sh + @bash ./scripts/package.sh package-debian: build @bash ./scripts/package-debian.sh @@ -40,7 +39,8 @@ clean: .*cache* \ ./build/ \ ./dist/*.gz \ - ./dist/debian/vdm.deb + ./dist/debian/vdm.deb \ + *.out @sudo rm -rf ./dist/debian/vdm/usr # TODO: until I sort out the tests to write test data consistently, these deps/ # directories etc. can kind of show up anywhere diff --git a/cmd/root.go b/cmd/root.go index c81edf3..0d24e5f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,7 +10,7 @@ import ( var rootCmd = cobra.Command{ Use: "vdm", Short: "vdm -- a Versioned-Dependency Manager", - Long: "vdm is used to manage arbitrary remote depdencies in a code repository", + Long: "vdm is used to manage arbitrary remote dependencies", TraverseChildren: true, } diff --git a/scripts/xbuild.sh b/scripts/xbuild.sh index 3e0d3c6..1cbd5fc 100755 --- a/scripts/xbuild.sh +++ b/scripts/xbuild.sh @@ -2,7 +2,10 @@ set -euo pipefail targets=$(go tool dist list | grep -E 'linux|windows|darwin' | grep -E 'amd64|arm64') -printf 'Will build for:\n%s\n\n' "${targets}" +printf 'Will build for:\n' +while read -r line ; do + printf '\t%s\n' "${line}" +done <<< "${targets}" for target in ${targets} ; do GOOS=$(echo "${target}" | cut -d'/' -f1) From 9d8698526cc1166c70a1c89826c7e961dd46dc12 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Thu, 25 Jul 2024 22:47:12 -0500 Subject: [PATCH 12/29] Change files to YAML, rename Spec to Remote and add wrapper Spec --- Makefile | 8 +-- README.md | 39 ++++++------- cmd/root.go | 2 +- cmd/sync.go | 38 ++++++------ cmd/sync_test.go | 18 +++--- go.mod | 2 +- internal/remote/doc.go | 4 -- internal/remote/file_test.go | 1 - internal/remotes/doc.go | 4 ++ internal/{remote => remotes}/file.go | 40 ++++++------- internal/remotes/file_test.go | 1 + internal/{remote => remotes}/git.go | 32 +++++------ internal/{remote => remotes}/git_test.go | 6 +- internal/vdmspec/spec.go | 73 +++++++++++++----------- internal/vdmspec/spec_test.go | 20 +++---- internal/vdmspec/validate.go | 70 ++++++++++++----------- internal/vdmspec/validate_test.go | 52 ++++++++++------- scripts/ci.sh | 29 +++++++++- testdata/vdm.json | 27 --------- testdata/vdm.yaml | 16 ++++++ 20 files changed, 256 insertions(+), 226 deletions(-) delete mode 100644 internal/remote/doc.go delete mode 100644 internal/remote/file_test.go create mode 100644 internal/remotes/doc.go rename internal/{remote => remotes}/file.go (60%) create mode 100644 internal/remotes/file_test.go rename internal/{remote => remotes}/git.go (65%) rename internal/{remote => remotes}/git_test.go (96%) delete mode 100644 testdata/vdm.json create mode 100644 testdata/vdm.yaml diff --git a/Makefile b/Makefile index 2a89918..5a9ec39 100644 --- a/Makefile +++ b/Makefile @@ -7,11 +7,9 @@ VERSION := 0.4.0 all: test package package-debian -.PHONY: test -test: clean - go vet ./... - go test -cover -coverprofile=./cover.out ./... - staticcheck ./... +.PHONY: ci +ci: clean + @bash ./scripts/ci.sh .PHONY: test-coverage test-coverage: test diff --git a/README.md b/README.md index c68c51c..cee3c09 100644 --- a/README.md +++ b/README.md @@ -3,28 +3,25 @@ `vdm` is an alternative to e.g. git submodules for managing arbitrary external dependencies for the same reasons, in a more sane way. -To get started, you'll need a `vdm` spec file, which is just a JSON array of all -your external dependencies along with (usually) their revisions & where you want -them to live in your repo: - -```jsonc -[ - { - "type": "git", // the default - "remote": "https://github.com/opensourcecorp/go-common", - "local_path": "./deps/go-common", - "version": "v0.2.0", // tag; can also be a branch, short or long commit hash, or the word 'latest' - }, - { - "type": "file", // the 'file' type assumes the version is in the remote field itself, so 'version' is omitted - "remote": "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto", - "local_path": "./deps/proto/http/http.proto" - } -] +To get started, you'll need a `vdm` spec file, which is just a YAML file +specifying all your external dependencies along with (usually) their revisions & +where you want them to live in your repo: + +```yaml +remotes: + + - type: "git" # the default + remote: "https://github.com/opensourcecorp/go-common" + local_path: "./deps/go-common" + version: "v0.2.0" # tag example; can also be a branch, short or long commit hash, or the word 'latest' + + - type: "file" # the 'file' type assumes the version is in the remote field itself, so 'version' is omitted + remote: "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto" + local_path: "./deps/proto/http/http.proto" ``` You can have as many dependency specifications in that array as you want. By -default, this spec file is called `vdm.json` and lives at the calling location +default, this spec file is called `vdm.yaml` and lives at the calling location (which is probably your repo's root), but you can call it whatever you want and point to it using the `-spec-file` flag to `vdm`. @@ -43,8 +40,8 @@ update your spec file and run `vdm sync` again. After running `vdm sync` with the above example spec file, your directory tree would look like this: -``` -./vdm.json +```txt +./vdm.yaml ./deps/ go-common/ diff --git a/cmd/root.go b/cmd/root.go index 0d24e5f..b5267da 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -22,7 +22,7 @@ type RootFlags struct { var RootFlagValues RootFlags func init() { - rootCmd.PersistentFlags().StringVar(&RootFlagValues.SpecFilePath, "specfile-path", "./vdm.json", "Path to vdm specfile") + rootCmd.PersistentFlags().StringVar(&RootFlagValues.SpecFilePath, "specfile-path", "./vdm.yaml", "Path to vdm specfile") rootCmd.PersistentFlags().BoolVar(&RootFlagValues.Debug, "debug", false, "Show debug logs") if RootFlagValues.Debug { diff --git a/cmd/sync.go b/cmd/sync.go index d5822e6..2ec6388 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -4,7 +4,7 @@ import ( "fmt" "path/filepath" - "github.com/opensourcecorp/vdm/internal/remote" + "github.com/opensourcecorp/vdm/internal/remotes" "github.com/opensourcecorp/vdm/internal/vdmspec" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -26,54 +26,52 @@ func syncExecute(_ *cobra.Command, _ []string) error { // sync does the heavy lifting to ensure that the local directory tree(s) match // the desired state as defined in the specfile. func sync() error { - specs, err := vdmspec.GetSpecsFromFile(RootFlagValues.SpecFilePath) + spec, err := vdmspec.GetSpecFromFile(RootFlagValues.SpecFilePath) if err != nil { return fmt.Errorf("getting specs from spec file: %w", err) } - for _, spec := range specs { - err := spec.Validate() - if err != nil { - return fmt.Errorf("your vdm spec file is malformed: %w", err) - } + err = spec.Validate() + if err != nil { + return fmt.Errorf("your vdm spec file is malformed: %w", err) } SpecLoop: - for _, spec := range specs { + for _, remote := range spec.Remotes { // process stored vdm metafile so we know what operations to actually // perform for existing directories - vdmMeta, err := spec.GetVDMMeta() + vdmMeta, err := remote.GetVDMMeta() if err != nil { return fmt.Errorf("getting vdm metadata file for sync: %w", err) } - if vdmMeta == (vdmspec.Spec{}) { - logrus.Infof("%s not found at local path '%s' -- will be created", vdmspec.MetaFileName, filepath.Join(spec.LocalPath)) + if vdmMeta == (vdmspec.Remote{}) { + logrus.Infof("%s not found at local path '%s' -- will be created", vdmspec.MetaFileName, filepath.Join(remote.LocalPath)) } else { - if vdmMeta.Version != spec.Version && vdmMeta.Remote != spec.Remote { - logrus.Infof("Will change '%s' from current local version spec '%s' to '%s'...", spec.Remote, vdmMeta.Version, spec.Version) + if vdmMeta.Version != remote.Version && vdmMeta.Remote != remote.Remote { + logrus.Infof("Will change '%s' from current local version spec '%s' to '%s'...", remote.Remote, vdmMeta.Version, remote.Version) panic("not implemented") } else { - logrus.Infof("Version unchanged (%s) in spec file for '%s' --> '%s', skipping", spec.Version, spec.Remote, spec.LocalPath) + logrus.Infof("Version unchanged (%s) in spec file for '%s' --> '%s', skipping", remote.Version, remote.Remote, remote.LocalPath) continue SpecLoop } } - switch spec.Type { + switch remote.Type { case "git", "": - remote.SyncGit(spec) + remotes.SyncGit(remote) case "file": - remote.SyncFile(spec) + remotes.SyncFile(remote) default: - return fmt.Errorf("unrecognized remote type '%s'", spec.Type) + return fmt.Errorf("unrecognized remote type '%s'", remote.Type) } - err = spec.WriteVDMMeta() + err = remote.WriteVDMMeta() if err != nil { return fmt.Errorf("could not write %s file to disk: %w", vdmspec.MetaFileName, err) } - logrus.Infof("%s -- Done.", spec.OpMsg()) + logrus.Infof("%s -- Done.", remote.OpMsg()) } logrus.Info("All done!") diff --git a/cmd/sync_test.go b/cmd/sync_test.go index 077dff2..14b4267 100644 --- a/cmd/sync_test.go +++ b/cmd/sync_test.go @@ -13,11 +13,11 @@ import ( const testVDMRoot = "../testdata" var ( - testSpecFilePath = filepath.Join(testVDMRoot, "vdm.json") + testSpecFilePath = filepath.Join(testVDMRoot, "vdm.yaml") ) func TestSync(t *testing.T) { - specs, err := vdmspec.GetSpecsFromFile(testSpecFilePath) + spec, err := vdmspec.GetSpecFromFile(testSpecFilePath) require.NoError(t, err) // Need to override for test @@ -27,25 +27,25 @@ func TestSync(t *testing.T) { t.Run("SyncGit", func(t *testing.T) { t.Run("spec[0] used a tag", func(t *testing.T) { - vdmMeta, err := specs[0].GetVDMMeta() + vdmMeta, err := spec.Remotes[0].GetVDMMeta() assert.NoError(t, err) assert.Equal(t, "v0.2.0", vdmMeta.Version) }) t.Run("spec[1] used 'latest'", func(t *testing.T) { - vdmMeta, err := specs[1].GetVDMMeta() + vdmMeta, err := spec.Remotes[1].GetVDMMeta() assert.NoError(t, err) assert.Equal(t, "latest", vdmMeta.Version) }) t.Run("spec[2] used a branch", func(t *testing.T) { - vdmMeta, err := specs[2].GetVDMMeta() + vdmMeta, err := spec.Remotes[2].GetVDMMeta() assert.NoError(t, err) assert.Equal(t, "main", vdmMeta.Version) }) t.Run("spec[3] used a hash", func(t *testing.T) { - vdmMeta, err := specs[3].GetVDMMeta() + vdmMeta, err := spec.Remotes[3].GetVDMMeta() assert.NoError(t, err) assert.Equal(t, "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f", vdmMeta.Version) }) @@ -53,15 +53,15 @@ func TestSync(t *testing.T) { t.Run("SyncFile", func(t *testing.T) { t.Run("spec[4] had an implcit version", func(t *testing.T) { - vdmMeta, err := specs[4].GetVDMMeta() + vdmMeta, err := spec.Remotes[4].GetVDMMeta() assert.NoError(t, err) assert.Equal(t, "", vdmMeta.Version) }) }) t.Cleanup(func() { - for _, spec := range specs { - err := os.RemoveAll(spec.LocalPath) + for _, remote := range spec.Remotes { + err := os.RemoveAll(remote.LocalPath) assert.NoError(t, err) } }) diff --git a/go.mod b/go.mod index 00cee8f..c25aebd 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.2 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -14,5 +15,4 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/internal/remote/doc.go b/internal/remote/doc.go deleted file mode 100644 index 256b804..0000000 --- a/internal/remote/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package remote defines logic for the various types of remotes that vdm supports. -*/ -package remote diff --git a/internal/remote/file_test.go b/internal/remote/file_test.go deleted file mode 100644 index fbe5b64..0000000 --- a/internal/remote/file_test.go +++ /dev/null @@ -1 +0,0 @@ -package remote diff --git a/internal/remotes/doc.go b/internal/remotes/doc.go new file mode 100644 index 0000000..cc7572b --- /dev/null +++ b/internal/remotes/doc.go @@ -0,0 +1,4 @@ +/* +Package remotes defines logic for the various types of remotes that vdm supports. +*/ +package remotes diff --git a/internal/remote/file.go b/internal/remotes/file.go similarity index 60% rename from internal/remote/file.go rename to internal/remotes/file.go index 4fbbabe..e65177b 100644 --- a/internal/remote/file.go +++ b/internal/remotes/file.go @@ -1,4 +1,4 @@ -package remote +package remotes import ( "errors" @@ -13,66 +13,66 @@ import ( ) // SyncFile is the root of the sync operations for "file" remote types. -func SyncFile(spec vdmspec.Spec) error { - fileExists, err := checkFileExists(spec) +func SyncFile(remote vdmspec.Remote) error { + fileExists, err := checkFileExists(remote) if err != nil { return fmt.Errorf("checking if file exists locally: %w", err) } if !fileExists { - logrus.Infof("File '%s' does not exist locally, retrieving", spec.LocalPath) - err = retrieveFile(spec) + logrus.Infof("File '%s' does not exist locally, retrieving", remote.LocalPath) + err = retrieveFile(remote) if err != nil { return fmt.Errorf("retrieving file: %w", err) } } else { - logrus.Infof("File '%s' already exists locally, skipping", spec.LocalPath) + logrus.Infof("File '%s' already exists locally, skipping", remote.LocalPath) } return nil } -func checkFileExists(spec vdmspec.Spec) (bool, error) { - fullPath, err := filepath.Abs(spec.LocalPath) +func checkFileExists(remote vdmspec.Remote) (bool, error) { + fullPath, err := filepath.Abs(remote.LocalPath) if err != nil { - return false, fmt.Errorf("determining abspath for file '%s': %w", spec.LocalPath, err) + return false, fmt.Errorf("determining abspath for file '%s': %w", remote.LocalPath, err) } - _, err = os.Stat(spec.LocalPath) + _, err = os.Stat(remote.LocalPath) if errors.Is(err, os.ErrNotExist) { return false, nil } else if err != nil { - return false, fmt.Errorf("couldn't check if %s exists at '%s': %w", spec.LocalPath, fullPath, err) + return false, fmt.Errorf("couldn't check if %s exists at '%s': %w", remote.LocalPath, fullPath, err) } return true, nil } -func retrieveFile(spec vdmspec.Spec) error { - resp, err := http.Get(spec.Remote) +func retrieveFile(remote vdmspec.Remote) error { + resp, err := http.Get(remote.Remote) if err != nil { - return fmt.Errorf("retrieving remote file '%s': %w", spec.Remote, err) + return fmt.Errorf("retrieving remote file '%s': %w", remote.Remote, err) } defer func() { if closeErr := resp.Body.Close(); closeErr != nil { - logrus.Errorf("closing response body after remote file '%s' retrieval: %v", spec.Remote, err) + logrus.Errorf("closing response body after remote file '%s' retrieval: %v", remote.Remote, err) } }() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unsuccessful status code '%d' from server when retrieving remote file '%s'", resp.StatusCode, spec.Remote) + return fmt.Errorf("unsuccessful status code '%d' from server when retrieving remote file '%s'", resp.StatusCode, remote.Remote) } // Note: I would normally use os.WriteFile() using the returned bytes // directly, but the internet says this os.Create()/io.Copy() approach // appears to be idiomatic - outFile, err := os.Create(spec.LocalPath) + outFile, err := os.Create(remote.LocalPath) if err != nil { - return fmt.Errorf("creating landing file '%s' for remote file: %w", spec.LocalPath, err) + return fmt.Errorf("creating landing file '%s' for remote file: %w", remote.LocalPath, err) } defer func() { if closeErr := outFile.Close(); closeErr != nil { - logrus.Errorf("closing local file '%s' after remote file '%s' retrieval: %v", spec.LocalPath, spec.Remote, err) + logrus.Errorf("closing local file '%s' after remote file '%s' retrieval: %v", remote.LocalPath, remote.Remote, err) } }() @@ -80,7 +80,7 @@ func retrieveFile(spec vdmspec.Spec) error { if err != nil { return fmt.Errorf("copying HTTP response to disk: ") } - logrus.Debugf("wrote %d bytes to '%s'", bytesWritten, spec.LocalPath) + logrus.Debugf("wrote %d bytes to '%s'", bytesWritten, remote.LocalPath) return nil } diff --git a/internal/remotes/file_test.go b/internal/remotes/file_test.go new file mode 100644 index 0000000..a558bee --- /dev/null +++ b/internal/remotes/file_test.go @@ -0,0 +1 @@ +package remotes diff --git a/internal/remote/git.go b/internal/remotes/git.go similarity index 65% rename from internal/remote/git.go rename to internal/remotes/git.go index d416649..4026a23 100644 --- a/internal/remote/git.go +++ b/internal/remotes/git.go @@ -1,4 +1,4 @@ -package remote +package remotes import ( "errors" @@ -12,23 +12,23 @@ import ( ) // SyncGit is the root of the sync operations for "git" remote types. -func SyncGit(spec vdmspec.Spec) error { - err := gitClone(spec) +func SyncGit(remote vdmspec.Remote) error { + err := gitClone(remote) if err != nil { return fmt.Errorf("cloing remote: %w", err) } - if spec.Version != "latest" { - logrus.Infof("%s -- Setting specified version...", spec.OpMsg()) - checkoutCmd := exec.Command("git", "-C", spec.LocalPath, "checkout", spec.Version) + if remote.Version != "latest" { + logrus.Infof("%s -- Setting specified version...", remote.OpMsg()) + checkoutCmd := exec.Command("git", "-C", remote.LocalPath, "checkout", remote.Version) checkoutOutput, err := checkoutCmd.CombinedOutput() if err != nil { return fmt.Errorf("error checking out specified revision: exec error '%w', with output: %s", err, string(checkoutOutput)) } } - logrus.Debugf("removing .git dir for local path '%s'", spec.LocalPath) - dotGitPath := filepath.Join(spec.LocalPath, ".git") + logrus.Debugf("removing .git dir for local path '%s'", remote.LocalPath) + dotGitPath := filepath.Join(remote.LocalPath, ".git") err = os.RemoveAll(dotGitPath) if err != nil { return fmt.Errorf("removing directory %s: %w", dotGitPath, err) @@ -48,25 +48,25 @@ func checkGitAvailable() error { return nil } -func gitClone(spec vdmspec.Spec) error { +func gitClone(remote vdmspec.Remote) error { err := checkGitAvailable() if err != nil { - return fmt.Errorf("remote '%s' is a git type, but git may not installed/available on PATH: %w", spec.Remote, err) + return fmt.Errorf("remote '%s' is a git type, but git may not installed/available on PATH: %w", remote.Remote, err) } // If users want "latest", then we can just do a depth-one clone and // skip the checkout operation. But if they want non-latest, we need the // full history to be able to find a specified revision var cloneCmdArgs []string - if spec.Version == "latest" { - logrus.Debugf("%s -- version specified as 'latest', so making shallow clone and skipping separate checkout operation", spec.OpMsg()) - cloneCmdArgs = []string{"clone", "--depth=1", spec.Remote, spec.LocalPath} + if remote.Version == "latest" { + logrus.Debugf("%s -- version specified as 'latest', so making shallow clone and skipping separate checkout operation", remote.OpMsg()) + cloneCmdArgs = []string{"clone", "--depth=1", remote.Remote, remote.LocalPath} } else { - logrus.Debugf("%s -- version specified as NOT latest, so making regular clone and will make separate checkout operation", spec.OpMsg()) - cloneCmdArgs = []string{"clone", spec.Remote, spec.LocalPath} + logrus.Debugf("%s -- version specified as NOT latest, so making regular clone and will make separate checkout operation", remote.OpMsg()) + cloneCmdArgs = []string{"clone", remote.Remote, remote.LocalPath} } - logrus.Infof("%s -- Retrieving...", spec.OpMsg()) + logrus.Infof("%s -- Retrieving...", remote.OpMsg()) cloneCmd := exec.Command("git", cloneCmdArgs...) cloneOutput, err := cloneCmd.CombinedOutput() if err != nil { diff --git a/internal/remote/git_test.go b/internal/remotes/git_test.go similarity index 96% rename from internal/remote/git_test.go rename to internal/remotes/git_test.go index 6f821f6..712b77f 100644 --- a/internal/remote/git_test.go +++ b/internal/remotes/git_test.go @@ -1,4 +1,4 @@ -package remote +package remotes import ( "os" @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/require" ) -func getTestGitSpec() vdmspec.Spec { +func getTestGitSpec() vdmspec.Remote { specLocalPath := "./deps/go-common" - return vdmspec.Spec{ + return vdmspec.Remote{ Type: "git", Remote: "https://github.com/opensourcecorp/go-common", Version: "v0.2.0", diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index 535eb25..9cad313 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -1,7 +1,6 @@ package vdmspec import ( - "encoding/json" "errors" "fmt" "os" @@ -9,23 +8,31 @@ import ( "strings" "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" ) type Spec struct { - Type string `json:"type,omitempty"` - Remote string `json:"remote"` - Version string `json:"version,omitempty"` - LocalPath string `json:"local_path"` + Remotes []Remote `json:"remotes" yaml:"remotes"` } -const MetaFileName = "VDMMETA" +type Remote struct { + Type string `json:"type,omitempty" yaml:"type,omitempty"` + Remote string `json:"remote" yaml:"remote"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` + LocalPath string `json:"local_path" yaml:"local_path"` +} + +const ( + MetaFileName = "VDMMETA" + typeFile = "file" +) -func (spec Spec) MakeMetaFilePath() string { - metaFilePath := filepath.Join(spec.LocalPath, MetaFileName) +func (r Remote) MakeMetaFilePath() string { + metaFilePath := filepath.Join(r.LocalPath, MetaFileName) // TODO: this is brittle, but it's the best I can think of right now - if spec.Type == "file" { - fileDir := filepath.Dir(spec.LocalPath) - fileName := filepath.Base(spec.LocalPath) + if r.Type == typeFile { + fileDir := filepath.Dir(r.LocalPath) + fileName := filepath.Base(r.LocalPath) // converts to e.g. 'VDMMETA_http.proto' metaFilePath = filepath.Join(fileDir, fmt.Sprintf("%s_%s", MetaFileName, fileName)) } @@ -33,9 +40,9 @@ func (spec Spec) MakeMetaFilePath() string { return metaFilePath } -func (spec Spec) WriteVDMMeta() error { - metaFilePath := spec.MakeMetaFilePath() - vdmMetaContent, err := json.MarshalIndent(spec, "", " ") +func (r Remote) WriteVDMMeta() error { + metaFilePath := r.MakeMetaFilePath() + vdmMetaContent, err := yaml.Marshal(r) if err != nil { return fmt.Errorf("writing %s: %w", metaFilePath, err) } @@ -51,38 +58,38 @@ func (spec Spec) WriteVDMMeta() error { return nil } -func (spec Spec) GetVDMMeta() (Spec, error) { - metaFilePath := spec.MakeMetaFilePath() +func (r Remote) GetVDMMeta() (Remote, error) { + metaFilePath := r.MakeMetaFilePath() _, err := os.Stat(metaFilePath) if errors.Is(err, os.ErrNotExist) { - return Spec{}, nil // this is ok, because it might literally not exist yet + return Remote{}, nil // this is ok, because it might literally not exist yet } else if err != nil { - return Spec{}, fmt.Errorf("couldn't check if %s exists at '%s': %w", MetaFileName, metaFilePath, err) + return Remote{}, fmt.Errorf("couldn't check if %s exists at '%s': %w", MetaFileName, metaFilePath, err) } vdmMetaFile, err := os.ReadFile(metaFilePath) if err != nil { logrus.Debugf("error reading VMDMMETA from disk: %v", err) - return Spec{}, fmt.Errorf("there was a problem reading the %s file from '%s': %w", MetaFileName, metaFilePath, err) + return Remote{}, fmt.Errorf("there was a problem reading the %s file from '%s': %w", MetaFileName, metaFilePath, err) } logrus.Debugf("%s contents read:\n%s", MetaFileName, string(vdmMetaFile)) - var vdmMeta Spec - err = json.Unmarshal(vdmMetaFile, &vdmMeta) + var vdmMeta Remote + err = yaml.Unmarshal(vdmMetaFile, &vdmMeta) if err != nil { logrus.Debugf("error during %s unmarshal: %v", MetaFileName, err) - return Spec{}, fmt.Errorf("there was a problem reading the contents of the %s file at '%s': %v", MetaFileName, metaFilePath, err) + return Remote{}, fmt.Errorf("there was a problem reading the contents of the %s file at '%s': %v", MetaFileName, metaFilePath, err) } logrus.Debugf("file %s unmarshalled: %+v", MetaFileName, vdmMeta) return vdmMeta, nil } -func GetSpecsFromFile(specFilePath string) ([]Spec, error) { +func GetSpecFromFile(specFilePath string) (Spec, error) { specFile, err := os.ReadFile(specFilePath) if err != nil { logrus.Debugf("error reading specfile from disk: %v", err) - return nil, fmt.Errorf( + return Spec{}, fmt.Errorf( strings.Join([]string{ "there was a problem reading your vdm file from '%s' -- does it not exist?", "Either pass the -spec-file flag, or create one in the default location (details in the README).", @@ -95,23 +102,23 @@ func GetSpecsFromFile(specFilePath string) ([]Spec, error) { } logrus.Debugf("specfile contents read:\n%s", string(specFile)) - var specs []Spec - err = json.Unmarshal(specFile, &specs) + var spec Spec + err = yaml.Unmarshal(specFile, &spec) if err != nil { logrus.Debugf("error during specfile unmarshal: %v", err) - return nil, fmt.Errorf("there was a problem reading the contents of your vdm spec file: %w", err) + return Spec{}, fmt.Errorf("there was a problem reading the contents of your vdm spec file: %w", err) } - logrus.Debugf("vdmSpecs unmarshalled: %+v", specs) + logrus.Debugf("vdmSpecs unmarshalled: %+v", spec) - return specs, nil + return spec, nil } // OpMsg constructs a loggable message outlining the specific operation being // performed at the moment -func (spec Spec) OpMsg() string { - if spec.Version != "" { - return fmt.Sprintf("%s@%s --> %s", spec.Remote, spec.Version, spec.LocalPath) +func (r Remote) OpMsg() string { + if r.Version != "" { + return fmt.Sprintf("%s@%s --> %s", r.Remote, r.Version, r.LocalPath) } else { - return fmt.Sprintf("%s --> %s", spec.Remote, spec.LocalPath) + return fmt.Sprintf("%s --> %s", r.Remote, r.LocalPath) } } diff --git a/internal/vdmspec/spec_test.go b/internal/vdmspec/spec_test.go index aef86e6..783baa4 100644 --- a/internal/vdmspec/spec_test.go +++ b/internal/vdmspec/spec_test.go @@ -15,13 +15,13 @@ const testVDMRoot = "../../testdata" var ( testVDMMetaFilePath = filepath.Join(testVDMRoot, MetaFileName) - testSpec = Spec{ + testRemote = Remote{ Remote: "https://some-remote", Version: "v1.0.0", LocalPath: testVDMRoot, } - testSpecFilePath = filepath.Join(testVDMRoot, "vdm.json") + testSpecFilePath = filepath.Join(testVDMRoot, "vdm.yaml") testVDMMetaContents = fmt.Sprintf( `{"remote": "https://some-remote", "version": "v1.0.0", "local_path": "%s"}`, @@ -34,9 +34,9 @@ func TestVDMMeta(t *testing.T) { err := os.WriteFile(testVDMMetaFilePath, []byte(testVDMMetaContents), 0644) require.NoError(t, err) - got, err := testSpec.GetVDMMeta() + got, err := testRemote.GetVDMMeta() assert.NoError(t, err) - assert.Equal(t, testSpec, got) + assert.Equal(t, testRemote, got) t.Cleanup(func() { err := os.RemoveAll(testVDMMetaFilePath) @@ -46,15 +46,15 @@ func TestVDMMeta(t *testing.T) { t.Run("WriteVDMMeta", func(t *testing.T) { // Needs to have parent dir(s) exist for write to work - err := os.MkdirAll(testSpec.LocalPath, 0644) + err := os.MkdirAll(testRemote.LocalPath, 0644) require.NoError(t, err) - err = testSpec.WriteVDMMeta() + err = testRemote.WriteVDMMeta() require.NoError(t, err) - got, err := testSpec.GetVDMMeta() + got, err := testRemote.GetVDMMeta() assert.NoError(t, err) - assert.Equal(t, testSpec, got) + assert.Equal(t, testRemote, got) t.Cleanup(func() { err := os.RemoveAll(testVDMMetaFilePath) @@ -63,9 +63,9 @@ func TestVDMMeta(t *testing.T) { }) t.Run("GetSpecsFromFile", func(t *testing.T) { - specs, err := GetSpecsFromFile(testSpecFilePath) + spec, err := GetSpecFromFile(testSpecFilePath) assert.NoError(t, err) - assert.Equal(t, 5, len(specs)) + assert.Equal(t, 5, len(spec.Remotes)) t.Cleanup(func() { err := os.RemoveAll(testVDMMetaFilePath) diff --git a/internal/vdmspec/validate.go b/internal/vdmspec/validate.go index 0c5abf7..c9d12f2 100644 --- a/internal/vdmspec/validate.go +++ b/internal/vdmspec/validate.go @@ -11,43 +11,45 @@ import ( func (spec Spec) Validate() error { var allErrors []error - // Remote field - logrus.Debugf("validating field 'Remote' for %+v", spec) - if len(spec.Remote) == 0 { - allErrors = append(allErrors, errors.New("all 'remote' fields must be non-zero length")) - } - protocolRegex := regexp.MustCompile(`(http(s?)://|git://|git@)`) - if !protocolRegex.MatchString(spec.Remote) { - allErrors = append( - allErrors, - fmt.Errorf("remote provided as '%s', but all 'remote' fields must begin with a protocol specifier or other valid prefix (e.g. 'https://', '(user|git)@', etc.)", spec.Remote), - ) - } + for remoteIndex, remote := range spec.Remotes { + // Remote field + logrus.Debugf("Index %d: validating field 'Remote' for %+v", remoteIndex, remote) + if len(remote.Remote) == 0 { + allErrors = append(allErrors, errors.New("all 'remote' fields must be non-zero length")) + } + protocolRegex := regexp.MustCompile(`(http(s?)://|git://|git@)`) + if !protocolRegex.MatchString(remote.Remote) { + allErrors = append( + allErrors, + fmt.Errorf("remote #%d provided as '%s', but all 'remote' fields must begin with a protocol specifier or other valid prefix (e.g. 'https://', '(user|git)@', etc.)", remoteIndex, remote.Remote), + ) + } - // Version field - logrus.Debugf("validating field 'Version' for %+v", spec) - if spec.Type == "git" && len(spec.Version) == 0 { - allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length. If you don't care about the version (even though you probably should), then use 'latest'")) - } - if spec.Type == "file" && len(spec.Version) > 0 { - logrus.Warnf("NOTE: Remote '%s' specified as type '%s', which does not take explicit version info (you provided '%s'); ignoring version field", spec.Remote, spec.Type, spec.Version) - } + // Version field + logrus.Debugf("Index %d: validating field 'Version' for %+v", remoteIndex, remote) + if remote.Type == "git" && len(remote.Version) == 0 { + allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length. If you don't care about the version (even though you probably should), then use 'latest'")) + } + if remote.Type == "file" && len(remote.Version) > 0 { + logrus.Warnf("NOTE: Remote #%d '%s' specified as type '%s', which does not take explicit version info (you provided '%s'); ignoring version field", remoteIndex, remote.Remote, remote.Type, remote.Version) + } - // LocalPath field - logrus.Debugf("validating field 'LocalPath' for %+v", spec) - if len(spec.LocalPath) == 0 { - allErrors = append(allErrors, errors.New("all 'local_path' fields must be non-zero length")) - } + // LocalPath field + logrus.Debugf("Index #%d: validating field 'LocalPath' for %+v", remoteIndex, remote) + if len(remote.LocalPath) == 0 { + allErrors = append(allErrors, errors.New("all 'local_path' fields must be non-zero length")) + } - // Type field - logrus.Debugf("validating field 'Type' for %+v", spec) - typeMap := map[string]int{ - "git": 1, - "": 2, // also git - "file": 3, - } - if _, ok := typeMap[spec.Type]; !ok { - allErrors = append(allErrors, fmt.Errorf("unrecognized remote type '%s'", spec.Type)) + // Type field + logrus.Debugf("Index #%d: validating field 'Type' for %+v", remoteIndex, remote) + typeMap := map[string]int{ + "git": 1, + "": 2, // also git + "file": 3, + } + if _, ok := typeMap[remote.Type]; !ok { + allErrors = append(allErrors, fmt.Errorf("unrecognized remote type '%s'", remote.Type)) + } } if len(allErrors) > 0 { diff --git a/internal/vdmspec/validate_test.go b/internal/vdmspec/validate_test.go index 38ed830..ffdc497 100644 --- a/internal/vdmspec/validate_test.go +++ b/internal/vdmspec/validate_test.go @@ -9,9 +9,11 @@ import ( func TestValidate(t *testing.T) { t.Run("passes", func(t *testing.T) { spec := Spec{ - Remote: "https://some-remote", - Version: "v1.0.0", - LocalPath: "./deps/some-remote", + Remotes: []Remote{{ + Remote: "https://some-remote", + Version: "v1.0.0", + LocalPath: "./deps/some-remote", + }}, } err := spec.Validate() assert.NoError(t, err) @@ -19,9 +21,11 @@ func TestValidate(t *testing.T) { t.Run("fails on zero-length remote", func(t *testing.T) { spec := Spec{ - Remote: "", - Version: "v1.0.0", - LocalPath: "./deps/some-remote", + Remotes: []Remote{{ + Remote: "", + Version: "v1.0.0", + LocalPath: "./deps/some-remote", + }}, } err := spec.Validate() assert.Error(t, err) @@ -29,9 +33,11 @@ func TestValidate(t *testing.T) { t.Run("fails on remote without valid protocol", func(t *testing.T) { spec := Spec{ - Remote: "some-remote", - Version: "v1.0.0", - LocalPath: "./deps/some-remote", + Remotes: []Remote{{ + Remote: "some-remote", + Version: "v1.0.0", + LocalPath: "./deps/some-remote", + }}, } err := spec.Validate() assert.Error(t, err) @@ -39,10 +45,12 @@ func TestValidate(t *testing.T) { t.Run("fails on zero-length version for git remote type", func(t *testing.T) { spec := Spec{ - Remote: "https://some-remote", - Version: "", - LocalPath: "./deps/some-remote", - Type: "git", + Remotes: []Remote{{ + Remote: "https://some-remote", + Version: "", + LocalPath: "./deps/some-remote", + Type: "git", + }}, } err := spec.Validate() assert.Error(t, err) @@ -50,10 +58,12 @@ func TestValidate(t *testing.T) { t.Run("fails on unrecognized remote type", func(t *testing.T) { spec := Spec{ - Remote: "https://some-remote", - Version: "", - LocalPath: "./deps/some-remote", - Type: "bad", + Remotes: []Remote{{ + Remote: "https://some-remote", + Version: "", + LocalPath: "./deps/some-remote", + Type: "bad", + }}, } err := spec.Validate() assert.Error(t, err) @@ -61,9 +71,11 @@ func TestValidate(t *testing.T) { t.Run("fails on zero-length local path", func(t *testing.T) { spec := Spec{ - Remote: "https://some-remote", - Version: "v1.0.0", - LocalPath: "", + Remotes: []Remote{{ + Remote: "https://some-remote", + Version: "v1.0.0", + LocalPath: "", + }}, } err := spec.Validate() assert.Error(t, err) diff --git a/scripts/ci.sh b/scripts/ci.sh index 3fe6905..4efbba6 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -1,4 +1,31 @@ #!/usr/bin/env bash set -euo pipefail -make test +failures=() + +printf '> Running CI checks\n' + +printf '>> Go vet\n' +if ! go vet ./... ; then + printf '>>> Failed go-vet check\n' + failures+=('go-vet') +fi + +printf '>> Go linter\n' +if ! staticcheck ./... ; then + printf '>>> Failed go-lint\n' + failures+=('go-lint') +fi + +printf '>> Go test\n' +if ! go test -cover -coverprofile=./cover.out ./... ; then + printf '>>> Failed go-test check\n' + failures+=('go-test') +fi + +if [[ "${#failures[@]}" -gt 0 ]] ; then + printf '> One or more checks failed, see output above\n' + exit 1 +fi + +printf '> All checks passed!\n' diff --git a/testdata/vdm.json b/testdata/vdm.json deleted file mode 100644 index 473830e..0000000 --- a/testdata/vdm.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - { - "remote": "https://github.com/opensourcecorp/go-common", - "version": "v0.2.0", - "local_path": "./deps/go-common-tag" - }, - { - "remote": "https://github.com/opensourcecorp/go-common", - "version": "latest", - "local_path": "./deps/go-common-latest" - }, - { - "remote": "https://github.com/opensourcecorp/go-common", - "version": "main", - "local_path": "./deps/go-common-branch" - }, - { - "remote": "https://github.com/opensourcecorp/go-common", - "version": "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f", - "local_path": "./deps/go-common-hash" - }, - { - "type": "file", - "remote": "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto", - "local_path": "./deps/http.proto" - } -] diff --git a/testdata/vdm.yaml b/testdata/vdm.yaml new file mode 100644 index 0000000..2cfab27 --- /dev/null +++ b/testdata/vdm.yaml @@ -0,0 +1,16 @@ +remotes: + - remote: "https://github.com/opensourcecorp/go-common" + version: "v0.2.0" + local_path: "./deps/go-common-tag" + - remote: "https://github.com/opensourcecorp/go-common" + version: "latest" + local_path: "./deps/go-common-latest" + - remote: "https://github.com/opensourcecorp/go-common" + version: "main" + local_path: "./deps/go-common-branch" + - remote: "https://github.com/opensourcecorp/go-common" + version: "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f" + local_path: "./deps/go-common-hash" + - type: "file" + remote: "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto" + local_path: "./deps/http.proto" From dc91c135384d2da5ac11d96419bd05d061485157 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Thu, 25 Jul 2024 23:11:18 -0500 Subject: [PATCH 13/29] Replace logrus with a small custom message printer --- cmd/root.go | 5 ----- cmd/sync.go | 12 ++++++------ go.mod | 2 -- go.sum | 5 ----- internal/message/message.go | 31 +++++++++++++++++++++++++++++++ internal/remotes/file.go | 12 ++++++------ internal/remotes/git.go | 16 ++++++++-------- internal/vdmspec/spec.go | 20 ++++++++++---------- internal/vdmspec/validate.go | 14 +++++++------- main.go | 4 ++-- 10 files changed, 70 insertions(+), 51 deletions(-) create mode 100644 internal/message/message.go diff --git a/cmd/root.go b/cmd/root.go index b5267da..a0172e3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,7 +3,6 @@ package cmd import ( "fmt" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -25,10 +24,6 @@ func init() { rootCmd.PersistentFlags().StringVar(&RootFlagValues.SpecFilePath, "specfile-path", "./vdm.yaml", "Path to vdm specfile") rootCmd.PersistentFlags().BoolVar(&RootFlagValues.Debug, "debug", false, "Show debug logs") - if RootFlagValues.Debug { - logrus.SetLevel(logrus.DebugLevel) - } - rootCmd.AddCommand(syncCmd) } diff --git a/cmd/sync.go b/cmd/sync.go index 2ec6388..edbedc7 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -4,9 +4,9 @@ import ( "fmt" "path/filepath" + "github.com/opensourcecorp/vdm/internal/message" "github.com/opensourcecorp/vdm/internal/remotes" "github.com/opensourcecorp/vdm/internal/vdmspec" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -46,13 +46,13 @@ SpecLoop: } if vdmMeta == (vdmspec.Remote{}) { - logrus.Infof("%s not found at local path '%s' -- will be created", vdmspec.MetaFileName, filepath.Join(remote.LocalPath)) + message.Infof("%s not found at local path '%s' -- will be created", vdmspec.MetaFileName, filepath.Join(remote.LocalPath)) } else { if vdmMeta.Version != remote.Version && vdmMeta.Remote != remote.Remote { - logrus.Infof("Will change '%s' from current local version spec '%s' to '%s'...", remote.Remote, vdmMeta.Version, remote.Version) + message.Infof("Will change '%s' from current local version spec '%s' to '%s'...", remote.Remote, vdmMeta.Version, remote.Version) panic("not implemented") } else { - logrus.Infof("Version unchanged (%s) in spec file for '%s' --> '%s', skipping", remote.Version, remote.Remote, remote.LocalPath) + message.Infof("Version unchanged (%s) in spec file for '%s' --> '%s', skipping", remote.Version, remote.Remote, remote.LocalPath) continue SpecLoop } } @@ -71,9 +71,9 @@ SpecLoop: return fmt.Errorf("could not write %s file to disk: %w", vdmspec.MetaFileName, err) } - logrus.Infof("%s -- Done.", remote.OpMsg()) + message.Infof("%s -- Done.", remote.OpMsg()) } - logrus.Info("All done!") + message.Infof("All done!") return nil } diff --git a/go.mod b/go.mod index c25aebd..01db9f7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/opensourcecorp/vdm go 1.19 require ( - github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.2 gopkg.in/yaml.v3 v3.0.1 @@ -14,5 +13,4 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect ) diff --git a/go.sum b/go.sum index 348225d..0df3271 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -16,13 +14,10 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/message/message.go b/internal/message/message.go new file mode 100644 index 0000000..1447f5f --- /dev/null +++ b/internal/message/message.go @@ -0,0 +1,31 @@ +// Package message controls message printing. THis isn't a "logging" package per +// se, but adds some niceties for log-like needs. +package message + +import ( + "fmt" + "os" +) + +func Debugf(format string, args ...any) { + if os.Getenv("DEBUG") != "" { + fmt.Printf("DEBUG: "+format+"\n", args...) + } +} + +func Infof(format string, args ...any) { + fmt.Printf(format+"\n", args...) +} + +func Warnf(format string, args ...any) { + fmt.Printf("WARNING: "+format+"\n", args...) +} + +func Errorf(format string, args ...any) { + fmt.Printf("ERROR: "+format+"\n", args...) +} + +func Fatalf(format string, args ...any) { + fmt.Printf("ERROR: "+format+"\n", args...) + os.Exit(1) +} diff --git a/internal/remotes/file.go b/internal/remotes/file.go index e65177b..8041961 100644 --- a/internal/remotes/file.go +++ b/internal/remotes/file.go @@ -8,8 +8,8 @@ import ( "os" "path/filepath" + "github.com/opensourcecorp/vdm/internal/message" "github.com/opensourcecorp/vdm/internal/vdmspec" - "github.com/sirupsen/logrus" ) // SyncFile is the root of the sync operations for "file" remote types. @@ -20,13 +20,13 @@ func SyncFile(remote vdmspec.Remote) error { } if !fileExists { - logrus.Infof("File '%s' does not exist locally, retrieving", remote.LocalPath) + message.Infof("File '%s' does not exist locally, retrieving", remote.LocalPath) err = retrieveFile(remote) if err != nil { return fmt.Errorf("retrieving file: %w", err) } } else { - logrus.Infof("File '%s' already exists locally, skipping", remote.LocalPath) + message.Infof("File '%s' already exists locally, skipping", remote.LocalPath) } return nil @@ -55,7 +55,7 @@ func retrieveFile(remote vdmspec.Remote) error { } defer func() { if closeErr := resp.Body.Close(); closeErr != nil { - logrus.Errorf("closing response body after remote file '%s' retrieval: %v", remote.Remote, err) + message.Errorf("closing response body after remote file '%s' retrieval: %v", remote.Remote, err) } }() @@ -72,7 +72,7 @@ func retrieveFile(remote vdmspec.Remote) error { } defer func() { if closeErr := outFile.Close(); closeErr != nil { - logrus.Errorf("closing local file '%s' after remote file '%s' retrieval: %v", remote.LocalPath, remote.Remote, err) + message.Errorf("closing local file '%s' after remote file '%s' retrieval: %v", remote.LocalPath, remote.Remote, err) } }() @@ -80,7 +80,7 @@ func retrieveFile(remote vdmspec.Remote) error { if err != nil { return fmt.Errorf("copying HTTP response to disk: ") } - logrus.Debugf("wrote %d bytes to '%s'", bytesWritten, remote.LocalPath) + message.Debugf("wrote %d bytes to '%s'", bytesWritten, remote.LocalPath) return nil } diff --git a/internal/remotes/git.go b/internal/remotes/git.go index 4026a23..a6f1f53 100644 --- a/internal/remotes/git.go +++ b/internal/remotes/git.go @@ -7,8 +7,8 @@ import ( "os/exec" "path/filepath" + "github.com/opensourcecorp/vdm/internal/message" "github.com/opensourcecorp/vdm/internal/vdmspec" - "github.com/sirupsen/logrus" ) // SyncGit is the root of the sync operations for "git" remote types. @@ -19,7 +19,7 @@ func SyncGit(remote vdmspec.Remote) error { } if remote.Version != "latest" { - logrus.Infof("%s -- Setting specified version...", remote.OpMsg()) + message.Infof("%s -- Setting specified version...", remote.OpMsg()) checkoutCmd := exec.Command("git", "-C", remote.LocalPath, "checkout", remote.Version) checkoutOutput, err := checkoutCmd.CombinedOutput() if err != nil { @@ -27,7 +27,7 @@ func SyncGit(remote vdmspec.Remote) error { } } - logrus.Debugf("removing .git dir for local path '%s'", remote.LocalPath) + message.Debugf("removing .git dir for local path '%s'", remote.LocalPath) dotGitPath := filepath.Join(remote.LocalPath, ".git") err = os.RemoveAll(dotGitPath) if err != nil { @@ -41,10 +41,10 @@ func checkGitAvailable() error { cmd := exec.Command("git", "--version") sysOutput, err := cmd.CombinedOutput() if err != nil { - logrus.Debugf("%s: %s", err.Error(), string(sysOutput)) + message.Debugf("%s: %s", err.Error(), string(sysOutput)) return errors.New("git does not seem to be available on your PATH, so cannot continue") } - logrus.Debug("git was found on PATH") + message.Debugf("git was found on PATH") return nil } @@ -59,14 +59,14 @@ func gitClone(remote vdmspec.Remote) error { // full history to be able to find a specified revision var cloneCmdArgs []string if remote.Version == "latest" { - logrus.Debugf("%s -- version specified as 'latest', so making shallow clone and skipping separate checkout operation", remote.OpMsg()) + message.Debugf("%s -- version specified as 'latest', so making shallow clone and skipping separate checkout operation", remote.OpMsg()) cloneCmdArgs = []string{"clone", "--depth=1", remote.Remote, remote.LocalPath} } else { - logrus.Debugf("%s -- version specified as NOT latest, so making regular clone and will make separate checkout operation", remote.OpMsg()) + message.Debugf("%s -- version specified as NOT latest, so making regular clone and will make separate checkout operation", remote.OpMsg()) cloneCmdArgs = []string{"clone", remote.Remote, remote.LocalPath} } - logrus.Infof("%s -- Retrieving...", remote.OpMsg()) + message.Infof("%s -- Retrieving...", remote.OpMsg()) cloneCmd := exec.Command("git", cloneCmdArgs...) cloneOutput, err := cloneCmd.CombinedOutput() if err != nil { diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index 9cad313..ffe3842 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -7,7 +7,7 @@ import ( "path/filepath" "strings" - "github.com/sirupsen/logrus" + "github.com/opensourcecorp/vdm/internal/message" "gopkg.in/yaml.v3" ) @@ -49,7 +49,7 @@ func (r Remote) WriteVDMMeta() error { vdmMetaContent = append(vdmMetaContent, []byte("\n")...) - logrus.Debugf("writing metadata file to '%s'", metaFilePath) + message.Debugf("writing metadata file to '%s'", metaFilePath) err = os.WriteFile(metaFilePath, vdmMetaContent, 0644) if err != nil { return fmt.Errorf("writing metadata file: %w", err) @@ -69,18 +69,18 @@ func (r Remote) GetVDMMeta() (Remote, error) { vdmMetaFile, err := os.ReadFile(metaFilePath) if err != nil { - logrus.Debugf("error reading VMDMMETA from disk: %v", err) + message.Debugf("error reading VMDMMETA from disk: %v", err) return Remote{}, fmt.Errorf("there was a problem reading the %s file from '%s': %w", MetaFileName, metaFilePath, err) } - logrus.Debugf("%s contents read:\n%s", MetaFileName, string(vdmMetaFile)) + message.Debugf("%s contents read:\n%s", MetaFileName, string(vdmMetaFile)) var vdmMeta Remote err = yaml.Unmarshal(vdmMetaFile, &vdmMeta) if err != nil { - logrus.Debugf("error during %s unmarshal: %v", MetaFileName, err) + message.Debugf("error during %s unmarshal: %v", MetaFileName, err) return Remote{}, fmt.Errorf("there was a problem reading the contents of the %s file at '%s': %v", MetaFileName, metaFilePath, err) } - logrus.Debugf("file %s unmarshalled: %+v", MetaFileName, vdmMeta) + message.Debugf("file %s unmarshalled: %+v", MetaFileName, vdmMeta) return vdmMeta, nil } @@ -88,7 +88,7 @@ func (r Remote) GetVDMMeta() (Remote, error) { func GetSpecFromFile(specFilePath string) (Spec, error) { specFile, err := os.ReadFile(specFilePath) if err != nil { - logrus.Debugf("error reading specfile from disk: %v", err) + message.Debugf("error reading specfile from disk: %v", err) return Spec{}, fmt.Errorf( strings.Join([]string{ "there was a problem reading your vdm file from '%s' -- does it not exist?", @@ -100,15 +100,15 @@ func GetSpecFromFile(specFilePath string) (Spec, error) { err, ) } - logrus.Debugf("specfile contents read:\n%s", string(specFile)) + message.Debugf("specfile contents read:\n%s", string(specFile)) var spec Spec err = yaml.Unmarshal(specFile, &spec) if err != nil { - logrus.Debugf("error during specfile unmarshal: %v", err) + message.Debugf("error during specfile unmarshal: %v", err) return Spec{}, fmt.Errorf("there was a problem reading the contents of your vdm spec file: %w", err) } - logrus.Debugf("vdmSpecs unmarshalled: %+v", spec) + message.Debugf("vdmSpecs unmarshalled: %+v", spec) return spec, nil } diff --git a/internal/vdmspec/validate.go b/internal/vdmspec/validate.go index c9d12f2..ba3eb6a 100644 --- a/internal/vdmspec/validate.go +++ b/internal/vdmspec/validate.go @@ -5,7 +5,7 @@ import ( "fmt" "regexp" - "github.com/sirupsen/logrus" + "github.com/opensourcecorp/vdm/internal/message" ) func (spec Spec) Validate() error { @@ -13,7 +13,7 @@ func (spec Spec) Validate() error { for remoteIndex, remote := range spec.Remotes { // Remote field - logrus.Debugf("Index %d: validating field 'Remote' for %+v", remoteIndex, remote) + message.Debugf("Index %d: validating field 'Remote' for %+v", remoteIndex, remote) if len(remote.Remote) == 0 { allErrors = append(allErrors, errors.New("all 'remote' fields must be non-zero length")) } @@ -26,22 +26,22 @@ func (spec Spec) Validate() error { } // Version field - logrus.Debugf("Index %d: validating field 'Version' for %+v", remoteIndex, remote) + message.Debugf("Index %d: validating field 'Version' for %+v", remoteIndex, remote) if remote.Type == "git" && len(remote.Version) == 0 { allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length. If you don't care about the version (even though you probably should), then use 'latest'")) } if remote.Type == "file" && len(remote.Version) > 0 { - logrus.Warnf("NOTE: Remote #%d '%s' specified as type '%s', which does not take explicit version info (you provided '%s'); ignoring version field", remoteIndex, remote.Remote, remote.Type, remote.Version) + message.Warnf("NOTE: Remote #%d '%s' specified as type '%s', which does not take explicit version info (you provided '%s'); ignoring version field", remoteIndex, remote.Remote, remote.Type, remote.Version) } // LocalPath field - logrus.Debugf("Index #%d: validating field 'LocalPath' for %+v", remoteIndex, remote) + message.Debugf("Index #%d: validating field 'LocalPath' for %+v", remoteIndex, remote) if len(remote.LocalPath) == 0 { allErrors = append(allErrors, errors.New("all 'local_path' fields must be non-zero length")) } // Type field - logrus.Debugf("Index #%d: validating field 'Type' for %+v", remoteIndex, remote) + message.Debugf("Index #%d: validating field 'Type' for %+v", remoteIndex, remote) typeMap := map[string]int{ "git": 1, "": 2, // also git @@ -54,7 +54,7 @@ func (spec Spec) Validate() error { if len(allErrors) > 0 { for _, err := range allErrors { - logrus.Errorf("validation failure: %s", err.Error()) + message.Errorf("validation failure: %s", err.Error()) } return fmt.Errorf("%d validation failure(s) found in your vdm spec file", len(allErrors)) } diff --git a/main.go b/main.go index 1fa94f7..1b1b1ae 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,11 @@ package main import ( "github.com/opensourcecorp/vdm/cmd" - "github.com/sirupsen/logrus" + "github.com/opensourcecorp/vdm/internal/message" ) func main() { if err := cmd.Execute(); err != nil { - logrus.Fatalf("running vdm: %v", err) + message.Fatalf("running vdm: %v", err) } } From e6b2e7cc329cffa09a3bef06d7d070b17a52c988 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Thu, 25 Jul 2024 23:30:45 -0500 Subject: [PATCH 14/29] Swap out staticcheck for revive because it catches missing docstrings --- cmd/sync.go | 8 ++++++-- internal/vdmspec/spec.go | 1 + main.go | 2 ++ scripts/ci.sh | 8 +++++++- staticcheck.conf | 3 --- 5 files changed, 16 insertions(+), 6 deletions(-) delete mode 100644 staticcheck.conf diff --git a/cmd/sync.go b/cmd/sync.go index edbedc7..f0c9afa 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -59,9 +59,13 @@ SpecLoop: switch remote.Type { case "git", "": - remotes.SyncGit(remote) + if err := remotes.SyncGit(remote); err != nil { + return fmt.Errorf("syncing git remote: %w", err) + } case "file": - remotes.SyncFile(remote) + if err := remotes.SyncFile(remote); err != nil { + return fmt.Errorf("syncing file remote: %w", err) + } default: return fmt.Errorf("unrecognized remote type '%s'", remote.Type) } diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index ffe3842..b0d9732 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -11,6 +11,7 @@ import ( "gopkg.in/yaml.v3" ) +// Spec defines the type Spec struct { Remotes []Remote `json:"remotes" yaml:"remotes"` } diff --git a/main.go b/main.go index 1b1b1ae..697c62c 100644 --- a/main.go +++ b/main.go @@ -10,3 +10,5 @@ func main() { message.Fatalf("running vdm: %v", err) } } + +type ABC struct{} diff --git a/scripts/ci.sh b/scripts/ci.sh index 4efbba6..44926e5 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -12,11 +12,17 @@ if ! go vet ./... ; then fi printf '>> Go linter\n' -if ! staticcheck ./... ; then +if ! go run github.com/mgechev/revive@latest --set_exit_status ./... ; then printf '>>> Failed go-lint\n' failures+=('go-lint') fi +printf '>> Go error checker\n' +if ! go run github.com/kisielk/errcheck@latest ./... ; then + printf '>>> Failed go-error-check\n' + failures+=('go-error-check') +fi + printf '>> Go test\n' if ! go test -cover -coverprofile=./cover.out ./... ; then printf '>>> Failed go-test check\n' diff --git a/staticcheck.conf b/staticcheck.conf deleted file mode 100644 index e30d13f..0000000 --- a/staticcheck.conf +++ /dev/null @@ -1,3 +0,0 @@ -checks = [ - "all", -] From 90726eae450341f30ac232e14d59816df1c26ead Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 13:52:16 -0500 Subject: [PATCH 15/29] Fix linter failures, add GHA workflow --- .github/workflows/ci.yaml | 19 +++++++++++++++++++ Containerfile | 14 ++++++++++++++ Makefile | 8 +++++++- cmd/root.go | 12 ++++++++---- cmd/sync.go | 10 ++++------ internal/message/message.go | 10 ++++++++++ internal/vdmspec/doc.go | 3 ++- internal/vdmspec/spec.go | 22 ++++++++++++++++++---- internal/vdmspec/validate.go | 2 ++ main.go | 3 +-- 10 files changed, 85 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 Containerfile diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..5e285fc --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,19 @@ +name: CI + +on: + push: + branches: + - main + pull_request: {} + +runs-on: ubuntu-latest + +jobs: + ci: + - uses: actions/checkout@v5 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.19' + - name: Run CI checks + run: 'make ci' diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..20d7d37 --- /dev/null +++ b/Containerfile @@ -0,0 +1,14 @@ +# This Containerfile isn't mean for creating artifacts etc., it's just a way to +# perform portable, local CI checks in case there are workstation-specific +# issues a developer faces. +FROM docker.io/library/golang:1.19 + +RUN apt-get update && apt-get install -y \ + ca-certificates \ + make \ + sudo + +COPY . /go/app +WORKDIR /go/app + +RUN make ci diff --git a/Makefile b/Makefile index 5a9ec39..4214681 100644 --- a/Makefile +++ b/Makefile @@ -5,12 +5,18 @@ BINNAME := vdm # DO NOT TOUCH -- use `make bump-version oldversion=X newversion=Y`! VERSION := 0.4.0 -all: test package package-debian +all: ci package package-debian .PHONY: ci ci: clean @bash ./scripts/ci.sh +# test is just an alias for ci +test: ci + +ci-container: + @docker build -f ./Containerfile -t vdm-test:latest . + .PHONY: test-coverage test-coverage: test go tool cover -html=./cover.out -o cover.html diff --git a/cmd/root.go b/cmd/root.go index a0172e3..2232222 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,23 +13,27 @@ var rootCmd = cobra.Command{ TraverseChildren: true, } -type RootFlags struct { +type rootFlags struct { SpecFilePath string Debug bool } -var RootFlagValues RootFlags +// RootFlagValues contains an initalized [rootFlags] struct with populated +// values. +var RootFlagValues rootFlags func init() { rootCmd.PersistentFlags().StringVar(&RootFlagValues.SpecFilePath, "specfile-path", "./vdm.yaml", "Path to vdm specfile") - rootCmd.PersistentFlags().BoolVar(&RootFlagValues.Debug, "debug", false, "Show debug logs") + rootCmd.PersistentFlags().BoolVar(&RootFlagValues.Debug, "debug", false, "Show debug messages during runtime") rootCmd.AddCommand(syncCmd) } +// Execute wraps the primary execution logic for vdm's root command, and returns +// any errors encountered to the caller. func Execute() error { if err := rootCmd.Execute(); err != nil { - return fmt.Errorf("executing root command: %v", err) + return fmt.Errorf("executing root command: %w", err) } return nil diff --git a/cmd/sync.go b/cmd/sync.go index f0c9afa..98973dc 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -2,7 +2,6 @@ package cmd import ( "fmt" - "path/filepath" "github.com/opensourcecorp/vdm/internal/message" "github.com/opensourcecorp/vdm/internal/remotes" @@ -46,15 +45,14 @@ SpecLoop: } if vdmMeta == (vdmspec.Remote{}) { - message.Infof("%s not found at local path '%s' -- will be created", vdmspec.MetaFileName, filepath.Join(remote.LocalPath)) + message.Infof("%s: %s not found at local path, will be created", remote.OpMsg(), vdmspec.MetaFileName) } else { if vdmMeta.Version != remote.Version && vdmMeta.Remote != remote.Remote { - message.Infof("Will change '%s' from current local version spec '%s' to '%s'...", remote.Remote, vdmMeta.Version, remote.Version) + message.Infof("%s: Will change '%s' from current local version spec '%s' to '%s'...", remote.OpMsg(), remote.Remote, vdmMeta.Version, remote.Version) panic("not implemented") - } else { - message.Infof("Version unchanged (%s) in spec file for '%s' --> '%s', skipping", remote.Version, remote.Remote, remote.LocalPath) - continue SpecLoop } + message.Infof("%s: version unchanged in spec file, skipping", remote.OpMsg()) + continue SpecLoop } switch remote.Type { diff --git a/internal/message/message.go b/internal/message/message.go index 1447f5f..f0f4f9c 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -7,24 +7,34 @@ import ( "os" ) +// Debugf prints out debug-level information messages with a formatting +// directive. func Debugf(format string, args ...any) { if os.Getenv("DEBUG") != "" { fmt.Printf("DEBUG: "+format+"\n", args...) } } +// Infof prints out debug-level information messages with a formatting +// directive. func Infof(format string, args ...any) { fmt.Printf(format+"\n", args...) } +// Warnf prints out debug-level information messages with a formatting +// directive. func Warnf(format string, args ...any) { fmt.Printf("WARNING: "+format+"\n", args...) } +// Errorf prints out debug-level information messages with a formatting +// directive. func Errorf(format string, args ...any) { fmt.Printf("ERROR: "+format+"\n", args...) } +// Fatalf prints out debug-level information messages with a formatting +// directive, and then exits with code 1. func Fatalf(format string, args ...any) { fmt.Printf("ERROR: "+format+"\n", args...) os.Exit(1) diff --git a/internal/vdmspec/doc.go b/internal/vdmspec/doc.go index 051c4bc..0f9316c 100644 --- a/internal/vdmspec/doc.go +++ b/internal/vdmspec/doc.go @@ -1,4 +1,5 @@ /* -Package vdmspec defines the [Spec] struct type, and its associated methods. +Package vdmspec defines the [Spec] and [Remote] struct types, and their +associated methods. */ package vdmspec diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index b0d9732..be813ce 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -11,11 +11,13 @@ import ( "gopkg.in/yaml.v3" ) -// Spec defines the +// Spec defines the overall structure of the vmd specfile. type Spec struct { Remotes []Remote `json:"remotes" yaml:"remotes"` } +// Remote defines the structure of each remote configuration in the vdm +// specfile. type Remote struct { Type string `json:"type,omitempty" yaml:"type,omitempty"` Remote string `json:"remote" yaml:"remote"` @@ -24,10 +26,16 @@ type Remote struct { } const ( + // MetaFileName is the name of the tracking file that vdm uses to record & + // track remote statuses on disk. MetaFileName = "VDMMETA" - typeFile = "file" + + // Remote types + typeFile = "file" ) +// MakeMetaFilePath constructs the metafile path that vdm will use to track a +// remote's state on disk. func (r Remote) MakeMetaFilePath() string { metaFilePath := filepath.Join(r.LocalPath, MetaFileName) // TODO: this is brittle, but it's the best I can think of right now @@ -41,6 +49,8 @@ func (r Remote) MakeMetaFilePath() string { return metaFilePath } +// WriteVDMMeta writes the metafile contents to disk, the path of which is +// determined by [Remote.MakeMetaFilePath]. func (r Remote) WriteVDMMeta() error { metaFilePath := r.MakeMetaFilePath() vdmMetaContent, err := yaml.Marshal(r) @@ -59,6 +69,8 @@ func (r Remote) WriteVDMMeta() error { return nil } +// GetVDMMeta reads the metafile from disk, and returns it for further +// processing. func (r Remote) GetVDMMeta() (Remote, error) { metaFilePath := r.MakeMetaFilePath() _, err := os.Stat(metaFilePath) @@ -86,6 +98,9 @@ func (r Remote) GetVDMMeta() (Remote, error) { return vdmMeta, nil } +// GetSpecFromFile reads the specfile from disk (the path of which is determined +// by the user-supplied flag value), and returns it for further processing of +// remotes. func GetSpecFromFile(specFilePath string) (Spec, error) { specFile, err := os.ReadFile(specFilePath) if err != nil { @@ -119,7 +134,6 @@ func GetSpecFromFile(specFilePath string) (Spec, error) { func (r Remote) OpMsg() string { if r.Version != "" { return fmt.Sprintf("%s@%s --> %s", r.Remote, r.Version, r.LocalPath) - } else { - return fmt.Sprintf("%s --> %s", r.Remote, r.LocalPath) } + return fmt.Sprintf("%s --> %s", r.Remote, r.LocalPath) } diff --git a/internal/vdmspec/validate.go b/internal/vdmspec/validate.go index ba3eb6a..fc381b8 100644 --- a/internal/vdmspec/validate.go +++ b/internal/vdmspec/validate.go @@ -8,6 +8,8 @@ import ( "github.com/opensourcecorp/vdm/internal/message" ) +// Validate performs runtime validations on the vdm specfile, and informs the +// caller of any failures encountered. func (spec Spec) Validate() error { var allErrors []error diff --git a/main.go b/main.go index 697c62c..7076e96 100644 --- a/main.go +++ b/main.go @@ -1,3 +1,4 @@ +// Package main provides the entrypoint into vdm's subcommands. package main import ( @@ -10,5 +11,3 @@ func main() { message.Fatalf("running vdm: %v", err) } } - -type ABC struct{} From c28d9772ddb8e7b6b78dc9dcd84df1e0697f7d14 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 13:58:05 -0500 Subject: [PATCH 16/29] ugh GHA --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5e285fc..92ee191 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,9 +2,9 @@ name: CI on: push: - branches: - - main - pull_request: {} + branches: [main] + pull_request: + types: [opened, reopened, synchronize] runs-on: ubuntu-latest From 2fc88b1da6d3b2228364c9bb010ef562646e478d Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 14:00:53 -0500 Subject: [PATCH 17/29] Use real checkout version --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 92ee191..540b6fd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,7 +10,7 @@ runs-on: ubuntu-latest jobs: ci: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v5 with: From 0c5b72c81e8f3bd07bf00d705c1473bb633c1344 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 14:01:42 -0500 Subject: [PATCH 18/29] Oops structure --- .github/workflows/ci.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 540b6fd..bcc94d9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,10 +10,11 @@ runs-on: ubuntu-latest jobs: ci: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.19' - - name: Run CI checks - run: 'make ci' + steps: + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.19' + - name: Run CI checks + run: 'make ci' From 0ce0284532549c1987b02afad968b7d5f1589239 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 14:02:14 -0500 Subject: [PATCH 19/29] Oops structure again --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bcc94d9..e5ebfa0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,10 +6,9 @@ on: pull_request: types: [opened, reopened, synchronize] -runs-on: ubuntu-latest - jobs: ci: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Go From a144ec27e1385921b17e47de9cbaa12927b02b8f Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 14:24:09 -0500 Subject: [PATCH 20/29] Add packaging check to CI suite --- Makefile | 5 ++--- scripts/ci.sh | 6 ++++++ scripts/package.sh | 2 +- scripts/xbuild.sh | 11 +++++++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4214681..42889b8 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,10 @@ BINNAME := vdm # DO NOT TOUCH -- use `make bump-version oldversion=X newversion=Y`! VERSION := 0.4.0 +.PHONY: % + all: ci package package-debian -.PHONY: ci ci: clean @bash ./scripts/ci.sh @@ -17,7 +18,6 @@ test: ci ci-container: @docker build -f ./Containerfile -t vdm-test:latest . -.PHONY: test-coverage test-coverage: test go tool cover -html=./cover.out -o cover.html xdg-open ./cover.html @@ -35,7 +35,6 @@ package: xbuild package-debian: build @bash ./scripts/package-debian.sh -.PHONY: clean clean: @rm -rf \ /tmp/$(BINNAME)-tests \ diff --git a/scripts/ci.sh b/scripts/ci.sh index 44926e5..46107af 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -29,6 +29,12 @@ if ! go test -cover -coverprofile=./cover.out ./... ; then failures+=('go-test') fi +printf '>> Packaging checker\n' +if ! make -s package ; then + printf '>>> Failed packaging check\n' + failures+=('packaging') +fi + if [[ "${#failures[@]}" -gt 0 ]] ; then printf '> One or more checks failed, see output above\n' exit 1 diff --git a/scripts/package.sh b/scripts/package.sh index dfc3274..6c29a6a 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -5,7 +5,7 @@ set -euo pipefail mkdir -p dist cd build || exit 1 for built in * ; do - printf 'Packaging for %s into dist/ ...\n' "${built}" + printf 'Packaging for %s into dist/\n' "${built}" cd "${built}" && tar -czf ../../dist/vdm_"${built}".tar.gz ./* cd - > /dev/null done diff --git a/scripts/xbuild.sh b/scripts/xbuild.sh index 1cbd5fc..c48c5f2 100755 --- a/scripts/xbuild.sh +++ b/scripts/xbuild.sh @@ -11,8 +11,15 @@ for target in ${targets} ; do GOOS=$(echo "${target}" | cut -d'/' -f1) GOARCH=$(echo "${target}" | cut -d'/' -f2) export GOOS GOARCH + + # Windows needs an .exe suffix to actually run + suffix='' + if [[ "${GOOS}" == 'windows' ]] ; then + suffix='.exe' + fi + outdir=build/"${GOOS}-${GOARCH}" mkdir -p "${outdir}" - printf "Building for %s-%s into build/ ...\n" "${GOOS}" "${GOARCH}" - go build -buildmode=pie -o "${outdir}"/vdm -ldflags "-s -w" + printf "Building for %s-%s into build/\n" "${GOOS}" "${GOARCH}" + go build -buildmode=pie -o "${outdir}/vdm${suffix}" -ldflags "-s -w" done From fa1f679c4843b07ff432aadeb0636763136c6fa7 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 14:58:52 -0500 Subject: [PATCH 21/29] Add version specifier tooling, fix CI packaging check failure --- Makefile | 11 ++--------- README.md | 2 +- VERSION | 3 +++ dist/debian/vdm/DEBIAN/control | 2 +- dist/man/man.1.md | 2 +- internal/vdmspec/spec.go | 2 +- scripts/bump-versions.sh | 33 +++++++++++++++++++++++++++++++++ scripts/xbuild.sh | 2 +- 8 files changed, 43 insertions(+), 14 deletions(-) create mode 100644 VERSION create mode 100644 scripts/bump-versions.sh diff --git a/Makefile b/Makefile index 42889b8..c34b1d5 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,6 @@ SHELL = /usr/bin/env bash -euo pipefail BINNAME := vdm -# DO NOT TOUCH -- use `make bump-version oldversion=X newversion=Y`! -VERSION := 0.4.0 - .PHONY: % all: ci package package-debian @@ -50,12 +47,8 @@ clean: @find . -type d -name '*deps*' -exec rm -rf {} + @find . -type f -name '*VDMMETA*' -delete -bump-version: clean - @if [[ -z "$(oldversion)" ]] || [[ -z "$(newversion)" ]] ; then printf 'ERROR: "oldversion" and "newversion" must be provided\n' && exit 1 ; fi - find . \ - -type f \ - -not -path './go.*' \ - -exec sed -i 's/$(oldversion)/$(newversion)/g' {} \; +bump-versions: clean + @bash ./scripts/bump-versions.sh "$${old_version:-}" pre-commit-hook: cp ./scripts/ci.sh ./.git/hooks/pre-commit diff --git a/README.md b/README.md index cee3c09..b93e98b 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ remotes: You can have as many dependency specifications in that array as you want. By default, this spec file is called `vdm.yaml` and lives at the calling location (which is probably your repo's root), but you can call it whatever you want and -point to it using the `-spec-file` flag to `vdm`. +point to it using the `--spec-file` flag to `vdm`. Once you have a spec file, just run: diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..b9bbca9 --- /dev/null +++ b/VERSION @@ -0,0 +1,3 @@ +# Version number used as an anchor for listed versions in various files in the +# tree, orchestrated by scripts/bump-versions.sh +0.2.0 diff --git a/dist/debian/vdm/DEBIAN/control b/dist/debian/vdm/DEBIAN/control index 33150b3..f57724c 100644 --- a/dist/debian/vdm/DEBIAN/control +++ b/dist/debian/vdm/DEBIAN/control @@ -1,5 +1,5 @@ Package: vdm -Version: 0.4.0 +Version: 0.2.0 Architecture: amd64 Maintainer: Ryan Price Description: Versioned Dependency Manager diff --git a/dist/man/man.1.md b/dist/man/man.1.md index 71498ec..55f177d 100644 --- a/dist/man/man.1.md +++ b/dist/man/man.1.md @@ -1,4 +1,4 @@ -% VDM(1) vdm 0.4.0 +% VDM(1) vdm 0.2.0 % Ryan J. Price % 2023 diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index be813ce..ac35c7f 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -108,7 +108,7 @@ func GetSpecFromFile(specFilePath string) (Spec, error) { return Spec{}, fmt.Errorf( strings.Join([]string{ "there was a problem reading your vdm file from '%s' -- does it not exist?", - "Either pass the -spec-file flag, or create one in the default location (details in the README).", + "Either pass the --spec-file flag, or create one in the default location (details in the README).", "Error details: %w"}, " ", ), diff --git a/scripts/bump-versions.sh b/scripts/bump-versions.sh new file mode 100644 index 0000000..5fb5b72 --- /dev/null +++ b/scripts/bump-versions.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +################################################################################ +# A singular, consistent version needs to match across several places across the +# codebase tree, such as the Debian Control file & man pages.This script is used +# to bump version identifiers in any locations listed in the loop at the bottom. +################################################################################ + +root="$(git rev-parse --show-toplevel)" + +old_version="${1:-}" +new_version="$(grep -v '#' "${root}"/VERSION)" + +if [[ -z "${old_version:-}" ]] ; then + printf 'ERROR: you must specify old_version as the first script argument\n' + exit 1 +fi + +if [[ -z "${new_version:-}" ]] ; then + printf 'ERROR: version unable to be determined from ./VERSION file; possible malformed?\n' + exit 1 +fi + +for f in \ + dist/debian/vdm/DEBIAN/control \ + dist/man/* \ +; do + if grep -q "${old_version}" "${root}/${f}" ; then + printf 'Updating version in %s: %s -> %s\n' "${f}" "${old_version}" "${new_version}" + sed -i "s/${old_version}/${new_version}/g" "${root}/${f}" + fi +done diff --git a/scripts/xbuild.sh b/scripts/xbuild.sh index c48c5f2..e5606b7 100755 --- a/scripts/xbuild.sh +++ b/scripts/xbuild.sh @@ -21,5 +21,5 @@ for target in ${targets} ; do outdir=build/"${GOOS}-${GOARCH}" mkdir -p "${outdir}" printf "Building for %s-%s into build/\n" "${GOOS}" "${GOARCH}" - go build -buildmode=pie -o "${outdir}/vdm${suffix}" -ldflags "-s -w" + go build -o "${outdir}/vdm${suffix}" done From a918b22b6016b144deeefd2c72ad8a3182e99d08 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 15:43:15 -0500 Subject: [PATCH 22/29] Realized debug flag setting wasn't working so fixed it, and some other things --- cmd/flagsupport.go | 19 +++++++++++ cmd/root.go | 26 ++++++++++++-- cmd/sync.go | 3 +- cmd/sync_test.go | 12 +++---- go.mod | 24 +++++++++++-- go.sum | 56 ++++++++++++++++++++++++++++--- internal/remotes/file.go | 6 ++-- internal/remotes/git.go | 8 ++--- internal/remotes/git_test.go | 2 +- internal/vdmspec/spec.go | 10 +++--- internal/vdmspec/spec_test.go | 12 +++---- internal/vdmspec/validate.go | 4 +-- internal/vdmspec/validate_test.go | 3 +- 13 files changed, 146 insertions(+), 39 deletions(-) create mode 100644 cmd/flagsupport.go diff --git a/cmd/flagsupport.go b/cmd/flagsupport.go new file mode 100644 index 0000000..31a122e --- /dev/null +++ b/cmd/flagsupport.go @@ -0,0 +1,19 @@ +package cmd + +import ( + "os" + + "github.com/opensourcecorp/vdm/internal/message" + "github.com/spf13/viper" +) + +// MaybeSetDebug sets the DEBUG environment variable if it was set as a flag by +// the caller. +func MaybeSetDebug() { + if viper.GetBool(debugFlagKey) { + err := os.Setenv("DEBUG", "true") + if err != nil { + message.Fatalf("internal error: unable to set environment variable DEBUG") + } + } +} diff --git a/cmd/root.go b/cmd/root.go index 2232222..fa196ba 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,7 +3,9 @@ package cmd import ( "fmt" + "github.com/opensourcecorp/vdm/internal/message" "github.com/spf13/cobra" + "github.com/spf13/viper" ) var rootCmd = cobra.Command{ @@ -11,6 +13,9 @@ var rootCmd = cobra.Command{ Short: "vdm -- a Versioned-Dependency Manager", Long: "vdm is used to manage arbitrary remote dependencies", TraverseChildren: true, + Run: func(_ *cobra.Command, _ []string) { + MaybeSetDebug() + }, } type rootFlags struct { @@ -22,9 +27,26 @@ type rootFlags struct { // values. var RootFlagValues rootFlags +// Flag name keys +const ( + specFilePathFlagKey string = "specfile-path" + debugFlagKey string = "debug" +) + func init() { - rootCmd.PersistentFlags().StringVar(&RootFlagValues.SpecFilePath, "specfile-path", "./vdm.yaml", "Path to vdm specfile") - rootCmd.PersistentFlags().BoolVar(&RootFlagValues.Debug, "debug", false, "Show debug messages during runtime") + var err error + + rootCmd.PersistentFlags().StringVar(&RootFlagValues.SpecFilePath, specFilePathFlagKey, "./vdm.yaml", "Path to vdm specfile") + err = viper.BindPFlag(specFilePathFlagKey, rootCmd.PersistentFlags().Lookup(specFilePathFlagKey)) + if err != nil { + message.Fatalf("internal error: unable to bind state of flag --%s", specFilePathFlagKey) + } + + rootCmd.PersistentFlags().BoolVar(&RootFlagValues.Debug, debugFlagKey, false, "Show debug messages during runtime") + err = viper.BindPFlag(debugFlagKey, rootCmd.PersistentFlags().Lookup(debugFlagKey)) + if err != nil { + message.Fatalf("internal error: unable to bind state of flag --%s", debugFlagKey) + } rootCmd.AddCommand(syncCmd) } diff --git a/cmd/sync.go b/cmd/sync.go index 98973dc..98ca357 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -16,6 +16,7 @@ var syncCmd = &cobra.Command{ } func syncExecute(_ *cobra.Command, _ []string) error { + MaybeSetDebug() if err := sync(); err != nil { return fmt.Errorf("executing sync command: %w", err) } @@ -73,7 +74,7 @@ SpecLoop: return fmt.Errorf("could not write %s file to disk: %w", vdmspec.MetaFileName, err) } - message.Infof("%s -- Done.", remote.OpMsg()) + message.Infof("%s: Done.", remote.OpMsg()) } message.Infof("All done!") diff --git a/cmd/sync_test.go b/cmd/sync_test.go index 14b4267..a7587ac 100644 --- a/cmd/sync_test.go +++ b/cmd/sync_test.go @@ -28,25 +28,25 @@ func TestSync(t *testing.T) { t.Run("SyncGit", func(t *testing.T) { t.Run("spec[0] used a tag", func(t *testing.T) { vdmMeta, err := spec.Remotes[0].GetVDMMeta() - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "v0.2.0", vdmMeta.Version) }) t.Run("spec[1] used 'latest'", func(t *testing.T) { vdmMeta, err := spec.Remotes[1].GetVDMMeta() - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "latest", vdmMeta.Version) }) t.Run("spec[2] used a branch", func(t *testing.T) { vdmMeta, err := spec.Remotes[2].GetVDMMeta() - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "main", vdmMeta.Version) }) t.Run("spec[3] used a hash", func(t *testing.T) { vdmMeta, err := spec.Remotes[3].GetVDMMeta() - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "2e6657f5ac013296167c4dd92fbb46f0e3dbdc5f", vdmMeta.Version) }) }) @@ -54,7 +54,7 @@ func TestSync(t *testing.T) { t.Run("SyncFile", func(t *testing.T) { t.Run("spec[4] had an implcit version", func(t *testing.T) { vdmMeta, err := spec.Remotes[4].GetVDMMeta() - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "", vdmMeta.Version) }) }) @@ -62,7 +62,7 @@ func TestSync(t *testing.T) { t.Cleanup(func() { for _, remote := range spec.Remotes { err := os.RemoveAll(remote.LocalPath) - assert.NoError(t, err) + require.NoError(t, err) } }) } diff --git a/go.mod b/go.mod index 01db9f7..c1cc5fd 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,31 @@ go 1.19 require ( github.com/spf13/cobra v1.7.0 - github.com/stretchr/testify v1.8.2 + github.com/spf13/viper v1.19.0 + github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 0df3271..365ddea 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,71 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/remotes/file.go b/internal/remotes/file.go index 8041961..e51715f 100644 --- a/internal/remotes/file.go +++ b/internal/remotes/file.go @@ -48,14 +48,14 @@ func checkFileExists(remote vdmspec.Remote) (bool, error) { return true, nil } -func retrieveFile(remote vdmspec.Remote) error { +func retrieveFile(remote vdmspec.Remote) (err error) { resp, err := http.Get(remote.Remote) if err != nil { return fmt.Errorf("retrieving remote file '%s': %w", remote.Remote, err) } defer func() { if closeErr := resp.Body.Close(); closeErr != nil { - message.Errorf("closing response body after remote file '%s' retrieval: %v", remote.Remote, err) + err = errors.Join(fmt.Errorf("closing response body after remote file '%s' retrieval: %w", remote.Remote, err)) } }() @@ -72,7 +72,7 @@ func retrieveFile(remote vdmspec.Remote) error { } defer func() { if closeErr := outFile.Close(); closeErr != nil { - message.Errorf("closing local file '%s' after remote file '%s' retrieval: %v", remote.LocalPath, remote.Remote, err) + err = errors.Join(fmt.Errorf("closing local file '%s' after remote file '%s' retrieval: %w", remote.LocalPath, remote.Remote, err)) } }() diff --git a/internal/remotes/git.go b/internal/remotes/git.go index a6f1f53..940a25c 100644 --- a/internal/remotes/git.go +++ b/internal/remotes/git.go @@ -19,7 +19,7 @@ func SyncGit(remote vdmspec.Remote) error { } if remote.Version != "latest" { - message.Infof("%s -- Setting specified version...", remote.OpMsg()) + message.Infof("%s: Setting specified version...", remote.OpMsg()) checkoutCmd := exec.Command("git", "-C", remote.LocalPath, "checkout", remote.Version) checkoutOutput, err := checkoutCmd.CombinedOutput() if err != nil { @@ -59,14 +59,14 @@ func gitClone(remote vdmspec.Remote) error { // full history to be able to find a specified revision var cloneCmdArgs []string if remote.Version == "latest" { - message.Debugf("%s -- version specified as 'latest', so making shallow clone and skipping separate checkout operation", remote.OpMsg()) + message.Debugf("%s: version specified as 'latest', so making shallow clone and skipping separate checkout operation", remote.OpMsg()) cloneCmdArgs = []string{"clone", "--depth=1", remote.Remote, remote.LocalPath} } else { - message.Debugf("%s -- version specified as NOT latest, so making regular clone and will make separate checkout operation", remote.OpMsg()) + message.Debugf("%s: version specified as NOT latest, so making regular clone and will make separate checkout operation", remote.OpMsg()) cloneCmdArgs = []string{"clone", remote.Remote, remote.LocalPath} } - message.Infof("%s -- Retrieving...", remote.OpMsg()) + message.Infof("%s: Retrieving...", remote.OpMsg()) cloneCmd := exec.Command("git", cloneCmdArgs...) cloneOutput, err := cloneCmd.CombinedOutput() if err != nil { diff --git a/internal/remotes/git_test.go b/internal/remotes/git_test.go index 712b77f..63d0f7f 100644 --- a/internal/remotes/git_test.go +++ b/internal/remotes/git_test.go @@ -40,7 +40,7 @@ func TestCheckGitAvailable(t *testing.T) { t.Run("no error when git is available", func(t *testing.T) { // Host of this test better have git available lol gitAvailable := checkGitAvailable() - assert.NoError(t, gitAvailable) + require.NoError(t, gitAvailable) }) t.Run("error when git is NOT available", func(t *testing.T) { diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index ac35c7f..d1294cc 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -82,7 +82,7 @@ func (r Remote) GetVDMMeta() (Remote, error) { vdmMetaFile, err := os.ReadFile(metaFilePath) if err != nil { - message.Debugf("error reading VMDMMETA from disk: %v", err) + message.Debugf("error reading VMDMMETA from disk: %w", err) return Remote{}, fmt.Errorf("there was a problem reading the %s file from '%s': %w", MetaFileName, metaFilePath, err) } message.Debugf("%s contents read:\n%s", MetaFileName, string(vdmMetaFile)) @@ -90,8 +90,8 @@ func (r Remote) GetVDMMeta() (Remote, error) { var vdmMeta Remote err = yaml.Unmarshal(vdmMetaFile, &vdmMeta) if err != nil { - message.Debugf("error during %s unmarshal: %v", MetaFileName, err) - return Remote{}, fmt.Errorf("there was a problem reading the contents of the %s file at '%s': %v", MetaFileName, metaFilePath, err) + message.Debugf("error during %s unmarshal: w", MetaFileName, err) + return Remote{}, fmt.Errorf("there was a problem reading the contents of the %s file at '%s': %w", MetaFileName, metaFilePath, err) } message.Debugf("file %s unmarshalled: %+v", MetaFileName, vdmMeta) @@ -104,7 +104,7 @@ func (r Remote) GetVDMMeta() (Remote, error) { func GetSpecFromFile(specFilePath string) (Spec, error) { specFile, err := os.ReadFile(specFilePath) if err != nil { - message.Debugf("error reading specfile from disk: %v", err) + message.Debugf("error reading specfile from disk: %w", err) return Spec{}, fmt.Errorf( strings.Join([]string{ "there was a problem reading your vdm file from '%s' -- does it not exist?", @@ -121,7 +121,7 @@ func GetSpecFromFile(specFilePath string) (Spec, error) { var spec Spec err = yaml.Unmarshal(specFile, &spec) if err != nil { - message.Debugf("error during specfile unmarshal: %v", err) + message.Debugf("error during specfile unmarshal: w", err) return Spec{}, fmt.Errorf("there was a problem reading the contents of your vdm spec file: %w", err) } message.Debugf("vdmSpecs unmarshalled: %+v", spec) diff --git a/internal/vdmspec/spec_test.go b/internal/vdmspec/spec_test.go index 783baa4..1795c8a 100644 --- a/internal/vdmspec/spec_test.go +++ b/internal/vdmspec/spec_test.go @@ -35,12 +35,12 @@ func TestVDMMeta(t *testing.T) { require.NoError(t, err) got, err := testRemote.GetVDMMeta() - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, testRemote, got) t.Cleanup(func() { err := os.RemoveAll(testVDMMetaFilePath) - assert.NoError(t, err) + require.NoError(t, err) }) }) @@ -53,23 +53,23 @@ func TestVDMMeta(t *testing.T) { require.NoError(t, err) got, err := testRemote.GetVDMMeta() - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, testRemote, got) t.Cleanup(func() { err := os.RemoveAll(testVDMMetaFilePath) - assert.NoError(t, err) + require.NoError(t, err) }) }) t.Run("GetSpecsFromFile", func(t *testing.T) { spec, err := GetSpecFromFile(testSpecFilePath) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, 5, len(spec.Remotes)) t.Cleanup(func() { err := os.RemoveAll(testVDMMetaFilePath) - assert.NoError(t, err) + require.NoError(t, err) }) }) } diff --git a/internal/vdmspec/validate.go b/internal/vdmspec/validate.go index fc381b8..b9215e2 100644 --- a/internal/vdmspec/validate.go +++ b/internal/vdmspec/validate.go @@ -15,7 +15,7 @@ func (spec Spec) Validate() error { for remoteIndex, remote := range spec.Remotes { // Remote field - message.Debugf("Index %d: validating field 'Remote' for %+v", remoteIndex, remote) + message.Debugf("Index #%d: validating field 'Remote' for %+v", remoteIndex, remote) if len(remote.Remote) == 0 { allErrors = append(allErrors, errors.New("all 'remote' fields must be non-zero length")) } @@ -28,7 +28,7 @@ func (spec Spec) Validate() error { } // Version field - message.Debugf("Index %d: validating field 'Version' for %+v", remoteIndex, remote) + message.Debugf("Index #%d: validating field 'Version' for %+v", remoteIndex, remote) if remote.Type == "git" && len(remote.Version) == 0 { allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length. If you don't care about the version (even though you probably should), then use 'latest'")) } diff --git a/internal/vdmspec/validate_test.go b/internal/vdmspec/validate_test.go index ffdc497..f4de2d0 100644 --- a/internal/vdmspec/validate_test.go +++ b/internal/vdmspec/validate_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestValidate(t *testing.T) { @@ -16,7 +17,7 @@ func TestValidate(t *testing.T) { }}, } err := spec.Validate() - assert.NoError(t, err) + require.NoError(t, err) }) t.Run("fails on zero-length remote", func(t *testing.T) { From 36b91b5ede0cbf847efc675bc4f92f8fd2d09094 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 15:47:32 -0500 Subject: [PATCH 23/29] Adding errors.Join means I need to bump the Go version --- .github/workflows/ci.yaml | 2 +- Containerfile | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e5ebfa0..4548fe0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,6 +14,6 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.19' + go-version: '1.20' - name: Run CI checks run: 'make ci' diff --git a/Containerfile b/Containerfile index 20d7d37..135ad78 100644 --- a/Containerfile +++ b/Containerfile @@ -1,7 +1,7 @@ # This Containerfile isn't mean for creating artifacts etc., it's just a way to # perform portable, local CI checks in case there are workstation-specific # issues a developer faces. -FROM docker.io/library/golang:1.19 +FROM docker.io/library/golang:1.20 RUN apt-get update && apt-get install -y \ ca-certificates \ diff --git a/go.mod b/go.mod index c1cc5fd..b1b5ed1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/opensourcecorp/vdm -go 1.19 +go 1.20 require ( github.com/spf13/cobra v1.7.0 From 5352eab7babc5f737f199e6b2d9fe213694fba88 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 16:20:38 -0500 Subject: [PATCH 24/29] Add tagger script, other tweaks --- .github/workflows/ci.yaml | 10 +++++++- Makefile | 6 ++++- dist/.gitignore | 3 ++- scripts/bump-versions.sh | 0 scripts/ci.sh | 18 +++++++++----- scripts/package.sh | 8 ++++++- scripts/tag-release.sh | 49 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 84 insertions(+), 10 deletions(-) mode change 100644 => 100755 scripts/bump-versions.sh create mode 100755 scripts/tag-release.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4548fe0..ee8561f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: CI +name: main on: push: @@ -17,3 +17,11 @@ jobs: go-version: '1.20' - name: Run CI checks run: 'make ci' + tag: + if: github.ref == 'refs/heads/main' + needs: ci + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Tag for release + run: 'make tag-release' diff --git a/Makefile b/Makefile index c34b1d5..2e0be30 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,8 @@ clean: *cache* \ .*cache* \ ./build/ \ - ./dist/*.gz \ + ./dist/*.tar.gz \ + ./dist/*.zip \ ./dist/debian/vdm.deb \ *.out @sudo rm -rf ./dist/debian/vdm/usr @@ -50,6 +51,9 @@ clean: bump-versions: clean @bash ./scripts/bump-versions.sh "$${old_version:-}" +tag-release: clean + @bash ./scripts/tag-release.sh + pre-commit-hook: cp ./scripts/ci.sh ./.git/hooks/pre-commit diff --git a/dist/.gitignore b/dist/.gitignore index a6f5919..2ff6fd3 100644 --- a/dist/.gitignore +++ b/dist/.gitignore @@ -1,5 +1,6 @@ # Packaged binaries for e.g. GitHub -*.gz +*.tar.gz +*.zip # The possible locations of the app binary itself, and its debfile vdm diff --git a/scripts/bump-versions.sh b/scripts/bump-versions.sh old mode 100644 new mode 100755 diff --git a/scripts/ci.sh b/scripts/ci.sh index 46107af..9f3f582 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -7,36 +7,42 @@ printf '> Running CI checks\n' printf '>> Go vet\n' if ! go vet ./... ; then - printf '>>> Failed go-vet check\n' + printf '>>> Failed go-vet check\n' > /dev/stderr failures+=('go-vet') fi printf '>> Go linter\n' if ! go run github.com/mgechev/revive@latest --set_exit_status ./... ; then - printf '>>> Failed go-lint\n' + printf '>>> Failed go-lint\n' > /dev/stderr failures+=('go-lint') fi printf '>> Go error checker\n' if ! go run github.com/kisielk/errcheck@latest ./... ; then - printf '>>> Failed go-error-check\n' + printf '>>> Failed go-error-check\n' > /dev/stderr failures+=('go-error-check') fi printf '>> Go test\n' if ! go test -cover -coverprofile=./cover.out ./... ; then - printf '>>> Failed go-test check\n' + printf '>>> Failed go-test check\n' > /dev/stderr failures+=('go-test') fi printf '>> Packaging checker\n' if ! make -s package ; then - printf '>>> Failed packaging check\n' + printf '>>> Failed packaging check\n' > /dev/stderr failures+=('packaging') fi +printf '>> Version tag checker\n' +if ! make -s tag-release ; then + printf '>>> Failed verion tag checker\n' > /dev/stderr + failures+=('version-tag') +fi + if [[ "${#failures[@]}" -gt 0 ]] ; then - printf '> One or more checks failed, see output above\n' + printf '> One or more checks failed, see output above\n' > /dev/stderr exit 1 fi diff --git a/scripts/package.sh b/scripts/package.sh index 6c29a6a..bd4a468 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -6,6 +6,12 @@ mkdir -p dist cd build || exit 1 for built in * ; do printf 'Packaging for %s into dist/\n' "${built}" - cd "${built}" && tar -czf ../../dist/vdm_"${built}".tar.gz ./* + cd "${built}" || exit 1 + # Windows might like .zips better, otherwise make .tar.gzs + if [[ "${built}" =~ 'windows' ]] ; then + zip -r9 ../../dist/vdm_"${built}".zip ./* + else + tar -czf ../../dist/vdm_"${built}".tar.gz ./* + fi cd - > /dev/null done diff --git a/scripts/tag-release.sh b/scripts/tag-release.sh new file mode 100755 index 0000000..f03728c --- /dev/null +++ b/scripts/tag-release.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +################################################################################ +# This script portably manages tagging of the HEAD commit based on the contents +# of the VERSION file. It performs some heuristic checks to make sure tagging +# will behave as expected. +################################################################################ + +root="$(git rev-parse --show-toplevel)" + +current_git_branch="$(git rev-parse --abbrev-ref HEAD)" +latest_git_tag="$(git tag --list | tail -n1)" +current_listed_version="$(grep -v '#' "${root:-}"/VERSION)" + +printf 'Current git branch: %s\n' "${current_git_branch:-}" +printf 'Latest git tag: %s\n' "${latest_git_tag:-}" +printf 'Current version listed in VERSION file: %s\n' "${current_listed_version:-}" + +failures=() + +if [[ "${current_git_branch}" == 'main' ]] ; then + # Fail if we forgot to bump VERSION + if [[ "${latest_git_tag}" == "${current_listed_version}" ]] ; then + printf 'ERROR: Identifier in VERSION still matches what is tagged on the main branch -- did you forget to update?\n' > /dev/stderr + failures+=('forgot-to-bump-VERSION') + fi + + # Fail if we forgot to bump versions across files + old_git_status="$(git status | grep -i -E 'modified')" + make -s bump-versions old_version="${current_listed_version}" + new_git_status="$(git status | grep -i -E 'modified')" + if [[ "$(diff <(echo "${old_git_status}") <(echo "${new_git_status}") | wc -l)" -gt 0 ]] ; then + printf 'ERROR: Files modified by version-bump check -- did you forget to update versions across the repo to match VERSION?\n' > /dev/stderr + failures+=('forgot-to-bump-other-versions') + fi + + if [[ "${#failures[@]}" -gt 0 ]] ; then + exit 1 + else + printf 'All checks passed, tagging & pushing new version: %s --> %s\n' "${latest_git_tag}" "${current_listed_version}" + git tag --force "v${current_listed_version}" + git push --tags + fi + +else + printf 'Not on main branch, nothing to do\n' + exit 0 +fi From 5ca9d00d866d19afbe7a807427464f3a915198c9 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 16:30:21 -0500 Subject: [PATCH 25/29] Update packaging, and add GHA for cutting a release --- .github/workflows/ci.yaml | 16 ++++++++++++++++ Makefile | 4 ++-- scripts/package.sh | 8 ++++---- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ee8561f..696bb73 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,3 +25,19 @@ jobs: - uses: actions/checkout@v4 - name: Tag for release run: 'make tag-release' + release: + if: github.ref == 'refs/heads/main' + needs: tag + run-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.20' + - name: Package binaries + run: 'make package' + - name: Create GitHub Release + run: gh release create "v$(grep -v '#' ./VERSION)" --generate-notes --latest ./dist/zipped/* + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Makefile b/Makefile index 2e0be30..37cc469 100644 --- a/Makefile +++ b/Makefile @@ -38,8 +38,8 @@ clean: *cache* \ .*cache* \ ./build/ \ - ./dist/*.tar.gz \ - ./dist/*.zip \ + ./dist/zipped/*.tar.gz \ + ./dist/zipped/*.zip \ ./dist/debian/vdm.deb \ *.out @sudo rm -rf ./dist/debian/vdm/usr diff --git a/scripts/package.sh b/scripts/package.sh index bd4a468..13daeda 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -2,16 +2,16 @@ set -euo pipefail # cd-jumps because it makes logs cleaner, not sorry -mkdir -p dist +mkdir -p dist/zipped cd build || exit 1 for built in * ; do - printf 'Packaging for %s into dist/\n' "${built}" + printf 'Packaging for %s into dist/zipped/\n' "${built}" cd "${built}" || exit 1 # Windows might like .zips better, otherwise make .tar.gzs if [[ "${built}" =~ 'windows' ]] ; then - zip -r9 ../../dist/vdm_"${built}".zip ./* + zip -r9 ../../dist/zipped/vdm_"${built}".zip ./* else - tar -czf ../../dist/vdm_"${built}".tar.gz ./* + tar -czf ../../dist/zipped/vdm_"${built}".tar.gz ./* fi cd - > /dev/null done From edff4249bc8039051bdab7e0e46c9f28bc62d85a Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 19:43:53 -0500 Subject: [PATCH 26/29] oops typo --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 696bb73..0d334d6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -28,7 +28,7 @@ jobs: release: if: github.ref == 'refs/heads/main' needs: tag - run-on: ubuntu-latest + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Go @@ -38,6 +38,6 @@ jobs: - name: Package binaries run: 'make package' - name: Create GitHub Release - run: gh release create "v$(grep -v '#' ./VERSION)" --generate-notes --latest ./dist/zipped/* + run: gh release create "v$(grep -v '#' ./VERSION)" --generate-notes --verify-tag --latest ./dist/zipped/* env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f76d2b9be582970e6223ad0f3c7c9410f9539d81 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 19:59:52 -0500 Subject: [PATCH 27/29] Catch some things that should have been consts --- README.md | 45 +++++++++++++++++++------------ cmd/sync.go | 4 +-- internal/vdmspec/spec.go | 10 ++++--- internal/vdmspec/validate.go | 10 +++---- internal/vdmspec/validate_test.go | 2 +- 5 files changed, 42 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index b93e98b..cafbeeb 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,32 @@ -# vdm: Versioned Dependency Manager +# vdm: A General-Purpose Versioned-Dependency Manager `vdm` is an alternative to e.g. git submodules for managing arbitrary external -dependencies for the same reasons, in a more sane way. +dependencies for the same reasons, in a more sane way. Unlike some other tools +that try to solve this problem, `vdm` is language-agnostic, and can be used for +any purpose that you would need remote development resources. -To get started, you'll need a `vdm` spec file, which is just a YAML file -specifying all your external dependencies along with (usually) their revisions & -where you want them to live in your repo: +To get started, you'll need a `vdm` spec file, which is just a YAML (or JSON) +file specifying all your external dependencies along with (usually) their +revisions & where you want them to live on your filesystem: ```yaml remotes: - - type: "git" # the default + - type: "git" # the default, and so can be omitted if desired remote: "https://github.com/opensourcecorp/go-common" local_path: "./deps/go-common" - version: "v0.2.0" # tag example; can also be a branch, short or long commit hash, or the word 'latest' + version: "v0.2.0" # tag example; can also be a branch, commit hash, or the word 'latest' - - type: "file" # the 'file' type assumes the version is in the remote field itself, so 'version' is omitted + - type: "file" # the 'file' type assumes the version is in the remote field itself somehow, so 'version' can be omitted remote: "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto" local_path: "./deps/proto/http/http.proto" ``` -You can have as many dependency specifications in that array as you want. By -default, this spec file is called `vdm.yaml` and lives at the calling location -(which is probably your repo's root), but you can call it whatever you want and -point to it using the `--spec-file` flag to `vdm`. +You can have as many dependency specifications in that array as you want, and +they can be stored wherever you want. By default, this spec file is called +`vdm.yaml` and lives at the calling location (which is probably your repo's +root), but you can call it whatever you want and point to it using the +`--spec-file` flag to `vdm`. Once you have a spec file, just run: @@ -31,14 +34,14 @@ Once you have a spec file, just run: vdm sync ``` -and `vdm` will process the spec file, grab your dependencies, put them where +and `vdm` will process the spec file, retrieve your dependencies, put them where they belong, and check out the right versions. By default, `vdm sync` also -removes the local `.git` directories for each remote, so as to not upset your -local Git tree. If you want to change the version/revision of a remote, just -update your spec file and run `vdm sync` again. +removes the local `.git` directories for each `git` remote, so as to not upset +your local Git tree. If you want to change the version/revision of a remote, +just update your spec file and run `vdm sync` again. After running `vdm sync` with the above example spec file, your directory tree -would look like this: +would look something like this: ```txt ./vdm.yaml @@ -48,6 +51,14 @@ would look like this: http.proto ``` +## A note about auth + +`vdm` has zero goals to be an authN/authZ manager. If a remote in your spec file +depends on a certain auth setup (an SSH key, something for HTTP basic auth like +a `.netrc` file, an `.npmrc` config file, etc.), that setup is out of `vdm`'s +scope. If required, you will need to ensure proper auth is configured before +running `vdm` commands. + ## Future work - Make the sync mechanism more robust, such that if your spec file changes to diff --git a/cmd/sync.go b/cmd/sync.go index 98ca357..cbf43ea 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -57,11 +57,11 @@ SpecLoop: } switch remote.Type { - case "git", "": + case vdmspec.GitType, "": if err := remotes.SyncGit(remote); err != nil { return fmt.Errorf("syncing git remote: %w", err) } - case "file": + case vdmspec.FileType: if err := remotes.SyncFile(remote); err != nil { return fmt.Errorf("syncing file remote: %w", err) } diff --git a/internal/vdmspec/spec.go b/internal/vdmspec/spec.go index d1294cc..bfe2e3b 100644 --- a/internal/vdmspec/spec.go +++ b/internal/vdmspec/spec.go @@ -28,10 +28,12 @@ type Remote struct { const ( // MetaFileName is the name of the tracking file that vdm uses to record & // track remote statuses on disk. - MetaFileName = "VDMMETA" + MetaFileName string = "VDMMETA" - // Remote types - typeFile = "file" + // GitType represents the string to match against for git remote types. + GitType string = "git" + // FileType represents the string to match against for file remote types. + FileType string = "file" ) // MakeMetaFilePath constructs the metafile path that vdm will use to track a @@ -39,7 +41,7 @@ const ( func (r Remote) MakeMetaFilePath() string { metaFilePath := filepath.Join(r.LocalPath, MetaFileName) // TODO: this is brittle, but it's the best I can think of right now - if r.Type == typeFile { + if r.Type == FileType { fileDir := filepath.Dir(r.LocalPath) fileName := filepath.Base(r.LocalPath) // converts to e.g. 'VDMMETA_http.proto' diff --git a/internal/vdmspec/validate.go b/internal/vdmspec/validate.go index b9215e2..2724244 100644 --- a/internal/vdmspec/validate.go +++ b/internal/vdmspec/validate.go @@ -29,10 +29,10 @@ func (spec Spec) Validate() error { // Version field message.Debugf("Index #%d: validating field 'Version' for %+v", remoteIndex, remote) - if remote.Type == "git" && len(remote.Version) == 0 { + if remote.Type == GitType && len(remote.Version) == 0 { allErrors = append(allErrors, errors.New("all 'version' fields for the 'git' remote type must be non-zero length. If you don't care about the version (even though you probably should), then use 'latest'")) } - if remote.Type == "file" && len(remote.Version) > 0 { + if remote.Type == FileType && len(remote.Version) > 0 { message.Warnf("NOTE: Remote #%d '%s' specified as type '%s', which does not take explicit version info (you provided '%s'); ignoring version field", remoteIndex, remote.Remote, remote.Type, remote.Version) } @@ -45,9 +45,9 @@ func (spec Spec) Validate() error { // Type field message.Debugf("Index #%d: validating field 'Type' for %+v", remoteIndex, remote) typeMap := map[string]int{ - "git": 1, - "": 2, // also git - "file": 3, + GitType: 1, + "": 2, // also git + FileType: 3, } if _, ok := typeMap[remote.Type]; !ok { allErrors = append(allErrors, fmt.Errorf("unrecognized remote type '%s'", remote.Type)) diff --git a/internal/vdmspec/validate_test.go b/internal/vdmspec/validate_test.go index f4de2d0..7428cee 100644 --- a/internal/vdmspec/validate_test.go +++ b/internal/vdmspec/validate_test.go @@ -50,7 +50,7 @@ func TestValidate(t *testing.T) { Remote: "https://some-remote", Version: "", LocalPath: "./deps/some-remote", - Type: "git", + Type: GitType, }}, } err := spec.Validate() From 5dc3133a7cb455b47cb5cfe3e6c4719d51fd6400 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 20:25:38 -0500 Subject: [PATCH 28/29] Add more to README, add Version field to rootCmd --- README.md | 59 ++++++++++++++++++++++++++++++++++++---- cmd/root.go | 11 +++++++- cmd/sync.go | 2 +- scripts/bump-versions.sh | 6 ++-- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index cafbeeb..6e8ab06 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,49 @@ # vdm: A General-Purpose Versioned-Dependency Manager -`vdm` is an alternative to e.g. git submodules for managing arbitrary external +`vdm` is an alternative to e.g. Git Submodules for managing arbitrary external dependencies for the same reasons, in a more sane way. Unlike some other tools that try to solve this problem, `vdm` is language-agnostic, and can be used for any purpose that you would need remote development resources. +`vdm` (and Git Submodules) can be used for many different purposes, but most +commonly as a way to track external dependencies that your own code might need, +but that you don't have a language-native way to specify. Some examples might +be: + +- You have a shared CI repo from which you need to access common scripts, build + tasks, etc. + +- You're building & testing a backend application and need to test serving + frontend code from it, and your team has that frontend code in another + repository. + +- Your team uses protocol buffers and you need to be able to import other loose + `.proto` files to generate your own code. + +`vdm` lets you clearly specify all those remote dependencies & more, and +retrieve them whenever you need them. + +## Getting Started + +### Installation + +`vdm` can be installed from [its GitHub Releases +page](https://github.com/opensourcecorp/vdm/releases). There is a zipped binary +for major platforms & architectures, and those are indicated in the Asset file +name. For example, if you have an M2 macOS laptop, you would download the +`vdm_darwin-arm64.tar.gz` file, and extract it to somewhere on your `$PATH`. + +If you have a recent version of the Go toolchain available, you can also install +or run `vdm` using `go`: + +```sh +go install github.com/opensourcecorp/vdm@ +# or +go run github.com/opensourcecorp/vdm@ ... +``` + +### Usage + To get started, you'll need a `vdm` spec file, which is just a YAML (or JSON) file specifying all your external dependencies along with (usually) their revisions & where you want them to live on your filesystem: @@ -34,11 +73,11 @@ Once you have a spec file, just run: vdm sync ``` -and `vdm` will process the spec file, retrieve your dependencies, put them where -they belong, and check out the right versions. By default, `vdm sync` also -removes the local `.git` directories for each `git` remote, so as to not upset -your local Git tree. If you want to change the version/revision of a remote, -just update your spec file and run `vdm sync` again. +and `vdm` will process the spec file, retrieve your dependencies as specified, +and put them where you told them to go. By default, `vdm sync` also removes the +local `.git` directories for each `git` remote, so as to not upset your local +Git tree. If you want to change the version/revision of a remote, just update +your spec file and run `vdm sync` again. After running `vdm sync` with the above example spec file, your directory tree would look something like this: @@ -51,6 +90,14 @@ would look something like this: http.proto ``` +## Dependencies + +`vdm` is distributed as a statically-linked binary per platform that has no +language-specific dependencies. However, note that at the time of this writing, +`vdm` *does* depends on `git` being installed if you specify any `git` remote +types. `vdm` will fail with an informative error if it can't find `git` on your +`$PATH`. + ## A note about auth `vdm` has zero goals to be an authN/authZ manager. If a remote in your spec file diff --git a/cmd/root.go b/cmd/root.go index fa196ba..dbcf060 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,19 +2,28 @@ package cmd import ( "fmt" + "os" "github.com/opensourcecorp/vdm/internal/message" "github.com/spf13/cobra" "github.com/spf13/viper" ) +// !!! DO NOT TOUCH, the version-bumper script handles updating this !!! +const vdmVersion string = "v0.2.0" + var rootCmd = cobra.Command{ Use: "vdm", Short: "vdm -- a Versioned-Dependency Manager", Long: "vdm is used to manage arbitrary remote dependencies", TraverseChildren: true, - Run: func(_ *cobra.Command, _ []string) { + Version: vdmVersion, + Run: func(cmd *cobra.Command, args []string) { MaybeSetDebug() + if len(args) == 0 { + cmd.Help() + os.Exit(0) + } }, } diff --git a/cmd/sync.go b/cmd/sync.go index cbf43ea..8a6c129 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -15,7 +15,7 @@ var syncCmd = &cobra.Command{ RunE: syncExecute, } -func syncExecute(_ *cobra.Command, _ []string) error { +func syncExecute(cmd *cobra.Command, args []string) error { MaybeSetDebug() if err := sync(); err != nil { return fmt.Errorf("executing sync command: %w", err) diff --git a/scripts/bump-versions.sh b/scripts/bump-versions.sh index 5fb5b72..79318bd 100755 --- a/scripts/bump-versions.sh +++ b/scripts/bump-versions.sh @@ -3,8 +3,9 @@ set -euo pipefail ################################################################################ # A singular, consistent version needs to match across several places across the -# codebase tree, such as the Debian Control file & man pages.This script is used -# to bump version identifiers in any locations listed in the loop at the bottom. +# codebase tree, such as the Debian Control file & man pages, the embedded +# version listed for the CLI command, etc. This script is used to bump version +# identifiers in any locations listed in the loop at the bottom. ################################################################################ root="$(git rev-parse --show-toplevel)" @@ -25,6 +26,7 @@ fi for f in \ dist/debian/vdm/DEBIAN/control \ dist/man/* \ + cmd/root.go \ ; do if grep -q "${old_version}" "${root}/${f}" ; then printf 'Updating version in %s: %s -> %s\n' "${f}" "${old_version}" "${new_version}" From d4bbe275d345d6f3dee508b73e836fcf0a83dba2 Mon Sep 17 00:00:00 2001 From: "Ryan J. Price" Date: Sun, 28 Jul 2024 20:56:14 -0500 Subject: [PATCH 29/29] Fix for file types possibly not having parent directories --- README.md | 15 +++++++-------- cmd/root.go | 5 ++++- cmd/sync.go | 2 +- cmd/sync_test.go | 17 ++++++++--------- internal/remotes/file.go | 21 +++++++++++++++++++++ internal/remotes/git_test.go | 6 ++++-- internal/vdmspec/spec_test.go | 22 +++++++++++----------- testdata/vdm.yaml | 2 +- 8 files changed, 57 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 6e8ab06..a6b1ff6 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,12 @@ dependencies for the same reasons, in a more sane way. Unlike some other tools that try to solve this problem, `vdm` is language-agnostic, and can be used for any purpose that you would need remote development resources. -`vdm` (and Git Submodules) can be used for many different purposes, but most -commonly as a way to track external dependencies that your own code might need, -but that you don't have a language-native way to specify. Some examples might -be: +`vdm` can be used for many different purposes, but most commonly as a way to +track external dependencies that your own code might need, but that you don't +have a language-native way to specify. Some examples might be: -- You have a shared CI repo from which you need to access common scripts, build - tasks, etc. +- You have a shared CI repo from which you need to access common shell scripts, + hosted build tasks, etc. - You're building & testing a backend application and need to test serving frontend code from it, and your team has that frontend code in another @@ -52,7 +51,7 @@ revisions & where you want them to live on your filesystem: remotes: - type: "git" # the default, and so can be omitted if desired - remote: "https://github.com/opensourcecorp/go-common" + remote: "https://github.com/opensourcecorp/go-common" # can specify as 'git@...' to use SSH instead local_path: "./deps/go-common" version: "v0.2.0" # tag example; can also be a branch, commit hash, or the word 'latest' @@ -114,4 +113,4 @@ running `vdm` commands. - Add `--keep-git-dir` flag so that `git` remote types don't wipe the `.git` directory at clone-time. -- Support more than just Git +- Support more than just `git` and `file` types, and make `file` better diff --git a/cmd/root.go b/cmd/root.go index dbcf060..680283d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,7 +21,10 @@ var rootCmd = cobra.Command{ Run: func(cmd *cobra.Command, args []string) { MaybeSetDebug() if len(args) == 0 { - cmd.Help() + err := cmd.Help() + if err != nil { + message.Fatalf("failed to print help message, somehow") + } os.Exit(0) } }, diff --git a/cmd/sync.go b/cmd/sync.go index 8a6c129..cbf43ea 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -15,7 +15,7 @@ var syncCmd = &cobra.Command{ RunE: syncExecute, } -func syncExecute(cmd *cobra.Command, args []string) error { +func syncExecute(_ *cobra.Command, _ []string) error { MaybeSetDebug() if err := sync(); err != nil { return fmt.Errorf("executing sync command: %w", err) diff --git a/cmd/sync_test.go b/cmd/sync_test.go index a7587ac..8708f01 100644 --- a/cmd/sync_test.go +++ b/cmd/sync_test.go @@ -1,7 +1,6 @@ package cmd import ( - "os" "path/filepath" "testing" @@ -25,6 +24,13 @@ func TestSync(t *testing.T) { err = sync() require.NoError(t, err) + // defer t.Cleanup(func() { + // for _, remote := range spec.Remotes { + // err := os.RemoveAll(remote.LocalPath) + // require.NoError(t, err) + // } + // }) + t.Run("SyncGit", func(t *testing.T) { t.Run("spec[0] used a tag", func(t *testing.T) { vdmMeta, err := spec.Remotes[0].GetVDMMeta() @@ -52,17 +58,10 @@ func TestSync(t *testing.T) { }) t.Run("SyncFile", func(t *testing.T) { - t.Run("spec[4] had an implcit version", func(t *testing.T) { + t.Run("spec[4] had an implicit version", func(t *testing.T) { vdmMeta, err := spec.Remotes[4].GetVDMMeta() require.NoError(t, err) assert.Equal(t, "", vdmMeta.Version) }) }) - - t.Cleanup(func() { - for _, remote := range spec.Remotes { - err := os.RemoveAll(remote.LocalPath) - require.NoError(t, err) - } - }) } diff --git a/internal/remotes/file.go b/internal/remotes/file.go index e51715f..ed9bd59 100644 --- a/internal/remotes/file.go +++ b/internal/remotes/file.go @@ -63,6 +63,11 @@ func retrieveFile(remote vdmspec.Remote) (err error) { return fmt.Errorf("unsuccessful status code '%d' from server when retrieving remote file '%s'", resp.StatusCode, remote.Remote) } + err = ensureParentDirs(remote.LocalPath) + if err != nil { + return fmt.Errorf("creating parent directories for file: %w", err) + } + // Note: I would normally use os.WriteFile() using the returned bytes // directly, but the internet says this os.Create()/io.Copy() approach // appears to be idiomatic @@ -84,3 +89,19 @@ func retrieveFile(remote vdmspec.Remote) (err error) { return nil } + +func ensureParentDirs(path string) error { + fullPath, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("determining abspath for file '%s': %w", path, err) + } + message.Debugf("absolute filepath for '%s' determined to be '%s'", path, fullPath) + dir := filepath.Dir(fullPath) + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + return fmt.Errorf("making directories: %w", err) + } + message.Debugf("created director(ies): %s", dir) + + return nil +} diff --git a/internal/remotes/git_test.go b/internal/remotes/git_test.go index 63d0f7f..6b2bde4 100644 --- a/internal/remotes/git_test.go +++ b/internal/remotes/git_test.go @@ -23,7 +23,8 @@ func TestSyncGit(t *testing.T) { spec := getTestGitSpec() err := SyncGit(spec) require.NoError(t, err) - t.Cleanup(func() { + + defer t.Cleanup(func() { if cleanupErr := os.RemoveAll(spec.LocalPath); cleanupErr != nil { t.Fatalf("removing specLocalPath: %v", cleanupErr) } @@ -54,7 +55,8 @@ func TestCheckGitAvailable(t *testing.T) { func TestGitClone(t *testing.T) { spec := getTestGitSpec() cloneErr := gitClone(spec) - t.Cleanup(func() { + + defer t.Cleanup(func() { if cleanupErr := os.RemoveAll(spec.LocalPath); cleanupErr != nil { t.Fatalf("removing specLocalPath: %v", cleanupErr) } diff --git a/internal/vdmspec/spec_test.go b/internal/vdmspec/spec_test.go index 1795c8a..70918cc 100644 --- a/internal/vdmspec/spec_test.go +++ b/internal/vdmspec/spec_test.go @@ -34,17 +34,22 @@ func TestVDMMeta(t *testing.T) { err := os.WriteFile(testVDMMetaFilePath, []byte(testVDMMetaContents), 0644) require.NoError(t, err) + defer t.Cleanup(func() { + err := os.RemoveAll(testVDMMetaFilePath) + require.NoError(t, err) + }) + got, err := testRemote.GetVDMMeta() require.NoError(t, err) assert.Equal(t, testRemote, got) + }) - t.Cleanup(func() { + t.Run("WriteVDMMeta", func(t *testing.T) { + defer t.Cleanup(func() { err := os.RemoveAll(testVDMMetaFilePath) require.NoError(t, err) }) - }) - t.Run("WriteVDMMeta", func(t *testing.T) { // Needs to have parent dir(s) exist for write to work err := os.MkdirAll(testRemote.LocalPath, 0644) require.NoError(t, err) @@ -55,21 +60,16 @@ func TestVDMMeta(t *testing.T) { got, err := testRemote.GetVDMMeta() require.NoError(t, err) assert.Equal(t, testRemote, got) + }) - t.Cleanup(func() { + t.Run("GetSpecsFromFile", func(t *testing.T) { + defer t.Cleanup(func() { err := os.RemoveAll(testVDMMetaFilePath) require.NoError(t, err) }) - }) - t.Run("GetSpecsFromFile", func(t *testing.T) { spec, err := GetSpecFromFile(testSpecFilePath) require.NoError(t, err) assert.Equal(t, 5, len(spec.Remotes)) - - t.Cleanup(func() { - err := os.RemoveAll(testVDMMetaFilePath) - require.NoError(t, err) - }) }) } diff --git a/testdata/vdm.yaml b/testdata/vdm.yaml index 2cfab27..039d1e7 100644 --- a/testdata/vdm.yaml +++ b/testdata/vdm.yaml @@ -13,4 +13,4 @@ remotes: local_path: "./deps/go-common-hash" - type: "file" remote: "https://raw.githubusercontent.com/googleapis/googleapis/master/google/api/http.proto" - local_path: "./deps/http.proto" + local_path: "./deps/proto/http/http.proto"