diff --git a/.github/workflows/common/base.cue b/.github/workflows/common/base.cue new file mode 100644 index 000000000..9c4357ee5 --- /dev/null +++ b/.github/workflows/common/base.cue @@ -0,0 +1,31 @@ +package common + +import "github.com/hofstadter-io/ghacue" + +#Workflow: ghacue.#Workflow & { + name: string + on: _ | *["pull_request"] + jobs: test: { + strategy: matrix: { + "go-version": ["1.16.x", "1.17.x"] + os: ["ubuntu-latest", "macos-latest"] + } + strategy: "fail-fast": false + "runs-on": "${{ matrix.os }}" + } +} + +#BuildSteps: [{ + name: "Install Go" + uses: "actions/setup-go@v2" + with: "go-version": "${{ matrix.go-version }}" +},{ + name: "Checkout code" + uses: "actions/checkout@v2" +},{ + name: "Download mods" + run: "go mod download" +},{ + name: "Build CLI" + run: "go install ./cmd/hof" +}] diff --git a/.github/workflows/default.cue b/.github/workflows/default.cue index a4727f4a7..7edd959cd 100644 --- a/.github/workflows/default.cue +++ b/.github/workflows/default.cue @@ -1,33 +1,20 @@ -import "github.com/hofstadter-io/ghacue" +package workflows -ghacue.#Workflow & { - name: "hof" - on: ["push", "pull_request"] +import "github.com/hofstadter-io/hof/.github/workflows/common" + +common.#Workflow & { + name: "default" + on: ["push"] jobs: test: { - strategy: matrix: { - "go-version": ["1.16.x"] - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - } - strategy: "fail-fast": false - "runs-on": "${{ matrix.os }}" - steps: [{ - name: "Install Go" - uses: "actions/setup-go@v2" - with: "go-version": "${{ matrix.go-version }}" - },{ - name: "Checkout code" - uses: "actions/checkout@v2" - },{ - name: "Download mods" - run: "go mod download" - },{ - name: "Build CLI" - run: "go install ./cmd/hof" - },{ - name: "Run tests" + steps: [ for step in common.#BuildSteps {step} ] + [{ + name: "Run gen tests" run: """ hof test test.cue -s gen -t test - hof test test.cue -s lib -t test -t mod + """ + },{ + name: "Run tester tests" + run: """ + hof test test/testers/api/postman.cue """ }] } diff --git a/.github/workflows/default.yml b/.github/workflows/default.yml index 57526d76e..2d3a75f94 100644 --- a/.github/workflows/default.yml +++ b/.github/workflows/default.yml @@ -1,17 +1,16 @@ -name: hof +name: default on: - push -- pull_request jobs: test: strategy: matrix: go-version: - 1.16.x + - 1.17.x os: - ubuntu-latest - macos-latest - - windows-latest fail-fast: false runs-on: ${{ matrix.os }} steps: @@ -25,8 +24,7 @@ jobs: run: go mod download - name: Build CLI run: go install ./cmd/hof - - name: Run tests - run: |- - hof test test.cue -s gen -t test - hof test test.cue -s lib -t test -t mod - services: {} + - name: Run gen tests + run: hof test test.cue -s gen -t test + - name: Run tester tests + run: hof test test/testers/api/postman.cue diff --git a/.github/workflows/test_mod.cue b/.github/workflows/test_mod.cue new file mode 100644 index 000000000..be91903b1 --- /dev/null +++ b/.github/workflows/test_mod.cue @@ -0,0 +1,25 @@ +package workflows + +import "github.com/hofstadter-io/hof/.github/workflows/common" + +common.#Workflow & { + name: "test_mod" + on: pull_request: { paths: ["lib/mod/**", "lib/yagu/repos/**", "lib/yagu/git.go", "lib/yagu/netrc.go", "lib/yagu/ssh.go"] } + jobs: test: { + environment: "hof mod testing" + steps: [ for step in common.#BuildSteps {step} ] + [{ + name: "Run mod tests" + run: """ + hof test test.cue -s lib -t test -t mod + """ + env: { + HOFMOD_SSHKEY: "${{secrets.HOFMOD_SSHKEY}}" + GITHUB_TOKEN: "${{secrets.HOFMOD_TOKEN}}" + GITLAB_TOKEN: "${{secrets.GITLAB_TOKEN}}" + BITBUCKET_USERNAME: "hofstadter" + BITBUCKET_PASSWORD: "${{secrets.BITBUCKET_TOKEN}}" + } + }] + } +} + diff --git a/.github/workflows/test_mod.yml b/.github/workflows/test_mod.yml new file mode 100644 index 000000000..3f80a8ea7 --- /dev/null +++ b/.github/workflows/test_mod.yml @@ -0,0 +1,41 @@ +name: test_mod +on: + pull_request: + paths: + - lib/mod/** + - lib/yagu/repos/** + - lib/yagu/git.go + - lib/yagu/netrc.go + - lib/yagu/ssh.go +jobs: + test: + strategy: + matrix: + go-version: + - 1.16.x + - 1.17.x + os: + - ubuntu-latest + - macos-latest + fail-fast: false + runs-on: ${{ matrix.os }} + environment: hof mod testing + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout code + uses: actions/checkout@v2 + - name: Download mods + run: go mod download + - name: Build CLI + run: go install ./cmd/hof + - name: Run mod tests + run: hof test test.cue -s lib -t test -t mod + env: + HOFMOD_SSHKEY: ${{secrets.HOFMOD_SSHKEY}} + GITHUB_TOKEN: ${{secrets.HOFMOD_TOKEN}} + GITLAB_TOKEN: ${{secrets.GITLAB_TOKEN}} + BITBUCKET_USERNAME: hofstadter + BITBUCKET_PASSWORD: ${{secrets.BITBUCKET_TOKEN}} diff --git a/.hof/shadow/Cli/cmd/hof/cmd/mod.go b/.hof/shadow/Cli/cmd/hof/cmd/mod.go index d3887f5c7..9e94c786d 100644 --- a/.hof/shadow/Cli/cmd/hof/cmd/mod.go +++ b/.hof/shadow/Cli/cmd/hof/cmd/mod.go @@ -11,20 +11,111 @@ import ( "github.com/hofstadter-io/hof/cmd/hof/cmd/mod" ) -var modLong = `The mod subcmd is a polyglot dependency management tool based on go mods. +var modLong = `hof mod is a flexible tool and library based on Go mods. -mod file format: +Use and create module systems with Minimum Version Selection (MVS) semantics +and manage dependencies go mod style. Mix any set of language, code bases, +git repositories, package managers, and subdirectories. - module = "" - = "version" +### Features - require ( - ... - ) +- Based on go mods MVS system, aiming for 100% reproducible builds. +- Recursive dependencies, version resolution, and code instrospection. +- Custom module systems with custom file names and vendor directories. +- Control configuration for naming, vendoring, and other behaviors. +- Polyglot support for multiple module systems and multiple languages within one tool. +- Works with any git system and supports the main features from go mods. +- Convert other vendor and module systems to MVS or just manage their files with MVS. +- Private repository support for GitHub, GitLab, Bitbucket, and git+SSH. - replace => - ...` + +### Usage + +# Print known languages in the current directory +hof mod info + +# Initialize this folder as a module +hof mod init + +# Add your requirements +vim .mods # go.mod like file + +# Pull in dependencies, no args discovers by *.mods and runs all +hof mod vendor [langs...] + +# See all of the commands +hof mod help + + +### Module File + +The module file holds the requirements for project. +It has the same format as a go.mod file. + +--- +# These are like golang import paths +# i.e. github.com/hofstadter-io/hof +module + +# Information about the module type / version +# some systems that take this into account +# go = 1.14 + = + +# Required dependencies section +require ( + # + github.com/hof-lang/cuemod--cli-golang v0.0.0 # This is latest on HEAD + github.com/hof-lang/cuemod--cli-golang v0.1.5 # This is a tag v0.1.5 (can omit 'v' in tag, but not here) +) + +# replace => [version] +replace github.com/hof-lang/cuemod--cli-golang => github.com/hofstadter-io/hofmod-cli-golang v0.2.0 +replace github.com/hof-lang/cuemod--cli-golang => ../../cuelibs/cuemod--cli-golang +--- + + +### Authentication and private modules + +hof mod prefers authenticated requests when fetching dependencies. +This increase rate limits with hosts and supports private modules. +Both token and sshkey base methods are supported. + +If you are using credentials, then private modules can be transparent. +An ENV VAR like GOPRIVATE and CUEPRIVATE can be used to require credentials. + +The following ENV VARS are used to set credentials. + +GITHUB_TOKEN +GITLAB_TOKEN +BITBUCKET_TOKEN or BITBUCKET_USERNAME / BITBUCKET_PASSWORD * + +SSH keys will be looked up in the following ~/.ssh/config, /etc/ssh/config, ~/.ssh/in_rsa + +You can configure the SSH key with + +HOF_SSHUSR and HOF_SSHKEY + +* The bitbucket method will depend on the account type and enterprise license. + + +### Custom Module Systems + +.mvsconfig.cue allows you to define custom module systems. +With some simple configuration, you can create and control +and vendored module system based on go mods. +You can also define global configurations. + +See the ./lib/mod/langs in the repository for examples. + +### Motivation + +- MVS has better semantics for vendoring and gets us closer to 100% reproducible builds. +- JS and Python can have MVS while still using the remainder of the tool chains. +- Prototype for cuelang module and vendor management. +- We need a module system for our [hof-lang](https://hof-lang.org) project. +` func ModPersistentPreRun(args []string) (err error) { diff --git a/Makefile b/Makefile index c2a023158..1be9dfe12 100644 --- a/Makefile +++ b/Makefile @@ -21,3 +21,12 @@ cloccode: cmd lang lib gen cmd clocdev: hof.cue design schema lang lib docs test cue.mods go.mod cloc --read-lang-def=$$HOME/hof/jumpfiles/assets/cloc_defs.txt --exclude-dir=cue.mod,vendor $^ + +WORKFLOWS = default \ + test_mod + +.PHONY: workflow +workflows = $(addprefix workflow_, $(WORKFLOWS)) +workflow: $(workflows) +$(workflows): workflow_%: + @cue export --out yaml .github/workflows/$(subst workflow_,,$@).cue > .github/workflows/$(subst workflow_,,$@).yml diff --git a/cmd/hof/cmd/mod.go b/cmd/hof/cmd/mod.go index d3887f5c7..9e94c786d 100644 --- a/cmd/hof/cmd/mod.go +++ b/cmd/hof/cmd/mod.go @@ -11,20 +11,111 @@ import ( "github.com/hofstadter-io/hof/cmd/hof/cmd/mod" ) -var modLong = `The mod subcmd is a polyglot dependency management tool based on go mods. +var modLong = `hof mod is a flexible tool and library based on Go mods. -mod file format: +Use and create module systems with Minimum Version Selection (MVS) semantics +and manage dependencies go mod style. Mix any set of language, code bases, +git repositories, package managers, and subdirectories. - module = "" - = "version" +### Features - require ( - ... - ) +- Based on go mods MVS system, aiming for 100% reproducible builds. +- Recursive dependencies, version resolution, and code instrospection. +- Custom module systems with custom file names and vendor directories. +- Control configuration for naming, vendoring, and other behaviors. +- Polyglot support for multiple module systems and multiple languages within one tool. +- Works with any git system and supports the main features from go mods. +- Convert other vendor and module systems to MVS or just manage their files with MVS. +- Private repository support for GitHub, GitLab, Bitbucket, and git+SSH. - replace => - ...` + +### Usage + +# Print known languages in the current directory +hof mod info + +# Initialize this folder as a module +hof mod init + +# Add your requirements +vim .mods # go.mod like file + +# Pull in dependencies, no args discovers by *.mods and runs all +hof mod vendor [langs...] + +# See all of the commands +hof mod help + + +### Module File + +The module file holds the requirements for project. +It has the same format as a go.mod file. + +--- +# These are like golang import paths +# i.e. github.com/hofstadter-io/hof +module + +# Information about the module type / version +# some systems that take this into account +# go = 1.14 + = + +# Required dependencies section +require ( + # + github.com/hof-lang/cuemod--cli-golang v0.0.0 # This is latest on HEAD + github.com/hof-lang/cuemod--cli-golang v0.1.5 # This is a tag v0.1.5 (can omit 'v' in tag, but not here) +) + +# replace => [version] +replace github.com/hof-lang/cuemod--cli-golang => github.com/hofstadter-io/hofmod-cli-golang v0.2.0 +replace github.com/hof-lang/cuemod--cli-golang => ../../cuelibs/cuemod--cli-golang +--- + + +### Authentication and private modules + +hof mod prefers authenticated requests when fetching dependencies. +This increase rate limits with hosts and supports private modules. +Both token and sshkey base methods are supported. + +If you are using credentials, then private modules can be transparent. +An ENV VAR like GOPRIVATE and CUEPRIVATE can be used to require credentials. + +The following ENV VARS are used to set credentials. + +GITHUB_TOKEN +GITLAB_TOKEN +BITBUCKET_TOKEN or BITBUCKET_USERNAME / BITBUCKET_PASSWORD * + +SSH keys will be looked up in the following ~/.ssh/config, /etc/ssh/config, ~/.ssh/in_rsa + +You can configure the SSH key with + +HOF_SSHUSR and HOF_SSHKEY + +* The bitbucket method will depend on the account type and enterprise license. + + +### Custom Module Systems + +.mvsconfig.cue allows you to define custom module systems. +With some simple configuration, you can create and control +and vendored module system based on go mods. +You can also define global configurations. + +See the ./lib/mod/langs in the repository for examples. + +### Motivation + +- MVS has better semantics for vendoring and gets us closer to 100% reproducible builds. +- JS and Python can have MVS while still using the remainder of the tool chains. +- Prototype for cuelang module and vendor management. +- We need a module system for our [hof-lang](https://hof-lang.org) project. +` func ModPersistentPreRun(args []string) (err error) { diff --git a/design/cli/cmds/mod.cue b/design/cli/cmds/mod.cue index 73312bca4..cd32ea35b 100644 --- a/design/cli/cmds/mod.cue +++ b/design/cli/cmds/mod.cue @@ -14,22 +14,7 @@ import ( Usage: "mod" Aliases: ["m"] Short: "mod subcmd is a polyglot dependency management tool based on go mods" - Long: """ - The mod subcmd is a polyglot dependency management tool based on go mods. - - mod file format: - - module = "" - - = "version" - - require ( - ... - ) - - replace => - ... - """ + Long: #ModLongHelp //Topics: #ModTopics //Examples: #ModExamples @@ -44,7 +29,6 @@ import ( """ Commands: [{ - // TBD: "✓" Name: "info" Usage: "info [language]" Short: "print info about languages and modders known to hof mod" @@ -71,71 +55,7 @@ import ( fmt.Println(msg) """ - //}, { - //TBD: "Ø" - //Name: "convert" - //Usage: "convert " - //Short: "convert another package system to MVS." - //Long: Short - - //Args: [{ - //Name: "lang" - //Type: "string" - //Required: true - //Help: "name of the language to print info about" - //}, { - //Name: "filename" - //Type: "string" - //Required: true - //Help: "existing package filename, depending on language" - //}] - - //Imports: #ModCmdImports - - //Body: """ - //err = mod.Convert(lang, filename) - //if err != nil { - //fmt.Println(err) - //os.Exit(1) - //} - //""" - - //}, { - //TBD: "Ø" - //Name: "graph" - //Usage: "graph" - //Short: "print module requirement graph" - //Long: Short - - //Imports: #ModCmdImports - - //Body: """ - //err = mod.ProcessLangs("graph", args) - //if err != nil { - //fmt.Println(err) - //os.Exit(1) - //} - //""" - - //}, { - //TBD: "Ø" - //Name: "status" - //Usage: "status" - //Short: "print module dependencies status" - //Long: Short - - //Imports: #ModCmdImports - - //Body: """ - //err = mod.ProcessLangs("status", args) - //if err != nil { - //fmt.Println(err) - //os.Exit(1) - //} - //""" - }, { - // TBD: "✓" Name: "init" Usage: "init " Short: "initialize a new module in the current directory" @@ -163,23 +83,6 @@ import ( } """ - //}, { - //TBD: "Ø" - //Name: "tidy" - //Usage: "tidy [langs...]" - //Short: "add missinad and remove unused modules" - //Long: Short - - //Imports: #ModCmdImports - - //Body: """ - //err = mod.ProcessLangs("tidy", args) - //if err != nil { - //fmt.Println(err) - //os.Exit(1) - //} - //""" - }, { // TBD: "β" Name: "vendor" @@ -197,361 +100,114 @@ import ( } """ - //}, { - //TBD: "Ø" - //Name: "verify" - //Usage: "verify [langs...]" - //Short: "verify dependencies have expected content" - //Long: Short - - //Imports: #ModCmdImports - - //Body: """ - //err = mod.ProcessLangs("verify", args) - //if err != nil { - //fmt.Println(err) - //os.Exit(1) - //} - //""" - }] } -#ModTopics: { - "readme": ##""" - # hof mod - a golang MVS system for anything - - `hof mod` is a flexible tool and library based on Go mods. - - Use and create module systems with [Minimum Version Selection](https://research.swtch.com/vgo-mvs) semantics - and manage dependencies `go mod` style. - Mix any set of language, code bases, git repositories, package managers, and subdirectories. - Manage polyglot and monorepo codebase dependencies with - [100% reproducible builds](https://github.com/golang/go/wiki/Modules#version-selection) from a single tool. - - - ### Features - - - Based on go mods MVS system, aiming for 100% reproducible builds. - - Recursive dependencies, version resolution, and code instrospection. - - Custom module systems with custom file names and vendor directories. - - Control configuration for naming, vendoring, and other behaviors. - - Polyglot support for multiple module systems and multiple languages within one tool. - - Works with any git system and supports the main features from go mods. - - Convert other vendor and module systems to MVS or just manage their files with MVS. - - Language support: - - - [golang](https://golang.org) - delegates to the go tool for the commands mirrored here - - [cuelang](https://cuelang.org) - builtin in default using the custom module feature - - [hofmods](https://hofstadter.io) - extends Cue with advanced code generation capabilities - - [custom](./docs/custom-modders.md) - Create your own locally or globally with `.mvsconfig` files - - Upcoming languages: Python and JavaScript - so they can have an MVS system and the benefits, - and `hof mod` can start supporting and fetching from package registries. - These language implementations will have flexibility to - manage with `hof mod` and the common toolchain to varying degrees. - Pull requests for improved language support are welcome. - - The main difference from go mods is that `hof mod`, generally, - is not introspecting your code to determine dependencies. - It relies on user management of the `.mods` file. - Since golang is exec'd out to, introspection is supported, - and as more languages improve, we look to similarly - improve this situation in `hof mod`. - A midstep to full custom implementation will be a - introspection custom module with some basic support - using file globs and regex lists. - - _Note, we also default to the plural `.mods/sums` files and `/` vendor directory. - This is 1) because cuelang reads from `cue.mod` directory, and 2) because it is less likely - to collide with existing directories. - You can also configure more behavior per language and module than go mods. - The go mods is shelled out to as a convience, and often languages impose restrictions._ - - - ### Usage - - ```shell - # Print known languages in the current directory - hof mod info - - # Initialize this folder as a module - hof mod init - - # Add your requirements - vim .mods # go.mod like file +#ModLongHelp: string & ##""" +hof mod is a flexible tool and library based on Go mods. - # Pull in dependencies, no args discovers by *.mods and runs all - hof mod vendor [langs...] +Use and create module systems with Minimum Version Selection (MVS) semantics +and manage dependencies go mod style. Mix any set of language, code bases, +git repositories, package managers, and subdirectories. - # See all of the commands - hof mod help - ``` +### Features - ### Module File +- Based on go mods MVS system, aiming for 100% reproducible builds. +- Recursive dependencies, version resolution, and code instrospection. +- Custom module systems with custom file names and vendor directories. +- Control configuration for naming, vendoring, and other behaviors. +- Polyglot support for multiple module systems and multiple languages within one tool. +- Works with any git system and supports the main features from go mods. +- Convert other vendor and module systems to MVS or just manage their files with MVS. +- Private repository support for GitHub, GitLab, Bitbucket, and git+SSH. - The module file holds the requirements for project. - It has the same format as a `go.mod` file. - ``` - # These are like golang import paths - # i.e. github.com/hofstadter-io/hof - module +### Usage - # Information about the module type / version - # some systems that take this into account - # go = 1.14 - = +# Print known languages in the current directory +hof mod info - # Required dependencies section - require ( - # - github.com/hof-lang/cuemod--cli-golang v0.0.0 # This is latest on HEAD - github.com/hof-lang/cuemod--cli-golang v0.1.5 # This is a tag v0.1.5 (can omit 'v' in tag, but not here) - ) +# Initialize this folder as a module +hof mod init - # replace => [version] - replace github.com/hof-lang/cuemod--cli-golang => github.com/hofstadter-io/hofmod-cli-golang v0.2.0 - replace github.com/hof-lang/cuemod--cli-golang => ../../cuelibs/cuemod--cli-golang - ``` +# Add your requirements +vim .mods # go.mod like file +# Pull in dependencies, no args discovers by *.mods and runs all +hof mod vendor [langs...] - ### Custom Module Systems +# See all of the commands +hof mod help - `.mvsconfig.cue` allows you to define custom module systems. - With some simple configuration, you can create and control - and vendored module system based on `go mods`. - You can also define global configurations. - See the [custom-modder docs](./docs/custom-modders.md) - to learn more about writing - you own module systems. +### Module File - This is the current Cue __modder__ configuration: +The module file holds the requirements for project. +It has the same format as a go.mod file. - ```cue - cue: { - Name: "cue" - Version: "0.1.1" - ModFile: "cue.mods" - SumFile: "cue.sums" - ModsDir: "cue.mod/pkg" - MappingFile: "cue.mod/modules.txt" - InitTemplates: { - "cue.mod/module.cue": """ - module "{{ .Module }}" - """ - } +--- +# These are like golang import paths +# i.e. github.com/hofstadter-io/hof +module - VendorIncludeGlobs: [] - VendorExcludeGlobs: [ - "/.git/**", - "**/cue.mod/pkg/**", - ] - } - ``` +# Information about the module type / version +# some systems that take this into account +# go = 1.14 + = - ### Motivation - - - MVS has better semantics for vendoring and gets us closer to 100% reproducible builds. - - JS and Python can have MVS while still using the remainder of the tool chains. - - Prototype for cuelang module and vendor management. - - We need a module system for our [hof-lang](https://hof-lang.org) project. - - #### Links about go mods - - [Using go modules](https://blog.golang.org/using-go-modules) - - [Go and Versioning](https://research.swtch.com/vgo) - - [More about version selection](https://github.com/golang/go/wiki/Modules#version-selection) - - - #### Other - - You may also like to look at the [hofmod-cli](https://github.com/hofstadter-io/hofmod-cli) project. - We use this to generate the CLI code and files for CI. - - You can find us in the - [cuelang slack](https://join.slack.com/t/cuelang/shared_invite/enQtNzQwODc3NzYzNTA0LTAxNWQwZGU2YWFiOWFiOWQ4MjVjNGQ2ZTNlMmIxODc4MDVjMDg5YmIyOTMyMjQ2MTkzMTU5ZjA1OGE0OGE1NmE) - for now. - - """ - - "overview": ##""" - hof mods enable you to manage dependencies and versions for just about anything. - - You can run `hof mod ...` for language specific operations or use - just `hof mod ` to auto-discover module systems and run the operation on each. - - Modules do not have to be tied to a language, they can be arbitrary git repos. - You have a lot of control for which files to include/exclude as well as - life-cycle operations for performing any tasks around your modules. - - """## - - - "mod-file": ##""" - The mod file is the same format as Golang mod files. The only difference is that - the "Go = 1.14" is replaced with " = ." - - """## - - "go": ##""" - hof mod for Golang uses the Go tool behind the scenes. - - """## - - "cue": ##""" - hof mod for Cuelang is a prototype until module and dependency management is in the Cue tool. +# Required dependencies section +require ( + # + github.com/hof-lang/cuemod--cli-golang v0.0.0 # This is latest on HEAD + github.com/hof-lang/cuemod--cli-golang v0.1.5 # This is a tag v0.1.5 (can omit 'v' in tag, but not here) +) - The version here is quite robust, however it lacks the code introspection like Golang, - so you are required to manage your own cue.mods file. +# replace => [version] +replace github.com/hof-lang/cuemod--cli-golang => github.com/hofstadter-io/hofmod-cli-golang v0.2.0 +replace github.com/hof-lang/cuemod--cli-golang => ../../cuelibs/cuemod--cli-golang +--- - """## - "js": ##""" - Not supported yet, but the idea is to use Go's MVS system on JS and other languages. +### Authentication and private modules - """## +hof mod prefers authenticated requests when fetching dependencies. +This increase rate limits with hosts and supports private modules. +Both token and sshkey base methods are supported. - "py": ##""" - Not supported yet, but the idea is to use Go's MVS system on Python and other languages. +If you are using credentials, then private modules can be transparent. +An ENV VAR like GOPRIVATE and CUEPRIVATE can be used to require credentials. - """## +The following ENV VARS are used to set credentials. - "custom": ##""" - hof mod allows you to create custom module systems and have MVS dependency management out of the box. +GITHUB_TOKEN +GITLAB_TOKEN +BITBUCKET_TOKEN or BITBUCKET_USERNAME / BITBUCKET_PASSWORD * - see: hof mod --example custom +SSH keys will be looked up in the following ~/.ssh/config, /etc/ssh/config, ~/.ssh/in_rsa - """## -} +You can configure the SSH key with -#ModExamples: { - cue: ##""" - # Create a Cue module: - - A Cue module example. - - ### Create a Cue module: - - ``` - hof mod init cue github.com/hofstadter-io/my-cue-mod - ``` - - ### Vendor dependencies - - ``` - hof mod vendor cue - ``` - - You can use replaces to work on code locally - - """## - - custom: ##""" - # Custom Modules and Dependency Systems - - hod mod gives you the ability to create - custom module systems, called Modders. - Modder is the struct name for the - internal code which controls how - modules and vendoring is handled. - You can configure as many of these as you like, - by providing global or local `.mvsconfig.cue` files. - - You create your own modders by createing `.mvsconfig.cue` files. - hog mod will look for these in two places before any commands are run. - - - A global `$HOME/.mvs/config.cue` - - A local `./.mvsconfig.cue` - - ### hof mod "modder" config file format - - - ```cue - // These two need to be the same - cue: { - Name: "cue" - // non-semver of the language - Version: "#.#.#" - - // Common defaults, can be anything - ModFile: ".mods" - SumFile: ".sums" - ModsDir: ".mod/pkg" - Checksum: ".mod/checksum.txt" - - // Controls for modders that want to shell out - // to common tools for certain commands - NoLoad: false - CommandInit: [[string]] - CommandGraph: [[string]] - CommandTidy: [[string]] - CommandVendor: [[string]] - CommandVerify: [[string]] - CommandStatus: [[string]] - - // Runs on init for this language - // filename/content key/pair values - // uses the golang text/template library - // inputs will be - // .Language - // .Module - // .Modder - InitTemplates: { - ".mod/module.": """ - module "{{ .Module }}" - """ - } - // Series of commands to be executed pre/post init - InitPreCommands: [[string]] - InitPostCommands: [[string]] - - // Same as the InitTemplates, but run during vendor command - VendorTemplates: { - ".mod/module.": """ - module "{{ .Module }}" - """ - } +HOF_SSHUSR and HOF_SSHKEY - VendorIncludeGlobs: [ - ".mvsconfig.cue", - ".mods", - ".sums", - ".mod/module.", - ".mod/modules.txt", - "**/*.", - ] - VendorExcludeGlobs: [".mod/pkg"] +* The bitbucket method will depend on the account type and enterprise license. - // Series of commands to be executed pre/post vendoring - VendorPreCommands: [[string]] - VendorPostCommands: [[string]] - // Use MVS to only manage the languages normal dependency file - ManageFileOnly: false +### Custom Module Systems - // Whether local replaces should use a symlink instead of copying files - SymlinkLocalReplaces: false +.mvsconfig.cue allows you to define custom module systems. +With some simple configuration, you can create and control +and vendored module system based on go mods. +You can also define global configurations. - // Controls the code introspection for dependency determiniation - IntrospectIncludeGlobs: ["**/*."] - IntrospectExcludeGlobs: [".mod/pkg"] - IntrospectExtractRegex: ["you will have to figure out a series of 'any match passes' regexps to pull out dependencies"] +See the ./lib/mod/langs in the repository for examples. - // This field determines the prefix to place in front of - // imports which have a single token or leverage package managers - // This is currently futurology for building MVS for Python and JavaScript - PackageManagerDefaultPrefix: "npm.js" - } - ``` +### Motivation - """## +- MVS has better semantics for vendoring and gets us closer to 100% reproducible builds. +- JS and Python can have MVS while still using the remainder of the tool chains. +- Prototype for cuelang module and vendor management. +- We need a module system for our [hof-lang](https://hof-lang.org) project. -} +"""## diff --git a/go.mod b/go.mod index a13221446..3ecfc1ff2 100644 --- a/go.mod +++ b/go.mod @@ -23,13 +23,14 @@ require ( github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.1.0 github.com/golang/protobuf v1.4.2 // indirect - github.com/google/go-github/v30 v30.1.0 + github.com/google/go-github/v38 v38.1.0 github.com/hofstadter-io/data-utils v0.0.0-20200128210141-0a3e569b27ed github.com/hofstadter-io/dotpath v0.0.0-20191027071558-52e2819b7d2d github.com/hofstadter-io/hof-lang v0.0.0-20180925221847-4d1c962b0729 github.com/kevinburke/ssh_config v1.1.0 github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f github.com/kr/pretty v0.2.0 + github.com/ktrysmt/go-bitbucket v0.9.24 github.com/mattn/go-colorable v0.1.6 // indirect github.com/mattn/go-zglob v0.0.2 github.com/mitchellh/mapstructure v1.3.2 // indirect @@ -46,7 +47,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.6.1 - github.com/xanzy/go-gitlab v0.50.1 + github.com/xanzy/go-gitlab v0.50.3 go.uber.org/zap v1.15.0 golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d diff --git a/go.sum b/go.sum index a81aec17c..e0a7eab80 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cuelang.org/go v0.4.0-rc.1 h1:X8fsqVhLCvXFhsWMGbI8rjTal45YOgt+ko+m7rOCySM= -cuelang.org/go v0.4.0-rc.1/go.mod h1:tz/edkPi+T37AZcb5GlPY+WJkL6KiDlDVupKwL3vvjs= cuelang.org/go v0.4.0 h1:GLJblw6m2WGGCA3k1v6Wbk9gTOt2qto48ahO2MmSd6I= cuelang.org/go v0.4.0/go.mod h1:tz/edkPi+T37AZcb5GlPY+WJkL6KiDlDVupKwL3vvjs= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -122,6 +120,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -138,10 +137,11 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= -github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github/v38 v38.1.0 h1:C6h1FkaITcBFK7gAmq4eFzt6gbhEhk7L5z6R3Uva+po= +github.com/google/go-github/v38 v38.1.0/go.mod h1:cStvrz/7nFr0FoENgG6GLbp53WaelXucT+BBz/3VKx4= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -204,6 +204,10 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v2.3.0+incompatible h1:EKhKbi34VQDWJtq+zpsKSEhkHHs9w2P8Izbq8IhLVSo= +github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kevinburke/ssh_config v1.1.0 h1:pH/t1WS9NzT8go394IqZeJTMHVm6Cr6ZJ6AQ+mdNo/o= github.com/kevinburke/ssh_config v1.1.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -220,6 +224,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ktrysmt/go-bitbucket v0.9.24 h1:8gjKNJ3UMV3Yc+eAc3yrvT45cDy2e41axNwVXHt19kg= +github.com/ktrysmt/go-bitbucket v0.9.24/go.mod h1:FWxy2UK7GlK5b0NSJGc5hPqnssVlkNnsChvyuOf/Xno= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= @@ -251,6 +257,7 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -340,8 +347,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/xanzy/go-gitlab v0.50.1 h1:eH1G0/ZV1j81rhGrtbcePjbM5Ern7mPA4Xjt+yE+2PQ= -github.com/xanzy/go-gitlab v0.50.1/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= +github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY= +github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -398,6 +405,7 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 h1:xUIPaMhvROX9dhPvRCenIJtU78+lbEenGbgqB5hfHCQ= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -417,6 +425,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -430,6 +439,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -497,6 +507,7 @@ google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/lib/mod/cache/fetch.go b/lib/mod/cache/fetch.go index ab6059f20..ae1fa0eaf 100644 --- a/lib/mod/cache/fetch.go +++ b/lib/mod/cache/fetch.go @@ -4,11 +4,9 @@ import ( "fmt" "os" - "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" - googithub "github.com/google/go-github/v30/github" - "github.com/hofstadter-io/hof/lib/yagu" + "github.com/hofstadter-io/hof/lib/yagu/repos/bbc" "github.com/hofstadter-io/hof/lib/yagu/repos/git" "github.com/hofstadter-io/hof/lib/yagu/repos/github" "github.com/hofstadter-io/hof/lib/yagu/repos/gitlab" @@ -26,6 +24,8 @@ func Fetch(lang, mod, ver, pev string) (err error) { return err } // not found, try fetching deps + + // determine private from ENV VAR in modder config (passed in as pev) private := MatchPrefixPatterns(os.Getenv(pev), mod) if err := fetch(lang, mod, ver, private); err != nil { return err @@ -33,143 +33,42 @@ func Fetch(lang, mod, ver, pev string) (err error) { } // else we have it already - // fmt.Println("Found in cache") - return nil } func fetch(lang, mod, ver string, private bool) error { - remote, owner, repo := parseModURL(mod) - tag := ver - - if private { - return fetchGit(lang, remote, owner, repo, tag, true) - } else if remote == "github.com" { - return fetchGitHub(lang, owner, repo, tag) - } else if remote == "gitlab.com" { - return fetchGitLab(lang, owner, repo, tag) - } - return fetchGit(lang, remote, owner, repo, tag, private) -} - -func fetchGit(lang, remote, owner, repo, tag string, private bool) error { - FS := memfs.New() - - if err := git.FetchGit(FS, remote, owner, repo, tag, private); err != nil { - return fmt.Errorf("While fetching from git\n%w\n", err) - } - - if err := Write(lang, remote, owner, repo, tag, FS); err != nil { - return fmt.Errorf("While writing to cache\n%w\n", err) - } - - return nil -} - -func fetchGitLab(lang, owner, repo, tag string) (err error) { - FS := memfs.New() - client, err := gitlab.NewClient() - if err != nil { - return err - } - - zReader, err := gitlab.FetchZip(client, owner, repo, tag) - if err != nil { - return fmt.Errorf("While fetching from GitLab\n%w\n", err) - } - - if err := yagu.BillyLoadFromZip(zReader, FS, true); err != nil { - return fmt.Errorf("While reading zipfile\n%w\n", err) - } - - if err := Write(lang, "gitlab.com", owner, repo, tag, FS); err != nil { - return fmt.Errorf("While writing to cache\n%w\n", err) - } - - return nil -} - -func fetchGitHub(lang, owner, repo, tag string) (err error) { + fmt.Println("fetch: ", lang, mod, ver, private) FS := memfs.New() - if tag == "v0.0.0" { - err = fetchGitHubBranch(FS, lang, owner, repo, "") - } else { - err = fetchGitHubTag(FS, lang, owner, repo, tag) - } - if err != nil { - return fmt.Errorf("While fetching from github\n%w\n", err) - } - - err = Write(lang, "github.com", owner, repo, tag, FS) - if err != nil { - return fmt.Errorf("While writing to cache\n%w\n", err) - } + remote, owner, repo := parseModURL(mod) + tag := ver - return nil -} -func fetchGitHubBranch(FS billy.Filesystem, lang, owner, repo, branch string) error { - client, err := github.NewClient() - if err != nil { - return err - } + // TODO, how to deal with self-hosted / enterprise repos? - if branch == "" { - r, err := github.GetRepo(client, owner, repo) - if err != nil { - return err + switch remote { + case "github.com": + if err := github.Fetch(FS, owner, repo, tag, private); err != nil { + return fmt.Errorf("While fetching from github\n%w\n", err) } - branch = *r.DefaultBranch - - fmt.Printf("%#+v\n", *r) - } - - // fmt.Println("Fetch github BRANCH", lang, owner, repo, branch) - - zReader, err := github.FetchBranchZip(client, owner, repo, branch) - if err != nil { - return fmt.Errorf("While fetching branch zipfile for %s/%s@%s\n%w\n", owner, repo, branch, err) - } - err = yagu.BillyLoadFromZip(zReader, FS, true) - if err != nil { - return fmt.Errorf("While reading branch zipfile\n%w\n", err) - } - - return nil -} -func fetchGitHubTag(FS billy.Filesystem, lang, owner, repo, tag string) error { - // fmt.Println("Fetch github TAG", lang, owner, repo, tag) - client, err := github.NewClient() - if err != nil { - return err - } - - tags, err := github.GetTags(client, owner, repo) - if err != nil { - return err - } + case "gitlab.com": + if err := gitlab.Fetch(FS, owner, repo, tag, private); err != nil { + return fmt.Errorf("While fetching from gitlab\n%w\n", err) + } - // The tag we are looking for - var T *googithub.RepositoryTag - for _, t := range tags { - if tag != "" && tag == *t.Name { - T = t - // fmt.Printf("FOUND %v\n", *t.Name) + case "bitbucket.org": + if err := bbc.Fetch(FS, owner, repo, tag, private); err != nil { + return fmt.Errorf("While fetching from bitbucket\n%w\n", err) } - } - if T == nil { - return fmt.Errorf("Did not find tag %q for 'https://github.com/%s/%s' @%s", tag, owner, repo, tag) - } - zReader, err := github.FetchTagZip(client, T) - if err != nil { - return fmt.Errorf("While fetching tag zipfile\n%w\n", err) + default: + if err := git.Fetch(FS, remote, owner, repo, tag, private); err != nil { + return fmt.Errorf("While fetching from git\n%w\n", err) + } } - err = yagu.BillyLoadFromZip(zReader, FS, true) - if err != nil { - return fmt.Errorf("While reading tag zipfile\n%w\n", err) + if err := Write(lang, remote, owner, repo, tag, FS); err != nil { + return fmt.Errorf("While writing to cache\n%w\n", err) } return nil diff --git a/lib/mod/cache/write.go b/lib/mod/cache/write.go index 0f519dce4..3adbd267d 100644 --- a/lib/mod/cache/write.go +++ b/lib/mod/cache/write.go @@ -1,7 +1,6 @@ package cache import ( - "fmt" "path/filepath" "github.com/go-git/go-billy/v5" @@ -22,7 +21,6 @@ func Outdir(lang, remote, owner, repo, tag string) string { } func Write(lang, remote, owner, repo, tag string, FS billy.Filesystem) error { - fmt.Printf("Saving %s mod %s/%s/%s@%s\n", lang, remote, owner, repo, tag) outdir := Outdir(lang, remote, owner, repo, tag) err := yagu.Mkdir(outdir) if err != nil { diff --git a/lib/mod/cli_test.go b/lib/mod/cli_test.go index 0bbc30dce..c73163113 100644 --- a/lib/mod/cli_test.go +++ b/lib/mod/cli_test.go @@ -1,6 +1,7 @@ package mod_test import ( + "fmt" "os" "testing" @@ -9,8 +10,14 @@ import ( ) func envSetup(env *runtime.Env) error { - if token := os.Getenv("GITHUB_TOKEN"); token != "" { - env.Vars = append(env.Vars, "GITHUB_TOKEN="+token) + vars := []string{ + "GITHUB_TOKEN", + "GITLAB_TOKEN", + } + for _, v := range vars { + if val := os.Getenv(v); val != "" { + env.Vars = append(env.Vars, fmt.Sprintf("%s=%s", v, val)) + } } env.Vars = append(env.Vars, "HOF_TELEMETRY_DISABLED=1") return nil @@ -35,3 +42,34 @@ func TestModBugs(t *testing.T) { WorkdirRoot: ".workdir/bugs", }) } + +func TestModAuthdApikeysTests(t *testing.T) { + yagu.Mkdir(".workdir/authd/apikeys") + runtime.Run(t, runtime.Params{ + Setup: envSetup, + Dir: "testdata/authd/apikeys", + Glob: "*.txt", + WorkdirRoot: ".workdir/authd/apikeys", + }) +} + +func TestModAuthdSshconfigTests(t *testing.T) { + yagu.Mkdir(".workdir/authd/sshconfig") + runtime.Run(t, runtime.Params{ + Setup: envSetup, + Dir: "testdata/authd/sshconfig", + Glob: "*.txt", + WorkdirRoot: ".workdir/authd/sshconfig", + }) +} + +//func TestModAuthdSshkeyTests(t *testing.T) { + //yagu.Mkdir(".workdir/authd/sshkey") + //runtime.Run(t, runtime.Params{ + //Setup: envSetup, + //Dir: "testdata/authd/sshkey", + //Glob: "*.txt", + //WorkdirRoot: ".workdir/authd/sshkey", + //}) +//} + diff --git a/lib/mod/hack.go b/lib/mod/hack.go index c250acfc1..882362877 100644 --- a/lib/mod/hack.go +++ b/lib/mod/hack.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "strings" - googithub "github.com/google/go-github/v30/github" + googithub "github.com/google/go-github/v38/github" "github.com/go-git/go-billy/v5/memfs" "github.com/hofstadter-io/hof/lib/mod/cache" @@ -43,7 +43,7 @@ func Hack(lang string, args []string) error { // Fetch and write to cache if tag found if T != nil { - zReader, err := github.FetchTagZip(client, T) + zReader, err := github.FetchTagZip(T) if err != nil { return fmt.Errorf("While fetching zipfile\n%w\n", err) } diff --git a/lib/mod/testdata/authd/apikeys/vendor__priv_bitbucket.txt b/lib/mod/testdata/authd/apikeys/vendor__priv_bitbucket.txt new file mode 100644 index 000000000..1d77051ba --- /dev/null +++ b/lib/mod/testdata/authd/apikeys/vendor__priv_bitbucket.txt @@ -0,0 +1,15 @@ +# hof mod vendor - with deps +exec hof mod vendor + +-- cue.mods -- +module github.com/test/priv-bitbucket + +cue v0.4.0 + +require ( + bitbucket.org/hofstadter/hofmod-test-priv-bitbucket v0.0.1 +) +-- cue.mod/module.cue -- +module: "github.com/test/priv-bitbucket" + +-- dummy_end -- diff --git a/lib/mod/testdata/authd/apikeys/vendor__priv_github.txt b/lib/mod/testdata/authd/apikeys/vendor__priv_github.txt new file mode 100644 index 000000000..99841af92 --- /dev/null +++ b/lib/mod/testdata/authd/apikeys/vendor__priv_github.txt @@ -0,0 +1,15 @@ +# hof mod vendor - with deps +exec hof mod vendor + +-- cue.mods -- +module github.com/test/priv-github + +cue v0.4.0 + +require ( + github.com/hofstadter-io/hofmod-test-priv-github v0.0.1 +) +-- cue.mod/module.cue -- +module: "github.com/test/priv-github" + +-- dummy_end -- diff --git a/lib/mod/testdata/authd/apikeys/vendor__priv_github_fail.txt b/lib/mod/testdata/authd/apikeys/vendor__priv_github_fail.txt new file mode 100644 index 000000000..f5577ba30 --- /dev/null +++ b/lib/mod/testdata/authd/apikeys/vendor__priv_github_fail.txt @@ -0,0 +1,16 @@ +# hof mod vendor - with deps +env GITHUB_TOKEN="thistokendoesnotexist" +! exec hof mod vendor + +-- cue.mods -- +module github.com/test/priv-github + +cue v0.4.0 + +require ( + github.com/hofstadter-io/hofmod-test-priv-github v0.0.1 +) +-- cue.mod/module.cue -- +module: "github.com/test/priv-github" + +-- dummy_end -- diff --git a/lib/mod/testdata/authd/apikeys/vendor__priv_gitlab.txt b/lib/mod/testdata/authd/apikeys/vendor__priv_gitlab.txt new file mode 100644 index 000000000..a5f2931d9 --- /dev/null +++ b/lib/mod/testdata/authd/apikeys/vendor__priv_gitlab.txt @@ -0,0 +1,15 @@ +# hof mod vendor - with deps +exec hof mod vendor + +-- cue.mods -- +module github.com/test/priv-gitlab + +cue v0.4.0 + +require ( + gitlab.com/_hofstadter/hofmod-test-priv-gitlab v0.0.1 +) +-- cue.mod/module.cue -- +module: "github.com/test/priv-gitlab" + +-- dummy_end -- diff --git a/lib/mod/testdata/authd/sshconfig/vendor__priv_github.txt b/lib/mod/testdata/authd/sshconfig/vendor__priv_github.txt new file mode 100644 index 000000000..afd997682 --- /dev/null +++ b/lib/mod/testdata/authd/sshconfig/vendor__priv_github.txt @@ -0,0 +1,16 @@ +# hof mod vendor - with deps +env GITHUB_TOKEN="" +exec hof mod vendor + +-- cue.mods -- +module github.com/test/priv-github + +cue v0.4.0 + +require ( + github.com/hofstadter-io/hofmod-test-priv-github v0.0.1 +) +-- cue.mod/module.cue -- +module: "github.com/test/priv-github" + +-- dummy_end -- diff --git a/lib/mod/testdata/authd/sshkey/vendor__priv_all.txt b/lib/mod/testdata/authd/sshkey/vendor__priv_all.txt new file mode 100644 index 000000000..c05942df5 --- /dev/null +++ b/lib/mod/testdata/authd/sshkey/vendor__priv_all.txt @@ -0,0 +1,31 @@ +# hof mod vendor - with deps + +env CUEPRIVATE=github.com,gitlab.com,bitbucket.org + +## blankout token vars for this test +env GITHUB_TOKEN="" +env GITLAB_TOKEN="" +env BITBUCKET_USERNAME="" +env BITBUCKET_PASSWORD="" + +## setup default sshkey auth +mkdir $HOME/.ssh +exec bash -c 'echo $HOFMOD_SSHKEY > $HOME/.ssh/id_rsa' + +## run cmd +exec hof mod vendor + +-- cue.mods -- +module github.com/test/priv-all + +cue v0.4.0 + +require ( + bitbucket.org/hofstadter/hofmod-test-priv-bitbucket v0.0.1 + github.com/hofstadter-io/hofmod-test-priv-github v0.0.1 + gitlab.com/_hofstadter/hofmod-test-priv-gitlab v0.0.1 +) +-- cue.mod/module.cue -- +module: "github.com/test/priv-all" + +-- dummy_end -- diff --git a/lib/test/api.go b/lib/test/api.go index ab62140d8..d4b1638bc 100644 --- a/lib/test/api.go +++ b/lib/test/api.go @@ -17,7 +17,7 @@ import ( const HTTP2_GOAWAY_CHECK = "http2: server sent GOAWAY and closed the connection" func RunAPI(T *Tester, verbose int) (err error) { - fmt.Println("api:", T.Name) + // fmt.Println("api:", T.Name) // make sure we resolve references and unifications val := T.Value.Eval() @@ -49,7 +49,15 @@ func runCase(T *Tester, verbose int, val cue.Value) (err error) { return err } - err = checkResponse(T, verbose, actual, expected) + + fail := val.LookupPath(cue.ParsePath("fail")) + failVal, err := fail.Bool() + if err != nil { + // likely not found + failVal = false + } + + err = checkResponse(T, verbose, actual, expected, failVal) return err } @@ -66,11 +74,12 @@ func buildRequest(T *Tester, verbose int, val cue.Value) (R *gorequest.SuperAgen } host := req.LookupPath(cue.ParsePath("host")) - path := req.LookupPath(cue.ParsePath("path")) hostStr, err := host.String() if err != nil { return } + + path := req.LookupPath(cue.ParsePath("path")) pathStr, err := path.String() if err != nil { return @@ -203,12 +212,12 @@ func makeRequest(T *Tester, verbose int, R *gorequest.SuperAgent) (gorequest.Res if verbose > 0 { fmt.Println(body) } - fmt.Println(body) + // fmt.Println(body) return resp, nil } -func checkResponse(T *Tester, verbose int, actual gorequest.Response, expect cue.Value) (err error) { +func checkResponse(T *Tester, verbose int, actual gorequest.Response, expect cue.Value, expectFail bool) (err error) { expect = expect.Eval() S, err := expect.Struct() @@ -220,7 +229,7 @@ func checkResponse(T *Tester, verbose int, actual gorequest.Response, expect cue label := iter.Label() value := iter.Value() - fmt.Println("checking:", label) + // fmt.Println("checking:", label) switch label { case "status": @@ -250,14 +259,17 @@ func checkResponse(T *Tester, verbose int, actual gorequest.Response, expect cue // TODO: bi-directional subsume to check for equality? result := value.Unify(V) if result.Err() != nil { - return result.Err() + if !expectFail { + return result.Err() + } } - fmt.Println("result: ", result) + // fmt.Println("result: ", result) err = result.Validate() if err != nil { - fmt.Println(value) - - return err + if !expectFail { + fmt.Println(value) + return err + } } diff --git a/lib/yagu/git.go b/lib/yagu/git.go index 2fab9b99e..4fb166c35 100644 --- a/lib/yagu/git.go +++ b/lib/yagu/git.go @@ -80,13 +80,6 @@ func SetupGitAuth(srcUrl, srcVer string, co *git.CloneOptions) error { } return nil } - if os.Getenv("GITHUB_APIKEY") != "" { - co.Auth = &http.BasicAuth{ - Username: "github-apikey", // yes, this can be anything except an empty string - Password: os.Getenv("GITHUB_APIKEY"), - } - return nil - } } // GitLab variations @@ -97,12 +90,6 @@ func SetupGitAuth(srcUrl, srcVer string, co *git.CloneOptions) error { } return nil } - if os.Getenv("GITLAB_APIKEY") != "" { - co.Auth = &http.TokenAuth{ - Token: os.Getenv("GITLAB_APIKEY"), - } - return nil - } if os.Getenv("GITLAB_USERNAME") != "" && os.Getenv("GITLAB_PASSWORD") != "" { co.Auth = &http.BasicAuth{ Username: os.Getenv("GITLAB_USERNAME"), @@ -120,12 +107,6 @@ func SetupGitAuth(srcUrl, srcVer string, co *git.CloneOptions) error { } return nil } - if os.Getenv("BITBUCKET_APIKEY") != "" { - co.Auth = &http.TokenAuth{ - Token: os.Getenv("BITBUCKET_APIKEY"), - } - return nil - } if os.Getenv("BITBUCKET_USERNAME") != "" && os.Getenv("BITBUCKET_PASSWORD") != "" { co.Auth = &http.BasicAuth{ Username: os.Getenv("BITBUCKET_USERNAME"), diff --git a/lib/yagu/repos/git/netrc.go b/lib/yagu/netrc.go similarity index 99% rename from lib/yagu/repos/git/netrc.go rename to lib/yagu/netrc.go index 7e471f87d..bce30d5f1 100644 --- a/lib/yagu/repos/git/netrc.go +++ b/lib/yagu/netrc.go @@ -1,4 +1,4 @@ -package git +package yagu import ( "fmt" diff --git a/lib/yagu/repos/git/netrc_test.go b/lib/yagu/netrc_test.go similarity index 98% rename from lib/yagu/repos/git/netrc_test.go rename to lib/yagu/netrc_test.go index 0b4637c63..9254d587e 100644 --- a/lib/yagu/repos/git/netrc_test.go +++ b/lib/yagu/netrc_test.go @@ -1,4 +1,4 @@ -package git +package yagu import ( "reflect" diff --git a/lib/yagu/repos/bbc/client.go b/lib/yagu/repos/bbc/client.go new file mode 100644 index 000000000..8a5340cf3 --- /dev/null +++ b/lib/yagu/repos/bbc/client.go @@ -0,0 +1,27 @@ +package bbc + +import ( + "os" + + "github.com/ktrysmt/go-bitbucket" +) + +const TokenEnv = "BITBUCKET_TOKEN" +const UsernameEnv = "BITBUCKET_USERNAME" +const PasswordEnv = "BITBUCKET_PASSWORD" + +func NewClient() (client *bitbucket.Client, err error) { + + // if token is empty, it means the same things as unauthenticated setup + if token := os.Getenv(TokenEnv); token != "" { + token := os.Getenv(TokenEnv) + client = bitbucket.NewOAuthbearerToken(token) + } else if password := os.Getenv(PasswordEnv); password != "" { + username := os.Getenv(UsernameEnv) + client = bitbucket.NewBasicAuth(username, password) + } else { + // if token is empty, it means the same things as unauthenticated setup + client = bitbucket.NewOAuthbearerToken("") + } + return client, err +} diff --git a/lib/yagu/repos/bbc/fetch.go b/lib/yagu/repos/bbc/fetch.go new file mode 100644 index 000000000..ba184f9c6 --- /dev/null +++ b/lib/yagu/repos/bbc/fetch.go @@ -0,0 +1,156 @@ +package bbc + +import ( + "archive/zip" + "bytes" + "fmt" + "os" + "strings" + + "github.com/go-git/go-billy/v5" + "github.com/ktrysmt/go-bitbucket" + "github.com/parnurzeal/gorequest" + + "github.com/hofstadter-io/hof/lib/yagu" + "github.com/hofstadter-io/hof/lib/yagu/repos/git" +) + +func Fetch(FS billy.Filesystem, owner, repo, tag string, private bool) (error) { + + // If private, and no token auth, try git protocol + // need to catch auth errors and suggest how to setup + if private && os.Getenv(TokenEnv) == "" { + fmt.Println("bitbucket git fallback") + return git.Fetch(FS, "bitbucket.org", owner, repo, tag, private) + } + + client, err := NewClient() + if err != nil { + return err + } + + var zReader *zip.Reader + + if tag == "v0.0.0" { + r, err := GetRepo(client, owner, repo) + if err != nil { + return err + } + + zReader, err = FetchBranchZip(owner, repo, r.Mainbranch.Name) + if err != nil { + return fmt.Errorf("While fetching branch zipfile for %s/%s@%s\n%w\n", owner, repo, r.Mainbranch.Name, err) + } + + } else { + tags, err := GetTags(client, owner, repo) + if err != nil { + return err + } + + // The tag we are looking for + var T *bitbucket.RepositoryTag + for _, t := range tags { + if tag != "" && tag == t.Name { + T = &t + // fmt.Printf("FOUND %v\n", *t.Name) + } + } + + if T == nil { + return fmt.Errorf("Did not find tag %q for 'https://bitbucket.org/%s/%s' @%s", tag, owner, repo, tag) + } + + zReader, err = FetchTagZip(owner, repo, T.Name) + if err != nil { + return fmt.Errorf("While fetching tag zipfile\n%w\n", err) + } + } + + if err != nil { + return fmt.Errorf("While fetching from bitbucket\n%w\n", err) + } + + if err := yagu.BillyLoadFromZip(zReader, FS, true); err != nil { + return fmt.Errorf("While reading zipfile\n%w\n", err) + } + + return nil +} + +func FetchTagZip(owner, repo, ver string) (*zip.Reader, error) { + + // url := *tag.ZipballURL + url := fmt.Sprintf("https://bitbucket.org/%s/%s/get/%s.zip", owner, repo, ver) + + req := gorequest.New().Get(url) + + // TODO, process auth logic here better, maybe find a way to DRY + if token := os.Getenv(TokenEnv); token != "" { + req.Header.Set("Authorization", "Bearer" + token) + } else if password := os.Getenv(PasswordEnv); password != "" { + username := os.Getenv(UsernameEnv) + req.SetBasicAuth(username, password) + } + + resp, data, errs := req.EndBytes() + + check := "http2: server sent GOAWAY and closed the connection" + if len(errs) != 0 && !strings.Contains(errs[0].Error(), check) { + fmt.Println("errs:", errs) + fmt.Println("resp:", resp) + fmt.Println("body:", len(data)) + return nil, errs[0] + } + + if len(errs) != 0 || resp.StatusCode >= 500 { + return nil, fmt.Errorf("Internal Error: " + string(resp.StatusCode)) + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("Bad Request: %v\n%v\n", resp.StatusCode, errs) + } + + r := bytes.NewReader(data) + + zfile, err := zip.NewReader(r, int64(len(data))) + + return zfile, err +} + +func FetchBranchZip(owner, repo, branch string) (*zip.Reader, error) { + + url := fmt.Sprintf("https://bitbucket.org/%s/%s/get/%s.zip", owner, repo, branch) + + req := gorequest.New().Get(url) + + // TODO, process auth logic here better, maybe find a way to DRY + if token := os.Getenv(TokenEnv); token != "" { + req.Header.Set("Authorization", "Bearer" + token) + } else if password := os.Getenv(PasswordEnv); password != "" { + username := os.Getenv(UsernameEnv) + req.SetBasicAuth(username, password) + } + + resp, data, errs := req.EndBytes() + + check := "http2: server sent GOAWAY and closed the connection" + if len(errs) != 0 && !strings.Contains(errs[0].Error(), check) { + fmt.Println("errs:", errs) + fmt.Println("resp:", resp) + fmt.Println("body:", len(data)) + return nil, errs[0] + } + + if len(errs) != 0 || resp.StatusCode >= 500 { + return nil, fmt.Errorf("Internal Error: " + string(resp.StatusCode)) + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("Bad Request: " + string(resp.StatusCode)) + } + + r := bytes.NewReader(data) + + zfile, err := zip.NewReader(r, int64(len(data))) + + return zfile, err +} diff --git a/lib/yagu/repos/bbc/info.go b/lib/yagu/repos/bbc/info.go new file mode 100644 index 000000000..42ff982fb --- /dev/null +++ b/lib/yagu/repos/bbc/info.go @@ -0,0 +1,71 @@ +package bbc + +import ( + "fmt" + "strings" + + "github.com/ktrysmt/go-bitbucket" +) + +func GetTagsSplit(client *bitbucket.Client, module string) ([]bitbucket.RepositoryTag, error) { + flds := strings.SplitN(module, "/", 1) + domain, rest := flds[0], flds[1] + + if domain != "bitbucket.org" { + return nil, fmt.Errorf("Bitbucket Tags Fetch called with non 'bitbucket.org' domain %q. Custom domains are a TODO", module) + } + + flds = strings.Split(rest, "/") + owner, repo := flds[0], flds[1] + + ro := &bitbucket.RepositoryTagOptions{ + Owner: owner, + RepoSlug: repo, + } + tags, err := client.Repositories.Repository.ListTags(ro) + if err != nil { + return nil, err + } + return tags.Tags, nil +} + +func GetRepo(client *bitbucket.Client, owner, repo string) (*bitbucket.Repository, error) { + ro := &bitbucket.RepositoryOptions{ + Owner: owner, + RepoSlug: repo, + } + r, err := client.Repositories.Repository.Get(ro) + return r, err +} + +func GetBranch(client *bitbucket.Client, owner, repo, branch string) (*bitbucket.RepositoryBranch, error) { + ro := &bitbucket.RepositoryBranchOptions{ + Owner: owner, + RepoSlug: repo, + } + b, err := client.Repositories.Repository.GetBranch(ro) + return b, err +} + +func GetBranches(client *bitbucket.Client, owner, repo, branch string) ([]bitbucket.RepositoryBranch, error) { + ro := &bitbucket.RepositoryBranchOptions{ + Owner: owner, + RepoSlug: repo, + } + bs, err := client.Repositories.Repository.ListBranches(ro) + return bs.Branches, err +} + +func GetTags(client *bitbucket.Client, owner, repo string) ([]bitbucket.RepositoryTag, error) { + _, err := GetRepo(client, owner, repo) + if err != nil { + return nil, err + } + ro := &bitbucket.RepositoryTagOptions{ + Owner: owner, + RepoSlug: repo, + } + tags, err := client.Repositories.Repository.ListTags(ro) + return tags.Tags, err +} + diff --git a/lib/yagu/repos/git/fetch.go b/lib/yagu/repos/git/fetch.go index fc1a3181b..5b2c52816 100644 --- a/lib/yagu/repos/git/fetch.go +++ b/lib/yagu/repos/git/fetch.go @@ -14,8 +14,15 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" + + "github.com/hofstadter-io/hof/lib/yagu" ) +// TODO, this file has inconsistency of auth creds adding +// between the functions, this should be cleaned up +// taking note that we want to create more consistency +// across the various repo types + func NewRemote(srcUrl string) (*GitRepo, error) { rc := &config.RemoteConfig{ @@ -93,7 +100,7 @@ func CloneRepoRef(srcUrl string, ref *plumbing.Reference) (*GitRepo, error) { // FetchGit clone the repository inside FS. // If private flag is set, it will look for netrc credentials, fallbacking to SSH -func FetchGit(FS billy.Filesystem, remote, owner, repo, tag string, private bool) error { +func Fetch(FS billy.Filesystem, remote, owner, repo, tag string, private bool) error { srcRepo := path.Join(owner, repo) gco := &gogit.CloneOptions{ URL: fmt.Sprintf("https://%s/%s", remote, srcRepo), @@ -106,12 +113,12 @@ func FetchGit(FS billy.Filesystem, remote, owner, repo, tag string, private bool } if private { - if netrc, err := NetrcCredentials(remote); err == nil { + if netrc, err := yagu.NetrcCredentials(remote); err == nil { gco.Auth = &http.BasicAuth{ Username: netrc.Login, Password: netrc.Password, } - } else if ssh, err := SSHCredentials(remote); err == nil { + } else if ssh, err := yagu.SSHCredentials(remote); err == nil { gco.Auth = ssh.Keys gco.URL = fmt.Sprintf("%s@%s:%s", ssh.User, remote, srcRepo) } else { diff --git a/lib/yagu/repos/git/index.go b/lib/yagu/repos/git/index.go index b7a68fa7e..d15f7ac41 100644 --- a/lib/yagu/repos/git/index.go +++ b/lib/yagu/repos/git/index.go @@ -11,7 +11,6 @@ import ( // given a git url, and a req (required) version // returns the minReference, allRefferences, and error func IndexGitRemote(url, req string) (*plumbing.Reference, []*plumbing.Reference, error) { - fmt.Println("indexing:", url, req) if !semver.IsValid(req) { return nil, nil, fmt.Errorf("Invalid SemVer v2 %q", req) diff --git a/lib/yagu/repos/git/ssh.go b/lib/yagu/repos/git/ssh.go deleted file mode 100644 index aeb1eac85..000000000 --- a/lib/yagu/repos/git/ssh.go +++ /dev/null @@ -1,37 +0,0 @@ -package git - -import ( - "os" - "strings" - - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/kevinburke/ssh_config" -) - -type SSHMachine struct { - User string - Keys *ssh.PublicKeys -} - -func SSHCredentials(machine string) (SSHMachine, error) { - pk, err := ssh_config.GetStrict(machine, "IdentityFile") - if err != nil { - return SSHMachine{}, err - } - if strings.HasPrefix(pk, "~") { - if hdir, err := os.UserHomeDir(); err == nil { - pk = strings.Replace(pk, "~", hdir, 1) - } - } - usr := ssh_config.Get(machine, "User") - if usr == "" { - usr = "git" - } - - pks, err := ssh.NewPublicKeysFromFile(usr, pk, "") - if err != nil { - return SSHMachine{}, err - } - - return SSHMachine{usr, pks}, nil -} diff --git a/lib/yagu/repos/github/client.go b/lib/yagu/repos/github/client.go index fe083c325..5a969f858 100644 --- a/lib/yagu/repos/github/client.go +++ b/lib/yagu/repos/github/client.go @@ -6,7 +6,7 @@ import ( "golang.org/x/oauth2" - "github.com/google/go-github/v30/github" + "github.com/google/go-github/v38/github" ) const TokenEnv = "GITHUB_TOKEN" @@ -20,7 +20,6 @@ func NewClient() (client *github.Client, err error) { ) tc := oauth2.NewClient(ctx, ts) client = github.NewClient(tc) - } else { client = github.NewClient(nil) } diff --git a/lib/yagu/repos/github/fetch.go b/lib/yagu/repos/github/fetch.go new file mode 100644 index 000000000..1598158f2 --- /dev/null +++ b/lib/yagu/repos/github/fetch.go @@ -0,0 +1,148 @@ +package github + +import ( + "archive/zip" + "bytes" + "fmt" + "os" + "strings" + + "github.com/go-git/go-billy/v5" + "github.com/google/go-github/v38/github" + "github.com/parnurzeal/gorequest" + + "github.com/hofstadter-io/hof/lib/yagu" + "github.com/hofstadter-io/hof/lib/yagu/repos/git" +) + +func Fetch(FS billy.Filesystem, owner, repo, tag string, private bool) (error) { + + // If private, and no token auth, try git protocol + // need to catch auth errors and suggest how to setup + if private && os.Getenv(TokenEnv) == "" { + fmt.Println("github git fallback") + return git.Fetch(FS, "github.com", owner, repo, tag, private) + } + + client, err := NewClient() + if err != nil { + return err + } + + var zReader *zip.Reader + + if tag == "v0.0.0" { + r, err := GetRepo(client, owner, repo) + if err != nil { + return err + } + + zReader, err = FetchBranchZip(owner, repo, *r.DefaultBranch) + if err != nil { + return fmt.Errorf("While fetching branch zipfile for %s/%s@%s\n%w\n", owner, repo, *r.DefaultBranch, err) + } + + } else { + tags, err := GetTags(client, owner, repo) + if err != nil { + return err + } + + // The tag we are looking for + var T *github.RepositoryTag + for _, t := range tags { + if tag != "" && tag == *t.Name { + T = t + } + } + + if T == nil { + return fmt.Errorf("Did not find tag %q for 'https://github.com/%s/%s' @%s", tag, owner, repo, tag) + } + + zReader, err = FetchTagZip(T) + if err != nil { + return fmt.Errorf("While fetching tag zipfile\n%w\n", err) + } + } + + if err != nil { + return fmt.Errorf("While fetching from github\n%w\n", err) + } + + if err := yagu.BillyLoadFromZip(zReader, FS, true); err != nil { + return fmt.Errorf("While reading zipfile\n%w\n", err) + } + + return nil +} + +func FetchTagZip(tag *github.RepositoryTag) (*zip.Reader, error) { + + url := *tag.ZipballURL + + req := gorequest.New().Get(url) + + // TODO, process auth logic here better, maybe find a way to DRY + if token := os.Getenv(TokenEnv); token != "" { + req.SetBasicAuth("github-token", token) + } + + resp, data, errs := req.EndBytes() + + check := "http2: server sent GOAWAY and closed the connection" + if len(errs) != 0 && !strings.Contains(errs[0].Error(), check) { + fmt.Println("errs:", errs) + fmt.Println("resp:", resp) + fmt.Println("body:", len(data)) + return nil, errs[0] + } + + if len(errs) != 0 || resp.StatusCode >= 500 { + return nil, fmt.Errorf("Internal Error: " + string(resp.StatusCode)) + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("Bad Request: %v\n%v\n", resp.StatusCode, errs) + } + + r := bytes.NewReader(data) + + zfile, err := zip.NewReader(r, int64(len(data))) + + return zfile, err +} + +func FetchBranchZip(owner, repo, branch string) (*zip.Reader, error) { + + url := fmt.Sprintf("https://github.com/%s/%s/archive/refs/heads/%s.zip", owner, repo, branch) + + req := gorequest.New().Get(url) + + // TODO, process auth logic here better, maybe find a way to DRY + if token := os.Getenv(TokenEnv); token != "" { + req.SetBasicAuth("github-token", token) + } + + resp, data, errs := req.EndBytes() + + check := "http2: server sent GOAWAY and closed the connection" + if len(errs) != 0 && !strings.Contains(errs[0].Error(), check) { + fmt.Println("errs:", errs) + fmt.Println("resp:", resp) + fmt.Println("body:", len(data)) + return nil, errs[0] + } + + if len(errs) != 0 || resp.StatusCode >= 500 { + return nil, fmt.Errorf("Internal Error: " + string(resp.StatusCode)) + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("Bad Request: " + string(resp.StatusCode)) + } + + r := bytes.NewReader(data) + + zfile, err := zip.NewReader(r, int64(len(data))) + + return zfile, err +} diff --git a/lib/yagu/repos/github/info.go b/lib/yagu/repos/github/info.go new file mode 100644 index 000000000..f2f5c69df --- /dev/null +++ b/lib/yagu/repos/github/info.go @@ -0,0 +1,47 @@ +package github + +import ( + "context" + "fmt" + "strings" + + "github.com/google/go-github/v38/github" +) + +func GetTagsSplit(client *github.Client, module string) ([]*github.RepositoryTag, error) { + flds := strings.SplitN(module, "/", 1) + domain, rest := flds[0], flds[1] + + if domain != "github.com" { + return nil, fmt.Errorf("Github Tags Fetch called with non 'github.com' domain %q. Custom domains are a TODO", module) + } + + flds = strings.Split(rest, "/") + owner, repo := flds[0], flds[1] + tags, _, err := client.Repositories.ListTags(context.Background(), owner, repo, nil) + if err != nil { + return nil, err + } + return tags, nil +} + +func GetRepo(client *github.Client, owner, repo string) (*github.Repository, error) { + r, _, err := client.Repositories.Get(context.Background(), owner, repo) + return r, err +} + +func GetBranch(client *github.Client, owner, repo, branch string) (*github.Branch, error) { + b, _, err := client.Repositories.GetBranch(context.Background(), owner, repo, branch, true) + return b, err +} + +func GetBranches(client *github.Client, owner, repo, branch string) ([]*github.Branch, error) { + bs, _, err := client.Repositories.ListBranches(context.Background(), owner, repo, nil) + return bs, err +} + +func GetTags(client *github.Client, owner, repo string) ([]*github.RepositoryTag, error) { + tags, _, err := client.Repositories.ListTags(context.Background(), owner, repo, nil) + return tags, err +} + diff --git a/lib/yagu/repos/github/tags.go b/lib/yagu/repos/github/tags.go deleted file mode 100644 index feb4f93d0..000000000 --- a/lib/yagu/repos/github/tags.go +++ /dev/null @@ -1,107 +0,0 @@ -package github - -import ( - "archive/zip" - "bytes" - "context" - "fmt" - "strings" - - "github.com/google/go-github/v30/github" - "github.com/parnurzeal/gorequest" -) - -func GetTagsSplit(client *github.Client, module string) ([]*github.RepositoryTag, error) { - flds := strings.SplitN(module, "/", 1) - domain, rest := flds[0], flds[1] - - if domain != "github.com" { - return nil, fmt.Errorf("Github Tags Fetch called with non 'github.com' domain %q", module) - } - - flds = strings.Split(rest, "/") - owner, repo := flds[0], flds[1] - tags, _, err := client.Repositories.ListTags(context.Background(), owner, repo, nil) - if err != nil { - return nil, err - } - return tags, nil -} - -func GetRepo(client *github.Client, owner, repo string) (*github.Repository, error) { - r, _, err := client.Repositories.Get(context.Background(), owner, repo) - return r, err -} - -func GetBranch(client *github.Client, owner, repo, branch string) (*github.Branch, error) { - b, _, err := client.Repositories.GetBranch(context.Background(), owner, repo, branch) - return b, err -} - -func GetBranches(client *github.Client, owner, repo, branch string) ([]*github.Branch, error) { - bs, _, err := client.Repositories.ListBranches(context.Background(), owner, repo, nil) - return bs, err -} - -func GetTags(client *github.Client, owner, repo string) ([]*github.RepositoryTag, error) { - tags, _, err := client.Repositories.ListTags(context.Background(), owner, repo, nil) - return tags, err -} - -func FetchTagZip(client *github.Client, tag *github.RepositoryTag) (*zip.Reader, error) { - - url := *tag.ZipballURL - - req := gorequest.New().Get(url) - resp, data, errs := req.EndBytes() - - check := "http2: server sent GOAWAY and closed the connection" - if len(errs) != 0 && !strings.Contains(errs[0].Error(), check) { - fmt.Println("errs:", errs) - fmt.Println("resp:", resp) - fmt.Println("body:", len(data)) - return nil, errs[0] - } - - if len(errs) != 0 || resp.StatusCode >= 500 { - return nil, fmt.Errorf("Internal Error: " + string(resp.StatusCode)) - } - if resp.StatusCode >= 400 { - return nil, fmt.Errorf("Bad Request: " + string(resp.StatusCode)) - } - - r := bytes.NewReader(data) - - zfile, err := zip.NewReader(r, int64(len(data))) - - return zfile, err -} - -func FetchBranchZip(client *github.Client, owner, repo, branch string) (*zip.Reader, error) { - - url := fmt.Sprintf("https://github.com/%s/%s/archive/refs/heads/%s.zip", owner, repo, branch) - - req := gorequest.New().Get(url) - resp, data, errs := req.EndBytes() - - check := "http2: server sent GOAWAY and closed the connection" - if len(errs) != 0 && !strings.Contains(errs[0].Error(), check) { - fmt.Println("errs:", errs) - fmt.Println("resp:", resp) - fmt.Println("body:", len(data)) - return nil, errs[0] - } - - if len(errs) != 0 || resp.StatusCode >= 500 { - return nil, fmt.Errorf("Internal Error: " + string(resp.StatusCode)) - } - if resp.StatusCode >= 400 { - return nil, fmt.Errorf("Bad Request: " + string(resp.StatusCode)) - } - - r := bytes.NewReader(data) - - zfile, err := zip.NewReader(r, int64(len(data))) - - return zfile, err -} diff --git a/lib/yagu/repos/gitlab/client.go b/lib/yagu/repos/gitlab/client.go index 157cc6783..0df560503 100644 --- a/lib/yagu/repos/gitlab/client.go +++ b/lib/yagu/repos/gitlab/client.go @@ -1,6 +1,7 @@ package gitlab import ( + "fmt" "os" "github.com/xanzy/go-gitlab" @@ -8,6 +9,15 @@ import ( const TokenEnv = "GITLAB_TOKEN" -func NewClient() (client *gitlab.Client, err error) { - return gitlab.NewClient(os.Getenv(TokenEnv)) +func NewClient(private bool) (client *gitlab.Client, err error) { + // TODO, there are multiple NewClient<> methods + // how to determine which to use and the inputs to them + // https://pkg.go.dev/github.com/xanzy/go-gitlab#NewOAuthClient + // Noting also that we prefer auth over non-auth for API rate limits + + token := os.Getenv(TokenEnv) + if private && token == "" { + return nil, fmt.Errorf("Private module requested and no auth token available for %s", TokenEnv) + } + return gitlab.NewClient(token) } diff --git a/lib/yagu/repos/gitlab/fetch.go b/lib/yagu/repos/gitlab/fetch.go index 7994dac04..78b8add41 100644 --- a/lib/yagu/repos/gitlab/fetch.go +++ b/lib/yagu/repos/gitlab/fetch.go @@ -4,27 +4,27 @@ import ( "archive/zip" "bytes" "fmt" + "os" "path" + "github.com/go-git/go-billy/v5" "github.com/xanzy/go-gitlab" + + "github.com/hofstadter-io/hof/lib/yagu" + "github.com/hofstadter-io/hof/lib/yagu/repos/git" ) -func fetchShaZip(client *gitlab.Client, pid interface{}, sha string) (*zip.Reader, error) { - format := "zip" - data, _, err := client.Repositories.Archive(pid, &gitlab.ArchiveOptions{ - Format: &format, - SHA: &sha, - }) - if err != nil { - return nil, err +func Fetch(FS billy.Filesystem, owner, repo, tag string, private bool) (error) { + if private && os.Getenv(TokenEnv) == "" { + fmt.Println("gitlab git fallback") + return git.Fetch(FS, "gitlab.com", owner, repo, tag, private) } - r := bytes.NewReader(data) - - return zip.NewReader(r, int64(len(data))) -} + client, err := NewClient(private) + if err != nil { + return err + } -func FetchZip(client *gitlab.Client, owner, repo, tag string) (*zip.Reader, error) { pid := path.Join(owner, repo) var sha string @@ -32,7 +32,7 @@ func FetchZip(client *gitlab.Client, owner, repo, tag string) (*zip.Reader, erro if tag == "v0.0.0" { bs, _, err := client.Branches.ListBranches(pid, nil) if err != nil { - return nil, err + return err } var branch *gitlab.Branch @@ -46,18 +46,43 @@ func FetchZip(client *gitlab.Client, owner, repo, tag string) (*zip.Reader, erro } if branch == nil { - return nil, fmt.Errorf("Could not find default branch for repository %s", pid) + return fmt.Errorf("Could not find default branch for repository %s", pid) } sha = branch.Commit.ID } else { t, _, err := client.Tags.GetTag(pid, tag) if err != nil { - return nil, err + return err } sha = t.Commit.ID } - return fetchShaZip(client, pid, sha) + zReader, err := fetchShaZip(client, pid, sha) + if err != nil { + return fmt.Errorf("While fetching from GitLab\n%w\n", err) + } + + if err := yagu.BillyLoadFromZip(zReader, FS, true); err != nil { + return fmt.Errorf("While reading zipfile\n%w\n", err) + } + + return nil } + +func fetchShaZip(client *gitlab.Client, pid, sha string) (*zip.Reader, error) { + format := "zip" + data, _, err := client.Repositories.Archive(pid, &gitlab.ArchiveOptions{ + Format: &format, + SHA: &sha, + }) + if err != nil { + return nil, err + } + + r := bytes.NewReader(data) + + return zip.NewReader(r, int64(len(data))) +} + diff --git a/lib/yagu/ssh.go b/lib/yagu/ssh.go new file mode 100644 index 000000000..ce8566f1f --- /dev/null +++ b/lib/yagu/ssh.go @@ -0,0 +1,86 @@ +package yagu + +import ( + "os" + "path/filepath" + "strings" + + "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/kevinburke/ssh_config" +) + +type SSHMachine struct { + User string + Keys *ssh.PublicKeys +} + +func SSHCredentials(machine string) (SSHMachine, error) { + pub := "" + usr := "git" + + // first look for a usr override, can be used for key var or default location + if u := os.Getenv("HOF_SSHUSR"); u != "" { + usr = u + } + + // look for env var key location + if key := os.Getenv("HOF_SSHKEY"); key != "" { + pks, err := ssh.NewPublicKeysFromFile(usr, key, "") + if err != nil { + return SSHMachine{}, err + } + return SSHMachine{usr, pks}, nil + } + + // try to get homedir + hdir, err := os.UserHomeDir() + if err != nil { + // no home dir? + return SSHMachine{}, err + } + + // try sshconfig + _, uerr := os.Lstat(filepath.Join(hdir, ".ssh", "config")) + _, serr := os.Lstat(filepath.Join("etc", "ssh", "ssh_config")) + if uerr == nil || serr == nil { + return getSSHConfigVals(machine) + } + + // fallback on default pubkey + pub = filepath.Join(hdir, ".ssh", "id_rsa") + pks, err := ssh.NewPublicKeysFromFile(usr, pub, "") + if err != nil { + return SSHMachine{}, err + } + + return SSHMachine{usr, pks}, nil +} + +func getSSHConfigVals(machine string) (SSHMachine, error) { + // try to lookup the machine in config + pub, err := ssh_config.GetStrict(machine, "IdentityFile") + if err != nil { + return SSHMachine{}, err + } + + // replace if key location has ~ + if strings.HasPrefix(pub, "~") { + // we already validated homedir from calling function + hdir, _ := os.UserHomeDir() + pub = strings.Replace(pub, "~", hdir, 1) + } + + // override user if defined in config + usr := ssh_config.Get(machine, "User") + if usr == "" { + usr = "git" + } + + // get key from filename + pks, err := ssh.NewPublicKeysFromFile(usr, pub, "") + if err != nil { + return SSHMachine{}, err + } + + return SSHMachine{usr, pks}, nil +} diff --git a/script/runtime/tests/homedir.txt b/script/runtime/tests/homedir.txt new file mode 100644 index 000000000..8165b377a --- /dev/null +++ b/script/runtime/tests/homedir.txt @@ -0,0 +1,6 @@ +env FOO=BAR + +mkdir $HOME/.ssh +exec bash -c 'echo $FOO > $HOME/.ssh/foo' +exists $HOME/.ssh/foo +cat $HOME/.ssh/foo diff --git a/test/testers/api-test.cue b/test/testers/api/postman.cue similarity index 72% rename from test/testers/api-test.cue rename to test/testers/api/postman.cue index 464cad8d8..8cf208470 100644 --- a/test/testers/api-test.cue +++ b/test/testers/api/postman.cue @@ -41,21 +41,37 @@ basic: { post: _ @test(api,basic,post) post: { - #args: { - foo: "bar" + req: basePost & { + data: { + cow: "moo" + } } + resp: { + status: 200 + body: { + json: close({ + cow: "moo" + }) + ... + } + } + } + + fail: _ @test(api,basic,fail) + fail: { + fail: true req: basePost & { data: { cow: "moo" + foo: "bar" } } resp: { status: 200 body: { - args: #args - json: { + json: close({ cow: "moo" - } + }) ... } }