Merge pull request #71 from hofstadter-io/mod-docs-n-tests

Mod docs n tests

.github/workflows/common/base.cue

@@ -0,0 +1,31 @@
+package common
+
+import "github.com/hofstadter-io/ghacue"
+
+#Workflow: ghacue.#Workflow & {
+	name: string
+	on: _ | *["pull_request"]
+	jobs: test: {
+		strategy: matrix: {
+			"go-version": ["1.16.x", "1.17.x"]
+			os: ["ubuntu-latest", "macos-latest"]
+		}
+		strategy: "fail-fast": false
+		"runs-on": "${{ matrix.os }}"
+	}
+}
+
+#BuildSteps: [{
+	name: "Install Go"
+	uses: "actions/setup-go@v2"
+	with: "go-version": "${{ matrix.go-version }}"
+},{
+	name: "Checkout code"
+	uses: "actions/checkout@v2"
+},{
+	name: "Download mods"
+	run: "go mod download"
+},{
+	name: "Build CLI"
+	run: "go install ./cmd/hof"
+}]

.github/workflows/default.cue

@@ -1,33 +1,20 @@
-import "github.com/hofstadter-io/ghacue"
+package workflows
 
-ghacue.#Workflow & {
-	name: "hof"
-	on: ["push", "pull_request"]
+import "github.com/hofstadter-io/hof/.github/workflows/common"
+
+common.#Workflow & {
+	name: "default"
+	on: ["push"]
 	jobs: test: {
-		strategy: matrix: {
-			"go-version": ["1.16.x"]
-			os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-		}
-		strategy: "fail-fast": false
-		"runs-on": "${{ matrix.os }}"
-		steps: [{
-			name: "Install Go"
-			uses: "actions/setup-go@v2"
-			with: "go-version": "${{ matrix.go-version }}"
-		},{
-			name: "Checkout code"
-			uses: "actions/checkout@v2"
-		},{
-			name: "Download mods"
-			run: "go mod download"
-		},{
-			name: "Build CLI"
-			run: "go install ./cmd/hof"
-		},{
-			name: "Run tests"
+		steps: [ for step in common.#BuildSteps {step} ] + [{
+			name: "Run gen tests"
 			run: """
 			hof test test.cue -s gen -t test
-			hof test test.cue -s lib -t test -t mod
+			"""
+		},{
+			name: "Run tester tests"
+			run: """
+			hof test test/testers/api/postman.cue
 			"""
 		}]
 	}

.github/workflows/default.yml

@@ -1,17 +1,16 @@
-name: hof
+name: default
 on:
 - push
-- pull_request
 jobs:
   test:
     strategy:
       matrix:
         go-version:
         - 1.16.x
+        - 1.17.x
         os:
         - ubuntu-latest
         - macos-latest
-        - windows-latest
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:
@@ -25,8 +24,7 @@ jobs:
       run: go mod download
     - name: Build CLI
       run: go install ./cmd/hof
-    - name: Run tests
-      run: |-
-        hof test test.cue -s gen -t test
-        hof test test.cue -s lib -t test -t mod
-    services: {}
+    - name: Run gen tests
+      run: hof test test.cue -s gen -t test
+    - name: Run tester tests
+      run: hof test test/testers/api/postman.cue

.github/workflows/test_mod.cue

@@ -0,0 +1,25 @@
+package workflows
+
+import "github.com/hofstadter-io/hof/.github/workflows/common"
+
+common.#Workflow & {
+	name: "test_mod"
+	on: pull_request: { paths: ["lib/mod/**", "lib/yagu/repos/**", "lib/yagu/git.go", "lib/yagu/netrc.go", "lib/yagu/ssh.go"] }
+	jobs: test: {
+		environment: "hof mod testing"
+		steps: [ for step in common.#BuildSteps {step} ] + [{
+			name: "Run mod tests"
+			run: """
+			hof test test.cue -s lib -t test -t mod
+			"""
+			env: {
+				HOFMOD_SSHKEY: "${{secrets.HOFMOD_SSHKEY}}"
+				GITHUB_TOKEN: "${{secrets.HOFMOD_TOKEN}}"
+				GITLAB_TOKEN: "${{secrets.GITLAB_TOKEN}}"
+				BITBUCKET_USERNAME: "hofstadter"
+				BITBUCKET_PASSWORD: "${{secrets.BITBUCKET_TOKEN}}"
+			}
+		}]
+	}
+}
+

.github/workflows/test_mod.yml

@@ -0,0 +1,41 @@
+name: test_mod
+on:
+  pull_request:
+    paths:
+    - lib/mod/**
+    - lib/yagu/repos/**
+    - lib/yagu/git.go
+    - lib/yagu/netrc.go
+    - lib/yagu/ssh.go
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version:
+        - 1.16.x
+        - 1.17.x
+        os:
+        - ubuntu-latest
+        - macos-latest
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    environment: hof mod testing
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Download mods
+      run: go mod download
+    - name: Build CLI
+      run: go install ./cmd/hof
+    - name: Run mod tests
+      run: hof test test.cue -s lib -t test -t mod
+      env:
+        HOFMOD_SSHKEY: ${{secrets.HOFMOD_SSHKEY}}
+        GITHUB_TOKEN: ${{secrets.HOFMOD_TOKEN}}
+        GITLAB_TOKEN: ${{secrets.GITLAB_TOKEN}}
+        BITBUCKET_USERNAME: hofstadter
+        BITBUCKET_PASSWORD: ${{secrets.BITBUCKET_TOKEN}}

.hof/shadow/Cli/cmd/hof/cmd/mod.go

@@ -11,20 +11,111 @@ import (
 	"github.com/hofstadter-io/hof/cmd/hof/cmd/mod"
 )
 
-var modLong = `The mod subcmd is a polyglot dependency management tool based on go mods.
+var modLong = `hof mod is a flexible tool and library based on Go mods.
 
-mod file format:
+Use and create module systems with Minimum Version Selection (MVS) semantics
+and manage dependencies go mod style. Mix any set of language, code bases,
+git repositories, package managers, and subdirectories.
 
-  module = "<module path>"
 
-  <name> = "version"
+### Features
 
-  require (
-    ...
-  )
+- Based on go mods MVS system, aiming for 100% reproducible builds.
+- Recursive dependencies, version resolution, and code instrospection.
+- Custom module systems with custom file names and vendor directories.
+- Control configuration for naming, vendoring, and other behaviors.
+- Polyglot support for multiple module systems and multiple languages within one tool.
+- Works with any git system and supports the main features from go mods.
+- Convert other vendor and module systems to MVS or just manage their files with MVS.
+- Private repository support for GitHub, GitLab, Bitbucket, and git+SSH.
 
-  replace <module path> => <local path>
-  ...`
+
+### Usage
+
+# Print known languages in the current directory
+hof mod info
+
+# Initialize this folder as a module
+hof mod init <lang> <module-path>
+
+# Add your requirements
+vim <lang>.mods  # go.mod like file
+
+# Pull in dependencies, no args discovers by *.mods and runs all
+hof mod vendor [langs...]
+
+# See all of the commands
+hof mod help
+
+
+### Module File
+
+The module file holds the requirements for project.
+It has the same format as a go.mod file.
+
+---
+# These are like golang import paths
+#   i.e. github.com/hofstadter-io/hof
+module <module-path> 
+
+# Information about the module type / version
+#  some systems that take this into account
+# go = 1.14
+<lang> = <version>
+
+# Required dependencies section
+require (
+	# <module-path> <module-semver>
+	github.com/hof-lang/cuemod--cli-golang v0.0.0      # This is latest on HEAD
+	github.com/hof-lang/cuemod--cli-golang v0.1.5      # This is a tag v0.1.5 (can omit 'v' in tag, but not here)
+)
+
+# replace <module-path> => <module-path|local-path> [version]
+replace github.com/hof-lang/cuemod--cli-golang => github.com/hofstadter-io/hofmod-cli-golang v0.2.0
+replace github.com/hof-lang/cuemod--cli-golang => ../../cuelibs/cuemod--cli-golang
+---
+
+
+### Authentication and private modules
+
+hof mod prefers authenticated requests when fetching dependencies.
+This increase rate limits with hosts and supports private modules.
+Both token and sshkey base methods are supported.
+
+If you are using credentials, then private modules can be transparent.
+An ENV VAR like GOPRIVATE and CUEPRIVATE can be used to require credentials.
+
+The following ENV VARS are used to set credentials.
+
+GITHUB_TOKEN
+GITLAB_TOKEN
+BITBUCKET_TOKEN or BITBUCKET_USERNAME / BITBUCKET_PASSWORD *
+
+SSH keys will be looked up in the following ~/.ssh/config, /etc/ssh/config, ~/.ssh/in_rsa
+
+You can configure the SSH key with
+
+HOF_SSHUSR and HOF_SSHKEY
+
+* The bitbucket method will depend on the account type and enterprise license.
+
+
+### Custom Module Systems
+
+.mvsconfig.cue allows you to define custom module systems.
+With some simple configuration, you can create and control
+and vendored module system based on go mods.
+You can also define global configurations.
+
+See the ./lib/mod/langs in the repository for examples.
+
+### Motivation
+
+- MVS has better semantics for vendoring and gets us closer to 100% reproducible builds.
+- JS and Python can have MVS while still using the remainder of the tool chains.
+- Prototype for cuelang module and vendor management.
+- We need a module system for our [hof-lang](https://hof-lang.org) project.
+`
 
 func ModPersistentPreRun(args []string) (err error) {
 

Makefile

@@ -21,3 +21,12 @@ cloccode: cmd lang lib gen cmd
 
 clocdev: hof.cue design schema lang lib docs test cue.mods go.mod 
 	cloc --read-lang-def=$$HOME/hof/jumpfiles/assets/cloc_defs.txt --exclude-dir=cue.mod,vendor $^
+
+WORKFLOWS = default \
+	test_mod
+
+.PHONY: workflow
+workflows = $(addprefix workflow_, $(WORKFLOWS))
+workflow: $(workflows)
+$(workflows): workflow_%:
+	@cue export --out yaml .github/workflows/$(subst workflow_,,$@).cue > .github/workflows/$(subst workflow_,,$@).yml