lib: Move to go modules and shed a few tears

The old system with vendor/ and git submodules worked great,
unfortunately FUD around git submodules seemed to scare people away and
golang moved to a go.mod system that adds a new lock file format instead
of using the built-in git version. It's now almost impossible to use
modern golang without this, so we've switched.

So much for the golang compatibility promise-- turns out it doesn't
apply to the useful parts that I actually care about like this.

Thanks to frebib for his incredibly valuable contributions to this
patch. This snide commit message is mine alone.

This patch also mixes in some changes due to legacy golang as we've also
bumped the minimum version to 1.16 in the docs and tests.

Lastly, we had to disable some tests and fix up a few other misc things
to get this passing. We've definitely hot bugs in the go.mod system, and
our Makefile tries to workaround those.

.github/workflows/test.yaml

@@ -27,10 +27,9 @@ jobs:
           # macos tests are currently failing in CI
           #- macos-latest
         golang_version:
-          # TODO: add 1.15.x and tip
+          # TODO: add 1.17.x and tip
           # minimum required and latest published go_version
-          #- 1.13
-          - 1.15
+          - 1.16
         test_block:
           - basic
           - shell

.gitignore

@@ -7,7 +7,6 @@ old/
 tmp/
 *WIP
 *_stringer.go
-bindata/*.go
 mgmt
 mgmt.static
 # crossbuild artifacts

.travis.yml

@@ -24,21 +24,21 @@ install: 'make deps'
 matrix:
   fast_finish: false
   allow_failures:
-  - go: 1.14.x
+  - go: 1.17.x
   - go: tip
   - os: osx
   # include only one build for osx for a quicker build as the nr. of these runners are sparse
   include:
   - name: "basic tests"
-    go: 1.13.x
+    go: 1.16.x
     env: TEST_BLOCK=basic
   - name: "shell tests"
-    go: 1.13.x
+    go: 1.16.x
     env: TEST_BLOCK=shell
   - name: "race tests"
-    go: 1.13.x
+    go: 1.16.x
     env: TEST_BLOCK=race
-  - go: 1.14.x
+  - go: 1.17.x
   - go: tip
   - os: osx
 script: 'TEST_BLOCK="$TEST_BLOCK" make test'

Makefile

@@ -53,6 +53,10 @@ GOOSARCHES ?= linux/amd64 linux/ppc64 linux/ppc64le linux/arm64 darwin/amd64
 GOHOSTOS = $(shell go env GOHOSTOS)
 GOHOSTARCH = $(shell go env GOHOSTARCH)
 
+# Work around a golang 1.16 bug https://github.com/golang/go/issues/44129
+# Can be removed when 1.17 is the minimum supported version
+export GOFLAGS ?= -mod=mod
+
 TOKEN_FEDORA-30 = fedora-30
 TOKEN_FEDORA-29 = fedora-29
 TOKEN_CENTOS-7 = centos-7
@@ -139,7 +143,6 @@ race:
 
 # generate go files from non-go source
 bindata: ## generate go files from non-go sources
-	$(MAKE) --quiet -C bindata
 	$(MAKE) --quiet -C lang/funcs
 
 generate:
@@ -170,15 +173,14 @@ GOOS=$(firstword $(subst -, ,$*))
 GOARCH=$(lastword $(subst -, ,$*))
 build/mgmt-%: $(GO_FILES) $(MCL_FILES) | bindata lang funcgen
 	@echo "Building: $(PROGRAM), os/arch: $*, version: $(SVERSION)..."
-	@time env GOOS=${GOOS} GOARCH=${GOARCH} go build -i -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS)
+	@time env GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS)
 
 # create a list of binary file names to use as make targets
 crossbuild_targets = $(addprefix build/mgmt-,$(subst /,-,${GOOSARCHES}))
 crossbuild: ${crossbuild_targets}
 
 clean: ## clean things up
 	$(MAKE) --quiet -C test clean
-	$(MAKE) --quiet -C bindata clean
 	$(MAKE) --quiet -C lang/funcs clean
 	$(MAKE) --quiet -C lang clean
 	$(MAKE) --quiet -C misc/mkosi clean

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13
+FROM golang:1.16
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 

docker/Dockerfile.build

@@ -6,7 +6,7 @@ ENV PATH=/opt/rh/rh-ruby22/root/usr/bin:/root/gopath/bin:/usr/local/sbin:/sbin:/
 ENV LD_LIBRARY_PATH=/opt/rh/rh-ruby22/root/usr/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
 ENV PKG_CONFIG_PATH=/opt/rh/rh-ruby22/root/usr/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}
 
-RUN yum -y install epel-release wget unzip git make which centos-release-scl gcc && sed -i "s/enabled=0/enabled=1/" /etc/yum.repos.d/epel-testing.repo && yum -y install rh-ruby22 && wget -O /opt/go1.9.1.linux-amd64.tar.gz https://storage.googleapis.com/golang/go1.9.1.linux-amd64.tar.gz && tar -C /usr/local -xzf /opt/go1.9.1.linux-amd64.tar.gz
+RUN yum -y install epel-release wget unzip git make which centos-release-scl gcc && sed -i "s/enabled=0/enabled=1/" /etc/yum.repos.d/epel-testing.repo && yum -y install rh-ruby22 && wget -O /opt/go1.16.8.linux-amd64.tar.gz https://storage.googleapis.com/golang/go1.16.8.linux-amd64.tar.gz && tar -C /usr/local -xzf /opt/go1.16.8.linux-amd64.tar.gz
 RUN mkdir -p $GOPATH/src/github.com/purpleidea && cd $GOPATH/src/github.com/purpleidea && git clone --recursive https://github.com/purpleidea/mgmt
 RUN go get -u gopkg.in/alecthomas/gometalinter.v1 && cd $GOPATH/src/github.com/purpleidea/mgmt && make deps && make build
 CMD ["/bin/bash"]

docker/Dockerfile.development

@@ -1,4 +1,4 @@
-FROM golang:1.13
+FROM golang:1.16
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 
@@ -27,8 +27,5 @@ WORKDIR /home/$USER_NAME/mgmt
 # Install dependencies
 RUN make deps
 
-# Chown $GOPATH
-RUN chown -R ${USER_ID}:${GROUP_ID} /go
-
 # Change user
 USER ${USER_NAME}

docs/development.md

@@ -28,7 +28,7 @@ required for running the _test_ suite.
 
 ### Build
 
-* `golang` 1.13 or higher (required, available in some distros and distributed
+* `golang` 1.16 or higher (required, available in some distros and distributed
 as a binary officially by [golang.org](https://golang.org/dl/))
 
 ### Runtime

docs/faq.md

@@ -216,17 +216,6 @@ requires a number of seconds as an argument.
 ./mgmt run lang examples/lang/hello0.mcl --converged-timeout=5
 ```
 
-### When I try to build `mgmt` I see: `no Go files in $GOPATH/src/github.com/purpleidea/mgmt/bindata`.
-
-Due to the arcane way that `golang` designed its `$GOPATH`, the main project
-directory must be inside your `$GOPATH`, and at the appropriate FQDN. This is:
-`$GOPATH/src/github.com/purpleidea/mgmt/`. If you have your project root outside
-of that directory, then you may get this error when you try to build it. In this
-case there is likely a `go get` version of the project at this location. Remove
-it and replace it with your git cloned directory. In my case, I like to work on
-things in `~/code/mgmt/`, so that path is a symlink that points to the long
-project directory.
-
 ### Why does my file resource error with `no such file or directory`?
 
 If you create a file resource and only specify the content like this:
@@ -364,6 +353,14 @@ Don't blindly use the tools that others tell you to. Learn what they do, think
 for yourself, and become a power user today! That process led us to using
 `git submodules`. Hopefully you'll come to the same conclusions that we did.
 
+**UPDATE:**
+
+After golang made it virtually impossible to build without `go.mod` stuff, we've
+switched to it since golang 1.16. I still think the above approach was better,
+and that the `go mod` tooling should have been a layer on top of git submodules
+so that we don't grow yet another lock file format, and existing folks who are
+comfortable with `git` can use those tools directly.
+
 ### Did you know that there is a band named `MGMT`?
 
 I didn't realize this when naming the project, and it is accidental. After much

docs/quick-start-guide.md

@@ -37,7 +37,7 @@ You'll need some dependencies, including `golang`, and some associated tools.
 
 #### Installing golang
 
-* You need golang version 1.13 or greater installed.
+* You need golang version 1.16 or greater installed.
 	* To install on rpm style systems: `sudo dnf install golang`
 	* To install on apt style systems: `sudo apt install golang`
 	* To install on macOS systems install [Homebrew](https://brew.sh)
@@ -63,13 +63,11 @@ export GOPATH=$HOME/gopath
 
 #### Getting the mgmt code and associated dependencies
 
-* Download the `mgmt` code into the `GOPATH`, and switch to that directory:
+* Download the `mgmt` code and switch to that directory:
 
 ```shell
-[ -z "$GOPATH" ] && mkdir ~/go/ || mkdir -p $GOPATH/src/github.com/purpleidea/
-cd $GOPATH/src/github.com/purpleidea/ || cd ~/go/
-git clone --recursive https://github.com/purpleidea/mgmt/
-cd $GOPATH/src/github.com/purpleidea/mgmt/ || cd ~/go/src/github.com/purpleidea/mgmt/
+git clone --recursive https://github.com/purpleidea/mgmt/ ~/mgmt/
+cd ~/mgmt/
 ```
 
 * Add `$GOPATH/bin` to `$PATH`

engine/resources/augeas.go

@@ -29,8 +29,6 @@ import (
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
-	// FIXME: we vendor go/augeas because master requires augeas 1.6.0
-	// and libaugeas-dev-1.6.0 is not yet available in a PPA.
 	"honnef.co/go/augeas"
 )
 

engine/resources/cron.go

@@ -33,10 +33,10 @@ import (
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
-	sdbus "github.com/coreos/go-systemd/dbus"
-	"github.com/coreos/go-systemd/unit"
-	systemdUtil "github.com/coreos/go-systemd/util"
-	"github.com/godbus/dbus"
+	sdbus "github.com/coreos/go-systemd/v22/dbus"
+	"github.com/coreos/go-systemd/v22/unit"
+	systemdUtil "github.com/coreos/go-systemd/v22/util"
+	"github.com/godbus/dbus/v5"
 )
 
 const (

engine/resources/hostname.go

@@ -27,7 +27,7 @@ import (
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
-	"github.com/godbus/dbus"
+	"github.com/godbus/dbus/v5"
 )
 
 func init() {

engine/resources/mount.go

@@ -35,11 +35,11 @@ import (
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
-	sdbus "github.com/coreos/go-systemd/dbus"
-	"github.com/coreos/go-systemd/unit"
-	systemdUtil "github.com/coreos/go-systemd/util"
+	sdbus "github.com/coreos/go-systemd/v22/dbus"
+	"github.com/coreos/go-systemd/v22/unit"
+	systemdUtil "github.com/coreos/go-systemd/v22/util"
 	fstab "github.com/deniswernert/go-fstab"
-	"github.com/godbus/dbus"
+	"github.com/godbus/dbus/v5"
 	"golang.org/x/sys/unix"
 )
 

engine/resources/msg.go

@@ -25,7 +25,7 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 
-	"github.com/coreos/go-systemd/journal"
+	"github.com/coreos/go-systemd/v22/journal"
 )
 
 func init() {

engine/resources/nspawn.go

@@ -30,10 +30,10 @@ import (
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
-	systemdDbus "github.com/coreos/go-systemd/dbus"
-	machined "github.com/coreos/go-systemd/machine1"
-	systemdUtil "github.com/coreos/go-systemd/util"
-	"github.com/godbus/dbus"
+	systemdDbus "github.com/coreos/go-systemd/v22/dbus"
+	machined "github.com/coreos/go-systemd/v22/machine1"
+	systemdUtil "github.com/coreos/go-systemd/v22/util"
+	"github.com/godbus/dbus/v5"
 )
 
 const (

engine/resources/packagekit/packagekit.go

@@ -29,7 +29,7 @@ import (
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
-	"github.com/godbus/dbus"
+	"github.com/godbus/dbus/v5"
 )
 
 // global tweaks of verbosity and code path

engine/resources/svc.go

@@ -30,9 +30,9 @@ import (
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
-	systemd "github.com/coreos/go-systemd/dbus" // change namespace
-	systemdUtil "github.com/coreos/go-systemd/util"
-	"github.com/godbus/dbus" // namespace collides with systemd wrapper
+	systemd "github.com/coreos/go-systemd/v22/dbus" // change namespace
+	systemdUtil "github.com/coreos/go-systemd/v22/util"
+	"github.com/godbus/dbus/v5" // namespace collides with systemd wrapper
 )
 
 func init() {

engine/util/util.go

@@ -33,7 +33,7 @@ import (
 	"github.com/purpleidea/mgmt/lang/types"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
-	"github.com/godbus/dbus"
+	"github.com/godbus/dbus/v5"
 )
 
 const (

etcd/callback.go

@@ -26,8 +26,8 @@ import (
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
-	etcd "go.etcd.io/etcd/clientv3" // "clientv3"
-	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	etcd "go.etcd.io/etcd/client/v3"
 )
 
 // nominateApply applies the changed watcher data onto our local caches.

etcd/chooser/chooser.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/purpleidea/mgmt/etcd/interfaces"
 
-	etcdtypes "go.etcd.io/etcd/pkg/types"
+	etcdtypes "go.etcd.io/etcd/client/pkg/v3/types"
 )
 
 // Data represents the input data that is passed to the chooser.

etcd/chooser/dynamicsize.go

@@ -25,8 +25,8 @@ import (
 
 	"github.com/purpleidea/mgmt/etcd/interfaces"
 
-	etcd "go.etcd.io/etcd/clientv3"
-	etcdtypes "go.etcd.io/etcd/pkg/types"
+	etcdtypes "go.etcd.io/etcd/client/pkg/v3/types"
+	etcd "go.etcd.io/etcd/client/v3"
 )
 
 // XXX: Test causing cluster shutdowns with: