Pin all go dependencies to specific commit IDs. (#922)

This prevents any compromises of upstream code from affecting us. The
downside is it requires careful maintenance of dependencies and
revisions.

This change removes the previous go dependency manager we were using
(https://github.com/rancher/trash) with govendor
(https://github.com/kardianos/govendor), as the former doesn't fulfill
all our requirements.

Also:
- Add a .keepme to logs/ so that it's already present in a fresh clone.
- Move go-related stuff from the top-level Makefile into go/
- Install `realpath` in circleci, as go/Makefile uses it, and it's not
  installed by default on ubuntu 14.04.
- Add target dependencies so that things like running go tests (or
  coverage) have the environement properly initalized.
- go/deps.txt and tools/godeps.py aren't needed any more, and are
  removed.

.gitignore

@@ -10,7 +10,8 @@ Thumbs.db
 
 # SCION generated files #
 #########################
-/logs/
+/logs/*
+!/logs/.keepme
 /gen/
 /traces/
 
@@ -23,7 +24,8 @@ topology/mininet/*.pyc
 #########################
 /proto/go.capnp
 /go/proto/*.capnp.go
-/go/vendor/
+/go/vendor/*/
+/go/vendor/.deps.stamp
 
 # vim generated files #
 #########################

Makefile

@@ -1,18 +1,19 @@
-.PHONY: all clean go clibs libscion libfilter liblwip libtcpmw libssocket dispatcher install uninstall goproto
+.PHONY: all clean go gohsr clibs libscion libfilter liblwip libtcpmw libssocket dispatcher libhsr install uninstall
 
-SRC_DIRS = lib/libscion lib/libfilter endhost/ssp sub/lwip-contrib lib/tcp endhost go/proto
+SRC_DIRS = lib/libscion lib/libfilter endhost/ssp sub/lwip-contrib lib/tcp endhost
 
 all: clibs dispatcher go
 
 clean:
 	$(foreach var,$(SRC_DIRS),$(MAKE) -C $(var) clean || exit 1;)
+	if type -P go >/dev/null; then cd go && make clean; fi
 
-go: goproto libscion
-	GOBIN=$$PWD/bin go install -v ./go/...
+go: libscion
+	# `make -C go` breaks if there are symlinks in $PWD
+	cd go && make
 
 gohsr: libhsr
-	GOBIN=$$PWD/bin go install -tags hsr -v ./go/border/...
-	sudo setcap cap_dac_read_search,cap_dac_override,cap_sys_admin,cap_net_raw+ep bin/border
+	cd go && make hsr
 
 # Order is important
 clibs: libscion libfilter libssocket liblwip libtcpmw
@@ -42,6 +43,3 @@ install: clibs dispatcher
 
 uninstall:
 	$(foreach var,$(SRC_DIRS),$(MAKE) -C $(var) uninstall || exit 1;)
-
-goproto:
-	$(MAKE) -C go/proto

circle.yml

@@ -30,7 +30,7 @@ dependencies:
     override:
         - docker/cache.sh restore
         - mv ~/cache ~/cache.old; mkdir ~/cache
-        - sudo apt-get install -y clang-3.4 capnproto
+        - sudo apt-get install -y clang-3.4 capnproto realpath
         - ./deps.sh golang
         - make -s go
         - ./docker.sh build

deps.sh

@@ -70,17 +70,28 @@ cmd_golang() {
         # testing building Go code inside docker.
         sudo DEBIAN_FRONTEND=noninteractive apt-get install $APTARGS --no-install-recommends golang-1.6 git
     fi
-    echo "Installing go tools"
-    go get -v $(<go/deps.txt)
-    echo "Installing managed go dependencies (via trash)"
-    trash -C go
+    if ! type -P govendor &>/dev/null; then
+        (
+            HOST=github.com
+            USER=kardianos
+            PROJECT=govendor
+            COMMIT=120a6099270fc9360236f4383430e2adda6181cc
+            GOPATH_BASE=${GOPATH%%:*}
+            echo "Installing govendor dep manager"
+            mkdir -p "${GOPATH_BASE}/src/$HOST/$USER"
+            cd "${GOPATH_BASE}/src/$HOST/$USER/"
+            [ ! -d "$PROJECT" ] && git clone "git@$HOST:$USER/$PROJECT.git"
+            cd "$PROJECT"
+            git fetch
+            git checkout "$COMMIT"
+            go install -v
+        );
+    fi
+    echo "Downloading go dependencies (via govendor)"
+    # `make -C go` breaks if there are symlinks in $PWD
+    ( cd go && make deps )
     echo "Copying go-capnproto2's go.capnp into proto/"
-    local srcdir=$(go list -f "{{.Dir}}" zombiezen.com/go/capnproto2)
-    cp ${srcdir:?}/std/go.capnp proto/go.capnp
-    echo "Generating go capnp code"
-    make goproto
-    echo "Installing go dependencies"
-    go get -v $(tools/godeps.py)
+    cp go/vendor/zombiezen.com/go/capnproto2/std/go.capnp proto/go.capnp
 }
 
 chk_go() {

go/Makefile

@@ -0,0 +1,51 @@
+.PHONY: all clean test coverage fmt deps_proto deps depspurge proto bin libs hsr
+
+LOCAL_DIRS = $(shell find * -maxdepth 0 -type d | grep -v '^vendor$$')
+LOCAL_PKGS = $(patsubst %, ./%/..., $(LOCAL_DIRS))
+LOCAL_GOBIN = $(shell realpath -s $$PWD/../bin)
+
+all: deps_proto bin
+
+clean:
+	rm -f gocover.html vendor/.deps.stamp
+	GOBIN=${LOCAL_GOBIN} go clean ${LOCAL_PKGS}
+	go clean -i ./vendor/...
+	cd proto && $(MAKE) clean
+
+test: deps_proto
+	govendor test +local
+
+coverage: deps_proto
+	gocov test ${LOCAL_PKGS} | gocov-html > gocover.html
+	@echo
+	@echo "Go coverage report here: file://$$PWD/gocover.html"
+
+fmt:
+	gofmt -d -s ${LOCAL_DIRS}
+
+deps_proto: proto
+
+deps: vendor/.deps.stamp
+
+vendor/.deps.stamp: vendor/vendor.json 
+	govendor sync
+	go install -v ./vendor/...
+	touch $@
+
+depspurge:
+	rm -f vendor/.deps.stamp
+	go clean -i ./vendor/...
+	find vendor/* -maxdepth 0 -type d -exec rm -rf ./{} \;
+
+proto: deps
+	cd proto && $(MAKE)
+
+bin: deps_proto
+	GOBIN=${LOCAL_GOBIN} govendor install -v +local,program
+
+libs: deps_proto
+	govendor install -v +local,^program
+
+hsr: libs
+	GOBIN=${LOCAL_GOBIN} go install -v -tags hsr ./border/...
+	sudo setcap cap_dac_read_search,cap_dac_override,cap_sys_admin,cap_net_raw+ep ../bin/border

go/border/hsr/empty.go

@@ -0,0 +1,17 @@
+// Copyright 2016 ETH Zurich
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Dummy file so that govendor doesn't error when it tries to install this package.
+
+package hsr

go/proto/Makefile

@@ -8,7 +8,7 @@ OUTS = $(patsubst $(PROTO_DIR)/%, %.go, $(SRCS))
 all: $(OUTS)
 
 %.go: $(PROTO_DIR)/%
-	capnp compile -I$(GOPATH)/src/zombiezen.com/go/capnproto2/std -ogo --src-prefix=$(PROTO_DIR) $<
+	capnp compile -I../vendor/zombiezen.com/go/capnproto2/std -ogo --src-prefix=$(PROTO_DIR) $<
 
 clean:
 	rm -f *.capnp.go