Vendor dependencies when running code generators. (#314)

Uses `go mod vendor` to vendor code-generation dependencies.

Goal were:
1. Use the same version of code-generator that knative uses.
2. Because of 1., remove all sed hacks when generating code
3. (Still to test) fix our code incompatibility with watchers

Solution was:
1. enforce go mod vendor by using replace directives in go.mod to bin k8s.io dependencies as close as we could to k8s version 1.12.9: the version used by knative/pkg and knative/serving.
2. Update our scripts to utilize the vendor direcotry
3. Because go mod vendor does not down;pad scripts, we download the code generation scripts manually. This makes me sad.

Other considerations:
1. Dep cannot be used because we need 2 different versions of knative/pkg, because of our dependency on knative build. Once we move to tekton we might be able to switch to dep, making us more consistent with knative.

Side effects were:
1. the go mod cache cannot be used with vendored dependencies, causing a massive slowdown in the generation script.

.gitignore

@@ -1 +1,2 @@
 bin
+vendor

ci/concourse/pipelines/pr-pipeline.yml

@@ -92,6 +92,7 @@ jobs:
   - get: ci-image
   - get: ci-src
   - task: check
+    timeout: 1h30m
     config:
       platform: linux
       image_resource:

go.mod

@@ -4,7 +4,6 @@ require (
 	contrib.go.opencensus.io/exporter/stackdriver v0.12.1 // indirect
 	github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
-	github.com/docker/docker v0.7.3-0.20190307005417-54dddadc7d5d // indirect
 	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
@@ -13,7 +12,7 @@ require (
 	github.com/google/go-containerregistry v0.0.0-20190306174256-678f6c51f585
 	github.com/google/uuid v1.1.1 // indirect
 	github.com/google/wire v0.2.2
-	github.com/gorilla/mux v1.6.2
+	github.com/gorilla/mux v1.7.0
 	github.com/imdario/mergo v0.3.7
 	github.com/knative/build v0.7.0
 	github.com/knative/pkg v0.0.0-20190621200921-9c5d970cbc9e
@@ -31,16 +30,14 @@ require (
 	github.com/spf13/cobra v0.0.3
 	go.opencensus.io v0.22.0 // indirect
 	go.uber.org/zap v1.9.1
-	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
-	golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect
-	golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 // indirect
 	google.golang.org/appengine v1.5.0 // indirect
 	gopkg.in/yaml.v2 v2.2.2
 	gotest.tools v2.2.0+incompatible // indirect
-	k8s.io/api v0.0.0-20190226173710-145d52631d00
-	k8s.io/apimachinery v0.0.0-20190221084156-01f179d85dbc
+	k8s.io/api v0.0.0
+	k8s.io/apimachinery v0.0.0
 	k8s.io/client-go v2.0.0-alpha.0.0.20190226174127-78295b709ec6+incompatible
-	k8s.io/kubernetes v1.14.2
+	k8s.io/code-generator v0.0.0
+	k8s.io/kubernetes v1.15.0
 	knative.dev/pkg v0.0.0-20190626215608-1104d6c75533
 )
 
@@ -54,3 +51,28 @@ replace github.com/google/go-cmp => github.com/google/go-cmp v0.3.0
 
 // Remove once https://github.com/google/kf/issues/238 is resolved
 replace github.com/knative/pkg => github.com/poy/knative-pkg v99.0.0+incompatible
+
+replace k8s.io/client-go => k8s.io/client-go v2.0.0-alpha.0.0.20190226174127-78295b709ec6+incompatible
+
+replace (
+	k8s.io/api => k8s.io/api v0.0.0-20190528110122-9ad12a4af326
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190528110544-fa58353d80f3
+	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190221084156-01f179d85dbc
+	k8s.io/apiserver => k8s.io/apiserver v0.0.0-20190528110248-2d60c3dee270
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20190528110732-ad79ea2fbc0f
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.0.0-20181009000525-95810021865e
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.0.0-20181003203521-0cc92547a631
+	k8s.io/code-generator => k8s.io/code-generator v0.0.0-20181128191024-b1289fc74931
+	k8s.io/component-base => k8s.io/component-base v0.0.0-20190107200011-1e2bcba2af7f
+	k8s.io/cri-api => k8s.io/cri-api v0.0.0-20190326082326-5c2568eea0b8
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.0.0-20190109195450-94d98b9371d9
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.0.0-20190528110328-0ab90e449f7e
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.0.0-20190528111014-463e5d26aa13
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.0.0-20190528110839-96abc4c8d1a4
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.0.0-20190528110942-86bc7e94eb9a
+	k8s.io/kubelet => k8s.io/kubelet v0.0.0-20190528110910-f5f997cd2103
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.0.0-20190416155406-4c85c9b0ae06
+	k8s.io/metrics => k8s.io/metrics v0.0.0-20190528110627-05eb8901940c
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.0.0-20190528110419-48d5cc0538c7
+
+)

hack/build.sh

@@ -17,4 +17,5 @@
 set -eux
 cd "${0%/*}"/..
 mkdir -p bin
-go build -o bin/kf ./cmd/kf/...
+go mod vendor
+go build -mod=vendor -o bin/kf ./cmd/kf/...

hack/check-code-generator.sh

@@ -18,7 +18,6 @@
 
 set -eux
 
-hack/setup-codegen.sh
 hack/update-codegen.sh
 
 if [ ! -z "$(git status --porcelain)" ]; then

hack/update-codegen.sh

@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -eu
+set -e
 
-export GO111MODULE=off
+export GO111MODULE=on
 
 GENERATOR_FLAGS=""
 while getopts "v" opt; do
@@ -29,120 +29,83 @@ while getopts "v" opt; do
 done
 
 HACK_DIR="${0%/*}"
-CODEGEN_PKG=$(go env GOPATH)/src/k8s.io/code-generator
-CODEGEN_PACKAGES=$(cat $HACK_DIR/codegen-packages.txt)
 KF_PACKAGE="github.com/google/kf"
-KF_PACKAGE_LOCATION="$(go env GOPATH)/src/$KF_PACKAGE"
+KF_PACKAGE_LOCATION="./"
 KF_RESOURCE="kf:v1alpha1"
 BUILD_RESOURCE="build:v1alpha1"
 HEADER_FILE=${KF_PACKAGE_LOCATION}/pkg/kf/internal/tools/option-builder/LICENSE_HEADER.go.txt
-KNATIVE_CODEGEN_PKG=$(go env GOPATH)/src/knative.dev/pkg
 
-if [ ! -d "$KF_PACKAGE_LOCATION" ]; then
-  echo "Cannot find go package $KF_PACKAGE" 1>&2
-  exit 1
+GENS=$1
+if [ "$GENS" = "" ]; then
+  GENS="all"
 fi
 
-install_package() {
-  local PACKAGE=$1
-  echo installing $PACKAGE
-  pushd $(go env GOPATH) &>/dev/null
-    go get -u $PACKAGE/...
-  popd &>/dev/null
+code-generator-gen() {
+  commit=$(cat go.mod | grep code-generator | grep =\> | tr '-' ' ' | awk '{print $NF}')
+  echo running code-generator $commit
+
+  CODEGEN_PKG=vendor/k8s.io/code-generator
+  curl -LOJ https://raw.githubusercontent.com/kubernetes/code-generator/${commit}/generate-groups.sh
+  chmod +x generate-groups.sh
+  mkdir -p $CODEGEN_PKG
+  mv generate-groups.sh $CODEGEN_PKG/generate-groups.sh
+
+  ${CODEGEN_PKG}/generate-groups.sh all \
+    "$KF_PACKAGE/pkg/client" \
+    "$KF_PACKAGE/pkg/apis" \
+    "$KF_RESOURCE" \
+    --go-header-file="$HEADER_FILE" \
+    "${GENERATOR_FLAGS}"
+  [ $? -ne 0 ] && echo Error running code-generator 1>&2 && exit 1
+
+  ${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \
+    "$KF_PACKAGE/pkg/client/build" \
+    "github.com/knative/build/pkg/apis" \
+    "$BUILD_RESOURCE" \
+    --go-header-file="$HEADER_FILE" \
+    "${GENERATOR_FLAGS}"
+  [ $? -ne 0 ] && echo Error running code-generator 1>&2 && exit 1
 }
 
-for PACKAGE in ${CODEGEN_PACKAGES}; do
-  if [ ! -d $(go env GOPATH)/src/${PACKAGE} ]; then
-    install_package $PACKAGE
-  fi
-done
-
-if [ "$GOPATH/src/github.com/google/kf" != "$(git rev-parse --show-toplevel)" ]; then
-    echo "The generator scripts aren't go module compatible (yet)." 1>&2
-    exit 1
-fi
-
-# Fix issues due to using old k8s.io/client-go
-# The generator wants to use a codec that is only available in a version of
-# k8s.io/apimachinery that we can't yet use.
-os_friendly_sed () {
-  echo "Applying $1 to $2"
-  if [ ! -e "$2" ]; then
-    echo file not found: $2 2>&1
-    exit 1
-  fi
-  sed "$1" "$2" > "$2.new"
-  mv "$2.new" "$2"
-}
-
-${CODEGEN_PKG}/generate-groups.sh all \
-  "$KF_PACKAGE/pkg/client" \
-  "$KF_PACKAGE/pkg/apis" \
-  "$KF_RESOURCE" \
-  --go-header-file="$HEADER_FILE" \
-  "${GENERATOR_FLAGS}"
-[ $? -ne 0 ] && echo Error running code-generator 1>&2 && exit 1
-
-${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \
-  "$KF_PACKAGE/pkg/client/build" \
-  "github.com/knative/build/pkg/apis" \
-  "$BUILD_RESOURCE" \
-  --go-header-file="$HEADER_FILE" \
-  "${GENERATOR_FLAGS}"
-[ $? -ne 0 ] && echo Error running code-generator 1>&2 && exit 1
-
-codegen_sed() {
-  local CLIENT=$1
-  local TYPE=$2
-  local VERSION=$3
-  local FILE=$4
-
-  if [ "$CLIENT" = "kf" ]; then
-    CLIENT=""
-  fi
-
-  os_friendly_sed 's/scheme.Codecs.WithoutConversion()/scheme.Codecs/g' "$KF_PACKAGE_LOCATION/pkg/client/$CLIENT/clientset/versioned/typed/$TYPE/$VERSION/$FILE.go"
-  os_friendly_sed 's/pt, //g' "$KF_PACKAGE_LOCATION/pkg/client/$CLIENT/clientset/versioned/typed/$TYPE/$VERSION/fake/fake_${FILE}.go"
-}
-
-TYPES="build_client build buildtemplate clusterbuildtemplate"
-for type in ${TYPES}; do
-  codegen_sed build build v1alpha1 $type
-done
-
-TYPES="kf_client $(ls ${KF_PACKAGE_LOCATION}/pkg/apis/kf/v1alpha1/ | grep 'types.go' | sed 's/_types.go//')"
-for type in ${TYPES}; do
-  codegen_sed kf kf v1alpha1 $type
-done
-
-# Do Knative injection generation
-${KNATIVE_CODEGEN_PKG}/hack/generate-knative.sh \
-  "injection" \
-  "github.com/google/kf/pkg/client" \
-  "github.com/google/kf/pkg/apis" \
-  "kf:v1alpha1" \
-  --go-header-file $HEADER_FILE
-
-${KNATIVE_CODEGEN_PKG}/hack/generate-knative.sh \
-  "injection" \
-  "github.com/google/kf/pkg/client/build" \
-  "github.com/knative/build/pkg/apis" \
-  "build:v1alpha1" \
-  --go-header-file $HEADER_FILE
-
-# For some reason the fake doesn't have the right imports (it's missing
-# k8s.io/client-go/rest)
-
-knative_injection_sed (){
-  local CLIENT=$1
-
-  if [ "$CLIENT" = "kf" ]; then
-    CLIENT=""
-  fi
-  os_friendly_sed 's|"k8s.io/apimachinery/pkg/runtime"|"k8s.io/apimachinery/pkg/runtime"\n"k8s.io/client-go/rest"|g' "${KF_PACKAGE_LOCATION}/pkg/client/$CLIENT/injection/client/fake/fake.go"
+knative-injection-gen() {
+  commit=$(cat go.mod | grep knative.dev/pkg | tr '-' ' ' | awk '{print $NF}')
+  echo running knative-injection-generator $commit
+
+  KNATIVE_CODEGEN_PKG=vendor/knative.dev/pkg/hack
+  curl -LOJ https://raw.githubusercontent.com/knative/pkg/${commit}/hack/generate-knative.sh
+  chmod +x generate-knative.sh
+  mkdir -p $KNATIVE_CODEGEN_PKG
+  mv generate-knative.sh $KNATIVE_CODEGEN_PKG/generate-knative.sh
+
+  # Do Knative injection generation
+  ${KNATIVE_CODEGEN_PKG}/generate-knative.sh \
+    "injection" \
+    "github.com/google/kf/pkg/client" \
+    "github.com/google/kf/pkg/apis" \
+    "kf:v1alpha1" \
+    --go-header-file $HEADER_FILE
+
+  ${KNATIVE_CODEGEN_PKG}/generate-knative.sh \
+    "injection" \
+    "github.com/google/kf/pkg/client/build" \
+    "github.com/knative/build/pkg/apis" \
+    "build:v1alpha1" \
+    --go-header-file $HEADER_FILE
 }
 
-knative_injection_sed kf
-knative_injection_sed build
+go mod vendor
+
+case $GENS in
+  k8s)
+    code-generator-gen
+    ;;
+  knative)
+    knative-injection-gen
+    ;;
+  all)
+    code-generator-gen
+    knative-injection-gen
+    ;;
+esac
 
 gofmt -s -w .