From e563678843790244967417562e8a74a7a1d2434a Mon Sep 17 00:00:00 2001 From: Lucas Roesler Date: Mon, 8 Mar 2021 14:23:33 +0100 Subject: [PATCH 1/6] Add short script to move vendor folder around **What** - Add Dockerfile testp to mov ethe vendor folder up from the function volder to function root. This allows the user to control the vendor folder and avoid collisions/validation errors from Go in modules mode - This new flow allows dynamic downloading of Go modules _or_ disabling Go modules and only using vendor. You can not use vendor _and_ modules at the same time due to how Go will validate that the vendor folder and the go.mod are insync Signed-off-by: Lucas Roesler --- template/golang-http/Dockerfile | 35 ++++++++---- .../github.com/openfaas/templates-sdk/LICENSE | 21 ------- .../openfaas/templates-sdk/go-http/README.md | 41 -------------- .../openfaas/templates-sdk/go-http/handler.go | 56 ------------------- template/golang-http/vendor/modules.txt | 2 - template/golang-middleware/Dockerfile | 16 ++++-- 6 files changed, 35 insertions(+), 136 deletions(-) delete mode 100644 template/golang-http/vendor/github.com/openfaas/templates-sdk/LICENSE delete mode 100644 template/golang-http/vendor/github.com/openfaas/templates-sdk/go-http/README.md delete mode 100644 template/golang-http/vendor/github.com/openfaas/templates-sdk/go-http/handler.go delete mode 100644 template/golang-http/vendor/modules.txt diff --git a/template/golang-http/Dockerfile b/template/golang-http/Dockerfile index d882607..9aeaf2d 100644 --- a/template/golang-http/Dockerfile +++ b/template/golang-http/Dockerfile @@ -8,41 +8,54 @@ ARG TARGETARCH RUN apk --no-cache add git -ENV CGO_ENABLED=0 - COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog RUN chmod +x /usr/bin/fwatchdog +ENV CGO_ENABLED=0 + RUN mkdir -p /go/src/handler WORKDIR /go/src/handler COPY . . +ARG GO111MODULE="off" +ARG GOPROXY="" +ARG GOFLAGS="" + +# Add user overrides to the root go.mod, which is the only place "replace" can be used +RUN cat function/GO_REPLACE.txt >> ./go.mod || exit 0 +RUN if [ -d ./function/vendor ]; then \ + echo "moving handler vendor" && \ + mv -f ./function/vendor . && go mod tidy; \ + else \ + echo "using modules or vendor not found "; \ + fi + # Run a gofmt and exclude all vendored code. RUN test -z "$(gofmt -l $(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./function/vendor/*"))" || { echo "Run \"gofmt -s -w\" on your Golang code"; exit 1; } -ARG GO111MODULE="off" -ARG GOPROXY="" +WORKDIR /go/src/handler/function + +RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go test ./... -cover + +WORKDIR /go/src/handler RUN CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build --ldflags "-s -w" -a -installsuffix cgo -o handler . -RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go test handler/function/... -cover FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine:3.13 # Add non root user and certs RUN apk --no-cache add ca-certificates \ && addgroup -S app && adduser -S -g app app -# Split instructions so that buildkit can run & cache +# Split instructions so that buildkit can run & cache # the previous command ahead of time. RUN mkdir -p /home/app \ && chown app /home/app WORKDIR /home/app -COPY --from=build /go/src/handler/handler . -COPY --from=build /usr/bin/fwatchdog . -COPY --from=build /go/src/handler/function/ . - -RUN chown -R app /home/app +COPY --from=build --chown=app /go/src/handler/handler . +COPY --from=build --chown=app /usr/bin/fwatchdog . +COPY --from=build --chown=app /go/src/handler/function/ . USER app diff --git a/template/golang-http/vendor/github.com/openfaas/templates-sdk/LICENSE b/template/golang-http/vendor/github.com/openfaas/templates-sdk/LICENSE deleted file mode 100644 index 7ecb7d7..0000000 --- a/template/golang-http/vendor/github.com/openfaas/templates-sdk/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 OpenFaaS - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/template/golang-http/vendor/github.com/openfaas/templates-sdk/go-http/README.md b/template/golang-http/vendor/github.com/openfaas/templates-sdk/go-http/README.md deleted file mode 100644 index a4a5be7..0000000 --- a/template/golang-http/vendor/github.com/openfaas/templates-sdk/go-http/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# go-http SDK - -An SDK for building OpenFaaS functions in Go - -## Installing - -Use `go get` to retrieve the SDK to add it to your `GOPATH` workspace, or -project's Go module dependencies. - - go get github.com/openfaas/templates-sdk/go-http - -To update the SDK use `go get -u` to retrieve the latest version of the SDK. - - go get -u github.com/openfaas/templates-sdk/go-http - - -## Features - -### Handler definition - -```go -type FunctionHandler interface { - Handle(req Request) (Response, error) -} -``` - -`FunctionHandler` interface is used by [golang-http](https://github.com/openfaas-incubator/golang-http-template/tree/master/template/golang-http) template to define a functions handler - -### Secrets -For the time being please use the secrets function from `github.com/openfaas/openfaas-cloud/sdk` - -See: https://github.com/openfaas/openfaas-cloud/blob/master/sdk/secrets.go - -Usage: - -```go -secret, err := sdk.ReadSecret("MY_SECRET") -if err != nil { - return fmt.Errorf("error reading secret. %v", err) -} -``` diff --git a/template/golang-http/vendor/github.com/openfaas/templates-sdk/go-http/handler.go b/template/golang-http/vendor/github.com/openfaas/templates-sdk/go-http/handler.go deleted file mode 100644 index 4f5d376..0000000 --- a/template/golang-http/vendor/github.com/openfaas/templates-sdk/go-http/handler.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) Alex Ellis 2018. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. -package handler - -import ( - "context" - "net/http" -) - -// Response of function call -type Response struct { - - // Body the body will be written back - Body []byte - - // StatusCode needs to be populated with value such as http.StatusOK - StatusCode int - - // Header is optional and contains any additional headers the function response should set - Header http.Header -} - -// Request of function call -type Request struct { - Body []byte - Header http.Header - QueryString string - Method string - Host string - ctx context.Context -} - -// Context is set for optional cancellation inflight requests. -func (r *Request) Context() context.Context { - return r.ctx -} - -// WithContext overides the context for the Request struct -func (r *Request) WithContext(ctx context.Context) { - // AE: Not keen on panic mid-flow in user-code, however stdlib also appears to do - // this. https://golang.org/src/net/http/request.go - // This is not setting a precedent for broader use of "panic" to handle errors. - if ctx == nil { - panic("nil context") - } - r.ctx = ctx -} - -// FunctionHandler used for a serverless Go method invocation -type FunctionHandler interface { - Handle(req Request) (Response, error) -} - -func init() { - -} diff --git a/template/golang-http/vendor/modules.txt b/template/golang-http/vendor/modules.txt deleted file mode 100644 index 08e7770..0000000 --- a/template/golang-http/vendor/modules.txt +++ /dev/null @@ -1,2 +0,0 @@ -# github.com/openfaas/templates-sdk v0.0.0-20200723092016-0ebf61253625 -github.com/openfaas/templates-sdk/go-http diff --git a/template/golang-middleware/Dockerfile b/template/golang-middleware/Dockerfile index ffaa3c9..da43ac8 100644 --- a/template/golang-middleware/Dockerfile +++ b/template/golang-middleware/Dockerfile @@ -17,16 +17,22 @@ RUN mkdir -p /go/src/handler WORKDIR /go/src/handler COPY . . +ARG GO111MODULE="off" +ARG GOPROXY="" +ARG GOFLAGS="" + # Add user overrides to the root go.mod, which is the only place "replace" can be used RUN cat function/GO_REPLACE.txt >> ./go.mod || exit 0 +RUN if [ -d ./function/vendor ] && [ "${GO111MODULE}" == "off" ]; then \ + echo "moving handler vendor" && \ + mv -f ./function/vendor .; \ + else \ + echo "using modules or vendor not found "; \ + fi # Run a gofmt and exclude all vendored code. RUN test -z "$(gofmt -l $(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./function/vendor/*"))" || { echo "Run \"gofmt -s -w\" on your Golang code"; exit 1; } -ARG GO111MODULE="off" -ARG GOPROXY="" -ARG GOFLAGS="" - WORKDIR /go/src/handler/function RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go test ./... -cover @@ -39,7 +45,7 @@ FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine:3.13 # Add non root user and certs RUN apk --no-cache add ca-certificates \ && addgroup -S app && adduser -S -g app app -# Split instructions so that buildkit can run & cache +# Split instructions so that buildkit can run & cache # the previous command ahead of time. RUN mkdir -p /home/app \ && chown app /home/app From 60d2b6d1b599a116bb9523f4257f332a093f5730 Mon Sep 17 00:00:00 2001 From: Lucas Roesler Date: Sat, 20 Mar 2021 10:39:59 +0100 Subject: [PATCH 2/6] Remove go mod tidy **What** - Remove the tidy step because it is not needed and does not work as expected Signed-off-by: Lucas Roesler --- template/golang-http/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/golang-http/Dockerfile b/template/golang-http/Dockerfile index 9aeaf2d..8cb1f4c 100644 --- a/template/golang-http/Dockerfile +++ b/template/golang-http/Dockerfile @@ -25,7 +25,7 @@ ARG GOFLAGS="" RUN cat function/GO_REPLACE.txt >> ./go.mod || exit 0 RUN if [ -d ./function/vendor ]; then \ echo "moving handler vendor" && \ - mv -f ./function/vendor . && go mod tidy; \ + mv -f ./function/vendor .; \ else \ echo "using modules or vendor not found "; \ fi From 4fb83f2577aad8ff901b703abd81656ddf4444f1 Mon Sep 17 00:00:00 2001 From: Lucas Roesler Date: Sat, 20 Mar 2021 10:42:23 +0100 Subject: [PATCH 3/6] Remove unneeded module check **What** - Remove check for modules = off from the middleware template. This isn't needed Signed-off-by: Lucas Roesler --- template/golang-middleware/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/golang-middleware/Dockerfile b/template/golang-middleware/Dockerfile index da43ac8..11fd947 100644 --- a/template/golang-middleware/Dockerfile +++ b/template/golang-middleware/Dockerfile @@ -23,7 +23,7 @@ ARG GOFLAGS="" # Add user overrides to the root go.mod, which is the only place "replace" can be used RUN cat function/GO_REPLACE.txt >> ./go.mod || exit 0 -RUN if [ -d ./function/vendor ] && [ "${GO111MODULE}" == "off" ]; then \ +RUN if [ -d ./function/vendor ]; then \ echo "moving handler vendor" && \ mv -f ./function/vendor .; \ else \ From 0d4c9d09d015fa19802315e4e95b4c3598b462b8 Mon Sep 17 00:00:00 2001 From: Lucas Roesler Date: Sat, 27 Mar 2021 12:06:29 +0100 Subject: [PATCH 4/6] Add script to manage go.mod and modules.txt **What** - Add scripting that handles copying the function's go.mod to the function root. This requires several mutations to handle renaming the module and dealig with local replacements of the function - Add scriptin to handle the vendor/modules.txt. It also handles mutations to deal with the local replacement of the function code Signed-off-by: Lucas Roesler --- template/golang-http/Dockerfile | 38 +++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/template/golang-http/Dockerfile b/template/golang-http/Dockerfile index 8cb1f4c..a0e82ea 100644 --- a/template/golang-http/Dockerfile +++ b/template/golang-http/Dockerfile @@ -25,9 +25,43 @@ ARG GOFLAGS="" RUN cat function/GO_REPLACE.txt >> ./go.mod || exit 0 RUN if [ -d ./function/vendor ]; then \ echo "moving handler vendor" && \ - mv -f ./function/vendor .; \ + mv -f ./function/vendor .;\ else \ - echo "using modules or vendor not found "; \ + echo "vendor not found "; \ + fi + +RUN if [ "${GO111MODULE}" == 'on' ]; then \ + # copy the user's go.mod + mv -f ./function/go.mod . && \ + mv -f ./function/go.sum . && \ + # clean up the go.mod, + # remove any local require for the handler/function, this can exist _if_ the + # developer has subpackages in their handler. It is ok to just remove it + # because we will replace it later + grep -v "handler/function" go.mod > gomod2; mv gomod2 go.mod && \ + # now update the go.mod + # first, replace handler/function to point at the local code + # second, we need to rename the module to handler to match our main + go mod edit \ + -replace=handler/function=./function \ + -module handler && \ + cat go.mod && \ + if [ -d ./vendor ]; then \ + # when vendored, we need to do similar edits to the vendor/modules.txt + # first we need to replace any possible copy of the handler code + rm -rf vendor/handler && \ + # in modules.txt, we remove existing references to the handler/function + # these are replaced later with the new structure + grep -v "handler/function" ./vendor/modules.txt> modulestext; mv modulestext ./vendor/modules.txt && \ + # add the mising replace to the vendor/modules.txt + echo "## explicit" >> ./vendor/modules.txt && \ + echo "# handler/function => ./function" >> ./vendor/modules.txt && \ + cat ./vendor/modules.txt; \ + else \ + echo "skip adding replace to ' ./vendor/modules.txt'"; \ + fi \ + else \ + echo "skip go.mod handling"; \ fi # Run a gofmt and exclude all vendored code. From 31c59311bea15776da8a8a8edb5375c7d9fc70a7 Mon Sep 17 00:00:00 2001 From: Lucas Roesler Date: Mon, 5 Apr 2021 11:09:43 +0200 Subject: [PATCH 5/6] Extra modules cleanup into a script and document it **What** - Extract the vendor and modules cleanup into a separate shell script. This provides more space for documenting each step cleanly. - This scripting also removes the need for the `GO_REPLACE.txt` hack, it will automatically modify any local replacements in the original go.mod file, so that they work as expected. - Update the README to describe how dependencies work in the three supported modes: modules with vendoring, modules without vendoring, and traditional vendoring via dep. Signed-off-by: Lucas Roesler --- README.md | 42 ++++- template/golang-http/Dockerfile | 45 +----- template/golang-http/modules-cleanup.sh | 144 ++++++++++++++++++ template/golang-middleware/Dockerfile | 11 +- template/golang-middleware/modules-cleanup.sh | 144 ++++++++++++++++++ 5 files changed, 329 insertions(+), 57 deletions(-) create mode 100644 template/golang-http/modules-cleanup.sh create mode 100644 template/golang-middleware/modules-cleanup.sh diff --git a/README.md b/README.md index 7b40e7a..292ab39 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ The two templates are equivalent with `golang-http` using a structured request/r You can manage dependencies in one of the following ways: * To use Go modules without vendoring, add `--build-arg GO111MODULE=on` to `faas-cli up`, you can also use `--build-arg GOPROXY=https://` if you want to use your own mirror for the modules +* You can also Go modules with vendoring, run `go mod vendor` in your function folder and add `--build-arg GO111MODULE=on` to `faas-cli up` * For traditional vendoring with `dep` give no argument, or add `--build-arg GO111MODULE=off` to `faas-cli up` ## 1.0 golang-http @@ -361,7 +362,37 @@ func Handle(w http.ResponseWriter, r *http.Request) { } ``` -#### Advanced usage - Go sub-modules via `GO_REPLACE.txt` +#### Advanced usage + +##### Sub-packages + +It is often natural to organize your code into sub-packages, for example you may have a function with the following structure + +``` +./echo +├── go.mod +├── go.sum +├── handler.go +└── pkg + └── version + └── version.go +``` + +First update your go.mod file to replace `handler/function` with your local folder + +```go +go mod edit -replace=handler/function=./ +``` + +Now if you want to reference the`version` sub-package, import it as + +```go +import "handler/function/pkg/version" +``` + +This replacement is handled gracefully by the template at build time and your local development environment will now recognize the sub-package. + +##### Go sub-modules For this example you will need to be using Go 1.13 or newer and Go modules, enable this via `faas-cli build --build-arg GO111MODULE=on`. @@ -384,16 +415,13 @@ func Echo(w http.ResponseWriter, r *http.Request) { } ``` -To include a relative module such as this new `handlers` package, you should create a `GO_REPLACE.txt` file as follows: +To include a relative module such as this new `handlers` package, you should update your `go.mod` file as follows: ``` -replace github.com/alexellis/vault/purchase/handlers => ./function/handlers +go mod edit -replace=github.com/alexellis/vault/purchase/handlers=./handlers ``` -How did we get to that? Let's say your GOPATH for your GitHub repo is: `github.com/alexellis/vault/` and your OpenFaaS function is `purchase` generated by `faas-cli new purchase --lang golang-middleware`. - -Your relative GOPATH is: `github.com/alexellis/vault/purchase`, so add a redirect as per below to redirect the "handlers" package to where it exists in the build container. - +At build time, this relative path will be handled correctly inside the template. Now if you want to reference the handlers package from within your `handler.go` write the following: diff --git a/template/golang-http/Dockerfile b/template/golang-http/Dockerfile index a0e82ea..80b0579 100644 --- a/template/golang-http/Dockerfile +++ b/template/golang-http/Dockerfile @@ -20,49 +20,10 @@ COPY . . ARG GO111MODULE="off" ARG GOPROXY="" ARG GOFLAGS="" +ARG DEBUG=0 -# Add user overrides to the root go.mod, which is the only place "replace" can be used -RUN cat function/GO_REPLACE.txt >> ./go.mod || exit 0 -RUN if [ -d ./function/vendor ]; then \ - echo "moving handler vendor" && \ - mv -f ./function/vendor .;\ - else \ - echo "vendor not found "; \ - fi - -RUN if [ "${GO111MODULE}" == 'on' ]; then \ - # copy the user's go.mod - mv -f ./function/go.mod . && \ - mv -f ./function/go.sum . && \ - # clean up the go.mod, - # remove any local require for the handler/function, this can exist _if_ the - # developer has subpackages in their handler. It is ok to just remove it - # because we will replace it later - grep -v "handler/function" go.mod > gomod2; mv gomod2 go.mod && \ - # now update the go.mod - # first, replace handler/function to point at the local code - # second, we need to rename the module to handler to match our main - go mod edit \ - -replace=handler/function=./function \ - -module handler && \ - cat go.mod && \ - if [ -d ./vendor ]; then \ - # when vendored, we need to do similar edits to the vendor/modules.txt - # first we need to replace any possible copy of the handler code - rm -rf vendor/handler && \ - # in modules.txt, we remove existing references to the handler/function - # these are replaced later with the new structure - grep -v "handler/function" ./vendor/modules.txt> modulestext; mv modulestext ./vendor/modules.txt && \ - # add the mising replace to the vendor/modules.txt - echo "## explicit" >> ./vendor/modules.txt && \ - echo "# handler/function => ./function" >> ./vendor/modules.txt && \ - cat ./vendor/modules.txt; \ - else \ - echo "skip adding replace to ' ./vendor/modules.txt'"; \ - fi \ - else \ - echo "skip go.mod handling"; \ - fi +# Lift the vendor and go.mod to the main package, cleanup any relative references +RUN sh modules-cleanup.sh # Run a gofmt and exclude all vendored code. RUN test -z "$(gofmt -l $(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./function/vendor/*"))" || { echo "Run \"gofmt -s -w\" on your Golang code"; exit 1; } diff --git a/template/golang-http/modules-cleanup.sh b/template/golang-http/modules-cleanup.sh new file mode 100644 index 0000000..b729e0f --- /dev/null +++ b/template/golang-http/modules-cleanup.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env sh + +set -e + +GO111MODULE=$(go env GO111MODULE) + +# move_vendor will copy the function's vendor folder, +# if it exists. +move_vendor() { + if [ ! -d ./function/vendor ]; then + echo "vendor not found" + return + fi + + echo "moving function vendor" + mv -f ./function/vendor . +} + + +# cleanup_gomod will move the function's go module +cleanup_gomod() { + + # Nothing to do when modules is explicitly off + # the z prefix protects against any SH wonkiness + # see https://stackoverflow.com/a/18264223 + if [ "z$GO111MODULE" = "zoff" ]; then + echo "modules disabled, skipping go.mod cleanup" + return; + fi + + echo "cleaning up go.mod" + + # Copy the user's go.mod + mv -f ./function/go.mod . + mv -f ./function/go.sum . + + # Clean up the go.mod + + # Cleanup any sub-module replacements. + # This requires modifying any replace that points to "./*", + # the user has will use this to reference sub-modules instead + # of sub-packages, which we cleanup below. + echo "cleanup local replace statements" + # 1. Replace references to the local folder with `./function` + sed -i 's/=> \.\//=> \.\/function\//' go.mod + + + # Remove any references to the handler/function module. + # It is ok to just remove it because we will replace it later. + # + # Note that these references may or may not exist. We expect the + # go.mod to have a replace statement _if_ developer has subpackages + # in their handler. In this case they will need a this replace statement + # + # replace handler/function => ./ + # + # `go mod` will then add a line that looks like + # + # handler/function v0.0.0-00010101000000-000000000000 + # + # both of these lines need to be replaced, this grep selects everything + # _except_ those offending lines. + grep -v "\shandler/function" go.mod > gomod2; mv gomod2 go.mod + + # Now update the go.mod + # + # 1. use replace so that imports of handler/function use the local code + # 2. we need to rename the module to handler because our main.go assumes + # this is the package name + go mod edit \ + -replace=handler/function=./function \ + -module handler + + + + if [ "$DEBUG" -eq 1 ]; then + cat go.mod + echo "" + fi +} + + +# cleanup_vendor_modulestxt will cleanup the modules.txt file in the vendor folder +# this file is needed when modules are enabled and it must be in sync with the +# go.mod. To function correctly we need to modify the references to handler/function, +# if they exist. +cleanup_vendor_modulestxt() { + if [ ! -d ./vendor ]; then + echo "no vendor found, skipping modules.txt cleanup" + return + fi + + # Nothing to do when modules is explicitly off + # the z prefix protects against any SH wonkiness + # see https://stackoverflow.com/a/18264223 + if [ "z$GO111MODULE" = "zoff" ]; then + echo "modules disabled, skipping modules.txt cleanup" + return; + fi + + echo "cleanup vendor/modules.txt" + + # just in case + touch "./vendor/modules.txt" + + # when vendored, we need to do similar edits to the vendor/modules.txt + # as we did to the go.mod + + # 1. we need to replace any possible copy of the handler code + rm -rf vendor/handler && \ + + # 2. in modules.txt, we remove existing references to the handler/function + # we reconstruct these in the last step + grep -v "\shandler/function" ./vendor/modules.txt> modulestext; mv modulestext ./vendor/modules.txt + + # 3. Handle any other local replacements. + # any replace that points to `./**` needs to be udpat echo "cleanup local replace statements" + sed -i 's/=> \.\//=> \.\/function\//' ./vendor/modules.txt + + # 4. To make the modules.txt consistent with the new go.mod, + # we add the mising replace to the vendor/modules.txt + echo "## explicit" >> ./vendor/modules.txt + echo "# handler/function => ./function" >> ./vendor/modules.txt + + if [ "$DEBUG" -eq 1 ]; then + cat ./vendor/modules.txt; + echo "" + fi +} + +# has_local_replacement checks if the file contains local go module replacement +has_local_replacement() { + return "$(grep -E -c '=> \./\S+' "$1")" +} + + +################ +# main +################ +move_vendor + +cleanup_gomod + +cleanup_vendor_modulestxt \ No newline at end of file diff --git a/template/golang-middleware/Dockerfile b/template/golang-middleware/Dockerfile index 11fd947..91007f4 100644 --- a/template/golang-middleware/Dockerfile +++ b/template/golang-middleware/Dockerfile @@ -20,15 +20,10 @@ COPY . . ARG GO111MODULE="off" ARG GOPROXY="" ARG GOFLAGS="" +ARG DEBUG=0 -# Add user overrides to the root go.mod, which is the only place "replace" can be used -RUN cat function/GO_REPLACE.txt >> ./go.mod || exit 0 -RUN if [ -d ./function/vendor ]; then \ - echo "moving handler vendor" && \ - mv -f ./function/vendor .; \ - else \ - echo "using modules or vendor not found "; \ - fi +# Lift the vendor and go.mod to the main package, cleanup any relative references +RUN sh modules-cleanup.sh # Run a gofmt and exclude all vendored code. RUN test -z "$(gofmt -l $(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./function/vendor/*"))" || { echo "Run \"gofmt -s -w\" on your Golang code"; exit 1; } diff --git a/template/golang-middleware/modules-cleanup.sh b/template/golang-middleware/modules-cleanup.sh new file mode 100644 index 0000000..b729e0f --- /dev/null +++ b/template/golang-middleware/modules-cleanup.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env sh + +set -e + +GO111MODULE=$(go env GO111MODULE) + +# move_vendor will copy the function's vendor folder, +# if it exists. +move_vendor() { + if [ ! -d ./function/vendor ]; then + echo "vendor not found" + return + fi + + echo "moving function vendor" + mv -f ./function/vendor . +} + + +# cleanup_gomod will move the function's go module +cleanup_gomod() { + + # Nothing to do when modules is explicitly off + # the z prefix protects against any SH wonkiness + # see https://stackoverflow.com/a/18264223 + if [ "z$GO111MODULE" = "zoff" ]; then + echo "modules disabled, skipping go.mod cleanup" + return; + fi + + echo "cleaning up go.mod" + + # Copy the user's go.mod + mv -f ./function/go.mod . + mv -f ./function/go.sum . + + # Clean up the go.mod + + # Cleanup any sub-module replacements. + # This requires modifying any replace that points to "./*", + # the user has will use this to reference sub-modules instead + # of sub-packages, which we cleanup below. + echo "cleanup local replace statements" + # 1. Replace references to the local folder with `./function` + sed -i 's/=> \.\//=> \.\/function\//' go.mod + + + # Remove any references to the handler/function module. + # It is ok to just remove it because we will replace it later. + # + # Note that these references may or may not exist. We expect the + # go.mod to have a replace statement _if_ developer has subpackages + # in their handler. In this case they will need a this replace statement + # + # replace handler/function => ./ + # + # `go mod` will then add a line that looks like + # + # handler/function v0.0.0-00010101000000-000000000000 + # + # both of these lines need to be replaced, this grep selects everything + # _except_ those offending lines. + grep -v "\shandler/function" go.mod > gomod2; mv gomod2 go.mod + + # Now update the go.mod + # + # 1. use replace so that imports of handler/function use the local code + # 2. we need to rename the module to handler because our main.go assumes + # this is the package name + go mod edit \ + -replace=handler/function=./function \ + -module handler + + + + if [ "$DEBUG" -eq 1 ]; then + cat go.mod + echo "" + fi +} + + +# cleanup_vendor_modulestxt will cleanup the modules.txt file in the vendor folder +# this file is needed when modules are enabled and it must be in sync with the +# go.mod. To function correctly we need to modify the references to handler/function, +# if they exist. +cleanup_vendor_modulestxt() { + if [ ! -d ./vendor ]; then + echo "no vendor found, skipping modules.txt cleanup" + return + fi + + # Nothing to do when modules is explicitly off + # the z prefix protects against any SH wonkiness + # see https://stackoverflow.com/a/18264223 + if [ "z$GO111MODULE" = "zoff" ]; then + echo "modules disabled, skipping modules.txt cleanup" + return; + fi + + echo "cleanup vendor/modules.txt" + + # just in case + touch "./vendor/modules.txt" + + # when vendored, we need to do similar edits to the vendor/modules.txt + # as we did to the go.mod + + # 1. we need to replace any possible copy of the handler code + rm -rf vendor/handler && \ + + # 2. in modules.txt, we remove existing references to the handler/function + # we reconstruct these in the last step + grep -v "\shandler/function" ./vendor/modules.txt> modulestext; mv modulestext ./vendor/modules.txt + + # 3. Handle any other local replacements. + # any replace that points to `./**` needs to be udpat echo "cleanup local replace statements" + sed -i 's/=> \.\//=> \.\/function\//' ./vendor/modules.txt + + # 4. To make the modules.txt consistent with the new go.mod, + # we add the mising replace to the vendor/modules.txt + echo "## explicit" >> ./vendor/modules.txt + echo "# handler/function => ./function" >> ./vendor/modules.txt + + if [ "$DEBUG" -eq 1 ]; then + cat ./vendor/modules.txt; + echo "" + fi +} + +# has_local_replacement checks if the file contains local go module replacement +has_local_replacement() { + return "$(grep -E -c '=> \./\S+' "$1")" +} + + +################ +# main +################ +move_vendor + +cleanup_gomod + +cleanup_vendor_modulestxt \ No newline at end of file From 28697c37859c3c51094533d36a0ba077e8b0d225 Mon Sep 17 00:00:00 2001 From: Lucas Roesler Date: Mon, 5 Apr 2021 11:25:53 +0200 Subject: [PATCH 6/6] Handle case when function is not a module **What** - Add check and skip when the funciton folder is not a module Signed-off-by: Lucas Roesler --- template/golang-http/modules-cleanup.sh | 5 +++++ template/golang-middleware/modules-cleanup.sh | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/template/golang-http/modules-cleanup.sh b/template/golang-http/modules-cleanup.sh index b729e0f..fcc5354 100644 --- a/template/golang-http/modules-cleanup.sh +++ b/template/golang-http/modules-cleanup.sh @@ -28,6 +28,11 @@ cleanup_gomod() { return; fi + if [ ! -f ./function/go.mod ]; then + echo "module not initialized, skipping go.mod cleanup" + return; + fi + echo "cleaning up go.mod" # Copy the user's go.mod diff --git a/template/golang-middleware/modules-cleanup.sh b/template/golang-middleware/modules-cleanup.sh index b729e0f..ab5d458 100644 --- a/template/golang-middleware/modules-cleanup.sh +++ b/template/golang-middleware/modules-cleanup.sh @@ -28,6 +28,11 @@ cleanup_gomod() { return; fi + if [ ! -f ./function/go.mod ]; then + echo "module not initialized, skipping go.mod cleanup" + return; + fi + echo "cleaning up go.mod" # Copy the user's go.mod @@ -98,6 +103,11 @@ cleanup_vendor_modulestxt() { return; fi + if [ -f ./function/go.mod ]; then + echo "module not initialized, skipping go.mod cleanup" + return; + fi + echo "cleanup vendor/modules.txt" # just in case