Upgrade to go version 1.20.5 (#1196)

Brief list of changes

* Changes gcsfuse binary to be statically-linked, removes dependency
   on libc.
* Update tools/package_gcsfuse_docker/Dockerfile base to golang:1.20.5
* Switch from os.user.Current() to os.Getuid() and os.Getgid()
* Add CGO_ENABLED=0 to disable CGO to tools/build_gcsfuse and to
   all other relevant go build/run/test calls.
* Add unit tests for MyUserAndGroup

Other related changes.

.github/workflows/ci.yml

@@ -26,11 +26,11 @@ jobs:
       run: sudo apt-get update && sudo apt-get install -y fuse3 libfuse-dev
     - name: Build
       run: |
-        go build ./...
+        CGO_ENABLED=0 go build ./...
         go install ./tools/build_gcsfuse
         build_gcsfuse . /tmp ${GITHUB_SHA}
     - name: Test
-      run: go test -p 1 -count 1 -v -cover ./...
+      run: CGO_ENABLED=0 go test -p 1 -count 1 -v -cover ./...
   lint:
     name: Lint
     runs-on: ubuntu-20.04

Dockerfile

@@ -17,7 +17,7 @@
 # Mount the gcsfuse to /mnt/gcs:
 #  > docker run --privileged --device /fuse -v /mnt/gcs:/gcs:rw,rshared gcsfuse
 
-FROM golang:1.20.4-alpine as builder
+FROM golang:1.20.5-alpine as builder
 
 RUN apk add git
 

docs/troubleshooting.md

@@ -6,7 +6,7 @@ This page enumerates some common user facing issues around GCSFuse and also disc
 | Generic Mounting Issue                                                                                                                                                                               | Most of the common mount point issues are around permissions on both local mount point and the Cloud Storage bucket. It is highly recommended to retry with --foreground --debug_fuse --debug_fs --debug_gcs --debug_http flags which would provide much more detailed logs to understand the errors better and possibly provide a solution.                                                                                                                                                                                                                                                                           |
 | Mount successful but files not visible                                                                                                                                                               | Try mounting the gcsfuse with --implicit-dir flag. Read the [semantics](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md) to know the reasoning.                                                                                                                                                                                                                                                                                                                                                                                                                                           |
 | Mount failed with fusermount3 exit status 1                                                                                                                                                          | It comes when the bucket is already mounted in a folder and we try to mount it again. You need to unmount first and then remount.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| Mount failed with error: Current requires cgo or $USER set in environment                                                                                                                            | It comes when we try mounting by building the gcsfuse codebase. To fix this, build the gcsfuse package by enabling the CGO_ENABLED flag in the go env and then mount back.<ol type="a"><li> Check the current value using - ```go env``` command. </li><li>If it is unset, set this using - ```export CGO_ENABLED=1``` command. </li></ol>                                                                                                                                                                                                                                                                             |
+| version `GLIBC_x.yz` not found                                                                                                                                                                       | GCSFuse should not be linking to glibc. Please either `export CGO_ENABLED=0` in your environment or prefix `CGO_ENABLED=0` to any `go build\|run\|test` commands that you're invoking.                                                                                                                                                                                              |
 | Mount get stuck with error: DefaultTokenSource: google: could not find default credentials                                                                                                           | Run ```gcloud auth application-default login``` command to fetch default credentials to the VM. This will fetch the credentials to the following locations: <ol type="a"><li>For linux - $HOME/.config/gcloud/application_default_credentials.json</li><li>For windows - %APPDATA%/gcloud/applicateion_default_credentials.json </li></ol>                                                                                                                                                                                                                                                                             |
 | Input/Output Error                                                                                                                                                                                   | It’s a generic error, but the most probable culprit is the bucket not having the right permission for Cloud Storage FUSE to operate on. Ref - [here](https://stackoverflow.com/questions/36382704/gcsfuse-input-output-error)                                                                                                                                                                                                                                                                                                                                                                                          |
 | Generic NO_PUBKEY Error - while installing Cloud Storage FUSE on ubuntu 22.04                                                                                                                        | It happens while running - ```sudo apt-get update``` - working on installing Cloud Storage FUSE. You just have to add the pubkey you get in the error using the below command: ```sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys <PUBKEY> ``` And then try running ```sudo apt-get update```                                                                                                                                                                                                                                                                                                            |

internal/perms/perms.go

@@ -17,35 +17,23 @@ package perms
 
 import (
 	"fmt"
-	"os/user"
-	"strconv"
+	"os"
 )
 
 // MyUserAndGroup returns the UID and GID of this process.
 func MyUserAndGroup() (uid uint32, gid uint32, err error) {
-	// Ask for the current user.
-	user, err := user.Current()
-	if err != nil {
-		err = fmt.Errorf("Fetching current user: %w", err)
-		return
-	}
-
-	// Parse UID.
-	uid64, err := strconv.ParseUint(user.Uid, 10, 32)
-	if err != nil {
-		err = fmt.Errorf("Parsing UID (%s): %w", user.Uid, err)
-		return
-	}
+	// An improvement idea is to invoke os.current.User() and use its partial output
+	// even when the call itself returned error.
+	signed_uid := os.Getuid()
+	signed_gid := os.Getgid()
 
-	// Parse GID.
-	gid64, err := strconv.ParseUint(user.Gid, 10, 32)
-	if err != nil {
-		err = fmt.Errorf("Parsing GID (%s): %w", user.Gid, err)
+	if signed_gid == -1 || signed_uid == -1 {
+		err = fmt.Errorf("Failed to get uid/gid. UID = %d, GID = %d", signed_uid, signed_gid)
 		return
 	}
 
-	uid = uint32(uid64)
-	gid = uint32(gid64)
+	uid = uint32(signed_uid)
+	gid = uint32(signed_gid)
 
 	return
 }

internal/perms/perms_test.go

@@ -0,0 +1,40 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// System permissions-related code unit tests.
+package perms_test
+
+import (
+	"syscall"
+	"testing"
+
+	"github.com/googlecloudplatform/gcsfuse/internal/perms"
+)
+
+func TestMyUserAndGroup(t *testing.T) {
+	t.Run("Default", func(t *testing.T) {
+		if uid, gid, err := perms.MyUserAndGroup(); err != nil {
+			t.Errorf("got = %v,%v,%v; want = _,_,nil", uid, gid, err)
+		}
+	})
+
+	t.Run("ChangedUidGid", func(t *testing.T) {
+		new_uid, new_gid := 111, 117
+		if err1, err2 := syscall.Setuid(new_uid), syscall.Setgid(new_gid); err1 == nil && err2 == nil {
+			if uid, gid, err := perms.MyUserAndGroup(); uid != uint32(new_uid) || gid != uint32(new_gid) || err != nil {
+				t.Errorf("got = %v,%v,%v; want = %v,%v,nil", uid, gid, err, new_uid, new_gid)
+			}
+		}
+	})
+}

perfmetrics/scripts/compare_fuse_types_using_fio.py

@@ -61,7 +61,7 @@ def _install_gcsfuse_source(gcs_bucket, gcsfuse_flags) -> None:
   os.system(f'''git clone {GCSFUSE_REPO}
             mkdir gcs
             cd gcsfuse
-            go run . {gcsfuse_flags} {gcs_bucket} ../gcs
+            CGO_ENABLED=0 go run . {gcsfuse_flags} {gcs_bucket} ../gcs
             cd ..
             ''')
 

perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh

@@ -6,8 +6,8 @@ echo "Installing git"
 sudo apt-get install git
 echo "Installing pip"
 sudo apt-get install pip -y
-echo "Installing go-lang 1.20.4"
-wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz
+echo "Installing go-lang 1.20.5"
+wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz
 sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local
 export PATH=$PATH:/usr/local/go/bin
 echo "Installing fio"
@@ -28,7 +28,7 @@ commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H)
 git checkout $commitId
 
 echo "Executing integration tests"
-GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test
+GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test
 
 # Checkout back to master branch to use latest CI test scripts in master.
 git checkout master

perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 
-wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz
+# Install golang
+wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz
 rm -rf /usr/local/go && tar -C /usr/local -xzf go_tar.tar.gz
 export PATH=$PATH:/usr/local/go/bin
 
 # Clone and build the gcsfuse master branch.
 git clone https://github.com/GoogleCloudPlatform/gcsfuse.git
 cd gcsfuse
-go build .
+CGO_ENABLED=0 go build .
 cd -
 
 # Create a directory for gcsfuse logs

perfmetrics/scripts/ml_tests/run_image_recognition_models.py

@@ -105,7 +105,7 @@ def _run_from_source(gcs_bucket, data_directory_name) -> None:
   os.system(f'''mkdir {data_directory_name}
             git clone {GITHUB_REPO}
             cd gcsfuse
-            go run . --implicit-dirs --stat-cache-capacity 1000000 --max-conns-per-host 100 --stackdriver-export-interval=60s {gcs_bucket} ../{data_directory_name}
+            CGO_ENABLED=0 go run . --implicit-dirs --stat-cache-capacity 1000000 --max-conns-per-host 100 --stackdriver-export-interval=60s {gcs_bucket} ../{data_directory_name}
             cd ..
             ''')
 

perfmetrics/scripts/ml_tests/setup.sh

@@ -4,7 +4,7 @@
 # >> source setup.sh
 
 # Go version to be installed.
-GO_VERSION=go1.20.4.linux-amd64.tar.gz
+GO_VERSION=go1.20.5.linux-amd64.tar.gz
 
 # This function will install the given module/dependency if it's not alredy 
 # installed.

perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh

@@ -5,14 +5,14 @@
 # and epochs functionality, and runs the model
 
 # Install go lang
-wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz
+wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz
 sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local
 export PATH=$PATH:/usr/local/go/bin
 
 # Clone the repo and build gcsfuse
 git clone "https://github.com/GoogleCloudPlatform/gcsfuse.git"
 cd gcsfuse
-go build .
+CGO_ENABLED=0 go build .
 cd -
 
 # Mount the bucket and run in background so that docker doesn't keep running after resnet_runner.py fails

perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh

@@ -22,8 +22,8 @@ pip install google-cloud
 pip install google-cloud-vision
 pip install google-api-python-client
 pip install prettytable
-echo Installing go-lang  1.20.4
-wget -O go_tar.tar.gz https://go.dev/dl/go1.20.4.linux-amd64.tar.gz
+echo Installing go-lang  1.20.5
+wget -O go_tar.tar.gz https://go.dev/dl/go1.20.5.linux-amd64.tar.gz
 sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local
 export PATH=$PATH:/usr/local/go/bin
 echo Installing fio
@@ -38,7 +38,7 @@ GCSFUSE_FLAGS="--implicit-dirs --max-conns-per-host 100"
 BUCKET_NAME=presubmit-perf-tests
 MOUNT_POINT=gcs
 # The VM will itself exit if the gcsfuse mount fails.
-go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT
+CGO_ENABLED=0 go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT
 touch result.txt
 # Running FIO test
 chmod +x perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh
@@ -53,13 +53,13 @@ echo checkout PR branch
 git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER
 
 # Executing integration tests
-GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test
+GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test
 
 # Executing perf tests
 echo Mounting gcs bucket from pr branch
 mkdir -p gcs
 # The VM will itself exit if the gcsfuse mount fails.
-go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT
+CGO_ENABLED=0 go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT
 # Running FIO test
 chmod +x perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh
 ./perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh

tools/build_gcsfuse/main.go

@@ -158,6 +158,7 @@ func buildBinaries(dstDir, srcDir, version string, buildArgs []string) (err erro
 			fmt.Sprintf("GOROOT=%s", runtime.GOROOT()),
 			fmt.Sprintf("GOPATH=%s", gopath),
 			fmt.Sprintf("GOCACHE=%s", gocache),
+			"CGO_ENABLED=0",
 		}
 
 		// Build.

tools/cd_scripts/e2e_test.sh

@@ -108,7 +108,7 @@ git checkout $(sed -n 2p ~/details.txt) |& tee -a ~/logs.txt
 
 #run tests with testbucket flag
 set +e
-GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=$(sed -n 3p ~/details.txt) --testInstalledPackage --timeout=60m &>> ~/logs.txt
+GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=$(sed -n 3p ~/details.txt) --testInstalledPackage --timeout=60m &>> ~/logs.txt
 
 if [ $? -ne 0 ];
 then

tools/containerize_gcsfuse_docker/Dockerfile

@@ -34,7 +34,7 @@ ARG OS_VERSION
 ARG OS_NAME
 
 # Image with gcsfuse installed and its package (.deb)
-FROM golang:1.20.4 as gcsfuse-package
+FROM golang:1.20.5 as gcsfuse-package
 
 RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm fuse && gem install --no-document bundler
 

tools/integration_tests/run_tests_mounted_directory.sh

@@ -21,6 +21,7 @@
 
 TEST_BUCKET_NAME=$1
 MOUNT_DIR=$2
+export CGO_ENABLED=0
 
 # Run integration tests for operations directory with static mounting
 gcsfuse --enable-storage-client-library=true --implicit-dirs=true $TEST_BUCKET_NAME $MOUNT_DIR

tools/package_gcsfuse_docker/Dockerfile

@@ -17,7 +17,7 @@
 # Copy the gcsfuse packages to the host:
 #   > docker run -it -v /tmp:/output gcsfuse-release cp -r /packages /output
 
-FROM golang:1.20.4 as builder
+FROM golang:1.20.5 as builder
 
 RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm && gem install --no-document bundler