Running all the integration test with --only-dir mounting

perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh

@@ -28,7 +28,7 @@ commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H)
 git checkout $commitId
 
 echo "Executing integration tests"
-GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test
+GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test -timeout=20m
 
 # Checkout back to master branch to use latest CI test scripts in master.
 git checkout master

perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh

@@ -52,7 +52,7 @@ echo checkout PR branch
 git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER
 
 # Executing integration tests
-GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test
+GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/... -p 1 --integrationTest -v --testbucket=gcsfuse-integration-test -timeout=20m
 
 # Executing perf tests
 echo Mounting gcs bucket from pr branch

tools/integration_tests/operations/operations_test.go

@@ -20,6 +20,8 @@ import (
 	"os"
 	"testing"
 
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/only_dir_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting"
 	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
 )
 
@@ -34,12 +36,18 @@ func TestMain(m *testing.M) {
 		{"--implicit-dirs=true"},
 		{"--implicit-dirs=false"}}
 
+	setup.CheckIfBothTheFlagsAreEnabled()
+
 	if setup.TestBucket() != "" && setup.MountedDirectory() != "" {
 		log.Printf("Both --testbucket and --mountedDirectory can't be specified at the same time.")
 		os.Exit(1)
 	}
 
-	successCode := setup.RunTests(flags, m)
+	successCode := static_mounting.RunTests(flags, m)
+
+	if successCode == 0 {
+		successCode = only_dir_mounting.RunTests(flags, m)
+	}
 
 	os.Exit(successCode)
 }

tools/integration_tests/readonly/readonly_test.go

@@ -18,10 +18,10 @@ package readonly_test
 import (
 	"log"
 	"os"
-	"os/exec"
 	"strings"
 	"testing"
 
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting"
 	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
 )
 
@@ -41,15 +41,6 @@ const ContentInFileInSubDirectoryTestBucket = "This is from directory Test/b fil
 const RenameFile = "rename.txt"
 const RenameDir = "rename"
 
-// Run shell script
-func runScriptForTestData(script string, testBucket string) {
-	cmd := exec.Command("/bin/bash", script, testBucket)
-	_, err := cmd.Output()
-	if err != nil {
-		panic(err)
-	}
-}
-
 func checkErrorForReadOnlyFileSystem(err error, t *testing.T) {
 	if !strings.Contains(err.Error(), "read-only file system") && !strings.Contains(err.Error(), "permission denied") {
 		t.Errorf("Incorrect error for readonly filesystem: %v", err.Error())
@@ -67,21 +58,23 @@ func TestMain(m *testing.M) {
 
 	flags := [][]string{{"--o=ro", "--implicit-dirs=true"}, {"--file-mode=544", "--dir-mode=544", "--implicit-dirs=true"}}
 
+	setup.CheckIfBothTheFlagsAreEnabled()
+
 	if setup.TestBucket() == "" && setup.MountedDirectory() != "" {
 		log.Printf("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.")
 		os.Exit(1)
 	}
 
 	// Clean the bucket for readonly testing.
-	runScriptForTestData("testdata/delete_objects.sh", setup.TestBucket())
+	setup.RunScriptForTestData("testdata/delete_objects.sh", setup.TestBucket())
 
 	// Create objects in bucket for testing.
-	runScriptForTestData("testdata/create_objects.sh", setup.TestBucket())
+	setup.RunScriptForTestData("testdata/create_objects.sh", setup.TestBucket())
 
-	successCode := setup.RunTests(flags, m)
+	successCode := static_mounting.RunTests(flags, m)
 
 	// Delete objects from bucket after testing.
-	runScriptForTestData("testdata/delete_objects.sh", setup.TestBucket())
+	setup.RunScriptForTestData("testdata/delete_objects.sh", setup.TestBucket())
 
 	os.Exit(successCode)
 }

tools/integration_tests/readonly/testdata/create_objects.sh

@@ -1,3 +1,17 @@
+# Copyright 2023 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Here $1 refers to the testBucket argument
 echo "This is from directory Test file a" >> a.txt
 gsutil mv a.txt gs://$1/Test/

tools/integration_tests/readonly/testdata/delete_objects.sh

@@ -1,3 +1,17 @@
+# Copyright 2023 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Here $1 refers to the testBucket argument
 gsutil rm -a gs://$1/**
 

tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go

@@ -20,6 +20,8 @@ import (
 	"os"
 	"testing"
 
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/only_dir_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting"
 	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
 )
 
@@ -37,12 +39,18 @@ func TestMain(m *testing.M) {
 
 	flags := [][]string{{"--rename-dir-limit=3", "--implicit-dirs"}, {"--rename-dir-limit=3"}}
 
+	setup.CheckIfBothTheFlagsAreEnabled()
+
 	if setup.TestBucket() != "" && setup.MountedDirectory() != "" {
 		log.Printf("Both --testbucket and --mountedDirectory can't be specified at the same time.")
 		os.Exit(1)
 	}
 
-	successCode := setup.RunTests(flags, m)
+	successCode := static_mounting.RunTests(flags, m)
+
+	if successCode == 0 {
+		successCode = only_dir_mounting.RunTests(flags, m)
+	}
 
 	os.Exit(successCode)
 }

tools/integration_tests/util/mounting/mounting.go

@@ -0,0 +1,58 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mounting
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+func MountGcsfuse(defaultArg []string, flags []string) error {
+	for i := 0; i < len(defaultArg); i++ {
+		flags = append(flags, defaultArg[i])
+	}
+
+	mountCmd := exec.Command(
+		setup.BinFile(),
+		flags...,
+	)
+
+	// Adding mount command in LogFile
+	file, err := os.OpenFile(setup.LogFile(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		fmt.Println("Could not open logfile")
+	}
+	defer file.Close()
+
+	_, err = file.WriteString(mountCmd.String() + "\n")
+	if err != nil {
+		fmt.Println("Could not write cmd to logFile")
+	}
+
+	output, err := mountCmd.CombinedOutput()
+	if err != nil {
+		log.Println(mountCmd.String())
+		return fmt.Errorf("cannot mount gcsfuse: %w\n", err)
+	}
+	if lines := bytes.Count(output, []byte{'\n'}); lines > 1 {
+		return fmt.Errorf("mount output: %q\n", output)
+	}
+	return nil
+}

tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go

@@ -0,0 +1,92 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package only_dir_mounting
+
+import (
+	"fmt"
+	"log"
+	"path"
+	"testing"
+
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+const DirectoryInTestBucket = "Test"
+
+func mountGcsfuseWithOnlyDir(flags []string, dir string) (err error) {
+	defaultArg := []string{"--only-dir",
+		dir,
+		"--debug_gcs",
+		"--debug_fs",
+		"--debug_fuse",
+		"--log-file=" + setup.LogFile(),
+		"--log-format=text",
+		setup.TestBucket(),
+		setup.MntDir()}
+
+	err = mounting.MountGcsfuse(defaultArg, flags)
+
+	return err
+}
+
+func mountGcsFuseForFlagsAndExecuteTests(flags [][]string, dir string, m *testing.M) (successCode int) {
+	for i := 0; i < len(flags); i++ {
+		if err := mountGcsfuseWithOnlyDir(flags[i], dir); err != nil {
+			setup.LogAndExit(fmt.Sprintf("mountGcsfuse: %v\n", err))
+		}
+		successCode = setup.ExecuteTestForFlagsSet(flags[i], m)
+	}
+	return
+}
+
+func executeTests(flags [][]string, m *testing.M) (successCode int) {
+	setup.SetUpTestDirForTestBucketFlag()
+
+	mountDirInBucket := path.Join(setup.TestBucket(), DirectoryInTestBucket)
+	// Clean the bucket.
+
+	setup.RunScriptForTestData("../util/mounting/only_dir_mounting/testdata/delete_objects.sh", setup.TestBucket())
+
+	// "Test" directory not exist in bucket.
+	mountGcsFuseForFlagsAndExecuteTests(flags, DirectoryInTestBucket, m)
+
+	// "Test" directory exist in bucket.
+	// Clean the bucket.
+	setup.RunScriptForTestData("../util/mounting/only_dir_mounting/testdata/delete_objects.sh", setup.TestBucket())
+
+	// Create Test directory in bucket.
+	setup.RunScriptForTestData("../util/mounting/only_dir_mounting/testdata/create_objects.sh", mountDirInBucket)
+
+	successCode = mountGcsFuseForFlagsAndExecuteTests(flags, DirectoryInTestBucket, m)
+
+	// Clean the bucket after testing.
+	setup.RunScriptForTestData("../util/mounting/only_dir_mounting/testdata/delete_objects.sh", setup.TestBucket())
+
+	return
+}
+
+func RunTests(flags [][]string, m *testing.M) (successCode int) {
+	setup.ParseSetUpFlags()
+
+	setup.RunTestsForMountedDirectoryFlag(m)
+
+	// Execute tests for testBucket
+	successCode = executeTests(flags, m)
+
+	log.Printf("Test log: %s\n", setup.LogFile())
+
+	return successCode
+}

tools/integration_tests/util/mounting/only_dir_mounting/testdata/create_objects.sh

@@ -0,0 +1,16 @@
+# Copyright 2023 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+touch a.txt
+gsutil mv a.txt gs://$1/

tools/integration_tests/util/mounting/only_dir_mounting/testdata/delete_objects.sh

@@ -0,0 +1,22 @@
+# Copyright 2023 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Here $1 refers to the testBucket argument
+gsutil rm -a gs://$1/**
+
+# If bucket is empty it will throw an CommandException.
+if [ $? -eq 1 ]; then
+  echo "Bucket is already empty."
+  exit 0
+fi