Merge pull request #37 from tsmetana/rebase-v.1.10.1

STOR-1163: Rebase to upstream v1.10.1

CHANGELOG.md

@@ -1 +1 @@
-See the [1.8 changelog in the master branch](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/blob/master/CHANGELOG/CHANGELOG-1.8.md).
+See the [1.10 changelog in the master branch](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/blob/master/CHANGELOG/CHANGELOG-1.10.md).

Dockerfile

@@ -14,7 +14,7 @@
 
 ARG BUILDPLATFORM
 
-FROM --platform=$BUILDPLATFORM golang:1.19.4 as builder
+FROM --platform=$BUILDPLATFORM golang:1.20.3 as builder
 
 ARG STAGINGVERSION
 ARG TARGETPLATFORM
@@ -24,7 +24,7 @@ ADD . .
 RUN GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') GCE_PD_CSI_STAGING_VERSION=$STAGINGVERSION make gce-pd-driver
 
 # Start from Kubernetes Debian base.
-FROM k8s.gcr.io/build-image/debian-base:buster-v1.9.0 as debian
+FROM gke.gcr.io/debian-base:bullseye-v1.4.3-gke.0 as debian
 # Install necessary dependencies
 # google_nvme_id script depends on the following packages: nvme-cli, xxd, bash
 RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash
@@ -74,28 +74,33 @@ COPY --from=debian /bin/ln /bin/ln
 COPY --from=debian /bin/udevadm /bin/udevadm
 
 # Copy shared libraries into distroless base.
-COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \
+COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre.so.3 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libtinfo.so.6 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libe2p.so.2 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libcom_err.so.2 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libdevmapper.so.1.02.1 \
-                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libe2p.so.2 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libext2fs.so.2 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libgcc_s.so.1 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/liblzma.so.5 \
-                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libmount.so.1 \
-                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre.so.3 \
-                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libreadline.so.5 \
-                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \
-                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libtinfo.so.6 \
-                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libudev.so.1 \
-                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libuuid.so.1 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libreadline.so.8 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libz.so.1 /lib/${LIB_DIR_PREFIX}-linux-gnu/
 
-COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libacl.so.1 \
+COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libbsd.so.0 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libinih.so.1 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmount.so.1 \         
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libudev.so.1 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libuuid.so.1 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libacl.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libattr.so.1 \
-                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicudata.so.63 \
-                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicui18n.so.63 \
-                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicuuc.so.63 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libedit.so.2 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicudata.so.67 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicui18n.so.67 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libicuuc.so.67 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libkmod.so.2 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmd.so.0 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre2-8.so.0 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libstdc++.so.6 /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/
 
 # Copy NVME support required script and rules into distroless base.

Dockerfile.Windows

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG BASE_IMAGE
-FROM --platform=$BUILDPLATFORM golang:1.19.4 AS builder
+FROM --platform=$BUILDPLATFORM golang:1.19.6 AS builder
 
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM

Dockerfile.debug

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.19.4 as builder
+FROM golang:1.19.6 as builder
 WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 ADD . .
 
@@ -33,6 +33,7 @@ FROM k8s.gcr.io/build-image/debian-base-amd64:buster-v1.5.0
 # Copy source code too to correlate the binary and the breakpoints
 WORKDIR /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver
 ADD . .
+COPY --from=builder /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/bin/gce-pd-csi-driver
 
 COPY --from=builder /go/bin/dlv /go/bin/dlv
 

Makefile

@@ -24,11 +24,10 @@ DRIVERWINDOWSBINARY=${DRIVERBINARY}.exe
 DOCKER=DOCKER_CLI_EXPERIMENTAL=enabled docker
 
 BASE_IMAGE_LTSC2019=mcr.microsoft.com/windows/servercore:ltsc2019
-BASE_IMAGE_20H2=mcr.microsoft.com/windows/servercore:20H2
 
 # Both arrays MUST be index aligned.
-WINDOWS_IMAGE_TAGS=ltsc2019 20H2
-WINDOWS_BASE_IMAGES=$(BASE_IMAGE_LTSC2019) $(BASE_IMAGE_20H2)
+WINDOWS_IMAGE_TAGS=ltsc2019
+WINDOWS_BASE_IMAGES=$(BASE_IMAGE_LTSC2019)
 
 GCFLAGS=""
 ifdef GCE_PD_CSI_DEBUG
@@ -38,7 +37,7 @@ endif
 all: gce-pd-driver gce-pd-driver-windows
 gce-pd-driver: require-GCE_PD_CSI_STAGING_VERSION
 	mkdir -p bin
-	go build -mod=vendor -gcflags=$(GCFLAGS) -ldflags "-X main.version=$(STAGINGVERSION)" -o bin/${DRIVERBINARY} ./cmd/gce-pd-csi-driver/
+	CGO_ENABLED=0 go build -mod=vendor -gcflags=$(GCFLAGS) -ldflags "-extldflags=static -X main.version=$(STAGINGVERSION)" -o bin/${DRIVERBINARY} ./cmd/gce-pd-csi-driver/
 
 gce-pd-driver-windows: require-GCE_PD_CSI_STAGING_VERSION
 ifeq (${GOARCH}, amd64)
@@ -61,14 +60,8 @@ build-and-push-windows-container-ltsc2019: require-GCE_PD_CSI_STAGING_IMAGE init
 		--build-arg BASE_IMAGE=$(BASE_IMAGE_LTSC2019) \
 		--build-arg STAGINGVERSION=$(STAGINGVERSION) --push .
 
-build-and-push-windows-container-20H2: require-GCE_PD_CSI_STAGING_IMAGE init-buildx
-	$(DOCKER) buildx build --file=Dockerfile.Windows --platform=windows \
-		-t $(STAGINGIMAGE):$(STAGINGVERSION)_20H2 \
-		--build-arg BASE_IMAGE=$(BASE_IMAGE_20H2) \
-		--build-arg STAGINGVERSION=$(STAGINGVERSION) --push .
-
-build-and-push-multi-arch: build-and-push-container-linux-amd64 build-and-push-container-linux-arm64 build-and-push-windows-container-ltsc2019 build-and-push-windows-container-20H2
-	$(DOCKER) manifest create --amend $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux_amd64 $(STAGINGIMAGE):$(STAGINGVERSION)_linux_arm64 $(STAGINGIMAGE):$(STAGINGVERSION)_20H2 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019
+build-and-push-multi-arch: build-and-push-container-linux-amd64 build-and-push-windows-container-ltsc2019
+	$(DOCKER) manifest create --amend $(STAGINGIMAGE):$(STAGINGVERSION) $(STAGINGIMAGE):$(STAGINGVERSION)_linux_amd64 $(STAGINGIMAGE):$(STAGINGVERSION)_ltsc2019
 	STAGINGIMAGE="$(STAGINGIMAGE)" STAGINGVERSION="$(STAGINGVERSION)" WINDOWS_IMAGE_TAGS="$(WINDOWS_IMAGE_TAGS)" WINDOWS_BASE_IMAGES="$(WINDOWS_BASE_IMAGES)" ./manifest_osversion.sh
 	$(DOCKER) manifest push -p $(STAGINGIMAGE):$(STAGINGVERSION)
 
@@ -130,10 +123,10 @@ ifndef GCE_PD_CSI_STAGING_VERSION
 endif
 
 init-buildx:
+	$(DOCKER) run --rm --privileged multiarch/qemu-user-static --reset --credential yes --persistent yes
 	# Ensure we use a builder that can leverage it (the default on linux will not)
 	-$(DOCKER) buildx rm multiarch-multiplatform-builder
-	$(DOCKER) buildx create --use --name=multiarch-multiplatform-builder
-	$(DOCKER) run --rm --privileged multiarch/qemu-user-static --reset --credential yes --persistent yes
+	$(DOCKER) buildx create --use --name=multiarch-multiplatform-builder --driver-opt network=host --driver-opt image=moby/buildkit:v0.10.6
 	# Register gcloud as a Docker credential helper.
 	# Required for "docker buildx build --push".
 	gcloud auth configure-docker --quiet

README.md

@@ -12,7 +12,7 @@ lifecycle of Google Compute Engine Persistent Disks.
 ## Project Status
 
 Status: GA
-Latest stable image: `k8s.gcr.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.7.2`
+Latest stable image: `k8s.gcr.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.9.2`
 
 ### Test Status
 
@@ -40,7 +40,7 @@ The following table captures the compatibility matrix of the core persistent dis
 | dev                                  | yes   |
 
 The manifest bundle which captures all the driver components (driver pod which includes the containers csi-provisioner, csi-resizer, csi-snapshotter, gce-pd-driver, csi-driver-registrar;
-csi driver object, rbacs, pod security policies etc) can be picked up from the master branch [overlays](deploy/kubernetes/overlays) directory. We structure the overlays directory, per minor version of kubernetes because not all driver components can be used with all kubernetes versions. 
+csi driver object, rbacs, pod security policies etc) can be picked up from the master branch [overlays](deploy/kubernetes/overlays) directory. We structure the overlays directory, per minor version of kubernetes because not all driver components can be used with all kubernetes versions.
 
 Example:
 
@@ -58,12 +58,14 @@ See Github [Issues](https://github.com/kubernetes-sigs/gcp-compute-persistent-di
 
 ### CreateVolume Parameters
 
-| Parameter        | Values                    | Default       | Description                                                                                        |
-|------------------|---------------------------|---------------|----------------------------------------------------------------------------------------------------|
-| type             | Any PD type (see [GCP documentation](https://cloud.google.com/compute/docs/disks#disk-types)), eg `pd-ssd` `pd-balanced` | `pd-standard` | Type allows you to choose between standard Persistent Disks  or Solid State Drive Persistent Disks |
-| replication-type | `none` OR `regional-pd`   | `none`        | Replication type allows you to choose between Zonal Persistent Disks or Regional Persistent Disks  |
-| disk-encryption-kms-key | Fully qualified resource identifier for the key to use to encrypt new disks. | Empty string. | Encrypt disk using Customer Managed Encryption Key (CMEK). See [GKE Docs](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek#create_a_cmek_protected_attached_disk) for details. |
-| labels           | `key1=value1,key2=value2` |               | Labels allow you to assign custom [GCE Disk labels](https://cloud.google.com/compute/docs/labeling-resources). |
+| Parameter                   | Values                    | Default       | Description                                                                                        |
+|-----------------------------|---------------------------|---------------|----------------------------------------------------------------------------------------------------|
+| type                        | Any PD type (see [GCP documentation](https://cloud.google.com/compute/docs/disks#disk-types)), eg `pd-ssd` `pd-balanced` | `pd-standard` | Type allows you to choose between standard Persistent Disks  or Solid State Drive Persistent Disks |
+| replication-type            | `none` OR `regional-pd`   | `none`        | Replication type allows you to choose between Zonal Persistent Disks or Regional Persistent Disks  |
+| disk-encryption-kms-key     | Fully qualified resource identifier for the key to use to encrypt new disks. | Empty string. | Encrypt disk using Customer Managed Encryption Key (CMEK). See [GKE Docs](https://cloud.google.com/kubernetes-engine/docs/how-to/using-cmek#create_a_cmek_protected_attached_disk) for details. |
+| labels                      | `key1=value1,key2=value2` |               | Labels allow you to assign custom [GCE Disk labels](https://cloud.google.com/compute/docs/labeling-resources). |
+| provisioned-iops-on-create  | string (int64 format). Values typically between 10,000 and 120,000 |               | Indicates how many IOPS to provision for the disk. See the [Extreme persistent disk documentation](https://cloud.google.com/compute/docs/disks/extreme-persistent-disk) for details, including valid ranges for IOPS. |
+| provisioned-throughput-on-create  | string (int64 format). Values typically between 1 and 7,124 mb per second |               | Indicates how much throughput to provision for the disk. See the [hyperdisk documentation](TBD) for details, including valid ranges for throughput. |
 
 ### Topology
 
@@ -97,7 +99,7 @@ Controller-level and node-level deployments will both have priorityClassName set
 
 ## Further Documentation
 
-[Local Development](docs/local-development.md)
+[Local Development](docs/kubernetes/development.md)
 
 For releasing new versions of this driver, googlers should consult [go/pdcsi-oss-release-process](go/pdcsi-oss-release-process).
 

cloudbuild.yaml

@@ -6,14 +6,14 @@ options:
   substitution_option: ALLOW_LOOSE
 
 steps:
-  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583'
+  - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20221214-1b4dd4d69a'
     entrypoint: make
     env:
-    - GCE_PD_CSI_STAGING_IMAGE=gcr.io/${_STAGING_PROJECT}/gcp-compute-persistent-disk-csi-driver
-    - GCE_PD_CSI_STAGING_VERSION=${_PULL_BASE_REF}
-    # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx
-    # set the home to /root explicitly to if using docker buildx
-    - HOME=/root
+      - GCE_PD_CSI_STAGING_IMAGE=gcr.io/${_STAGING_PROJECT}/gcp-compute-persistent-disk-csi-driver
+      - GCE_PD_CSI_STAGING_VERSION=${_PULL_BASE_REF}
+      # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx
+      # set the home to /root explicitly to if using docker buildx
+      - HOME=/root
     args:
       - build-and-push-multi-arch
 
@@ -22,5 +22,5 @@ substitutions:
   _PULL_BASE_REF: 'master'
 
 tags:
-- 'gcp-compute-persistent-disk-csi-driver'
-- ${_PULL_BASE_REF}
+  - 'gcp-compute-persistent-disk-csi-driver'
+  - ${_PULL_BASE_REF}

cmd/gce-pd-csi-driver/main.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
+	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils"
 	gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
 	metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata"
 	driver "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-pd-csi-driver"
@@ -36,6 +37,7 @@ import (
 var (
 	cloudConfigFilePath  = flag.String("cloud-config", "", "Path to GCE cloud provider config")
 	endpoint             = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint")
+	computeEndpoint      = flag.String("compute-endpoint", "", "If set, used as the endpoint for the GCE API.")
 	runControllerService = flag.Bool("run-controller-service", true, "If set to false then the CSI driver does not activate its controller service (default: true)")
 	runNodeService       = flag.Bool("run-node-service", true, "If set to false then the CSI driver does not activate its node service (default: true)")
 	httpEndpoint         = flag.String("http-endpoint", "", "The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`). The default is empty string, which means metrics endpoint is disabled.")
@@ -57,7 +59,9 @@ var (
 	waitForOpBackoffSteps     = flag.Int("wait-op-backoff-steps", 100, "Steps for wait for operation backoff")
 	waitForOpBackoffCap       = flag.Duration("wait-op-backoff-cap", 0, "Cap for wait for operation backoff")
 
-	maxprocs = flag.Int("maxprocs", 1, "GOMAXPROCS override")
+	maxProcs                = flag.Int("maxprocs", 1, "GOMAXPROCS override")
+	maxConcurrentFormat     = flag.Int("max-concurrent-format", 1, "The maximum number of concurrent format exec calls")
+	concurrentFormatTimeout = flag.Duration("concurrent-format-timeout", 1*time.Minute, "The maximum duration of a format operation before its concurrency token is released")
 
 	version string
 )
@@ -86,7 +90,7 @@ func main() {
 func handle() {
 	var err error
 
-	runtime.GOMAXPROCS(*maxprocs)
+	runtime.GOMAXPROCS(*maxProcs)
 	klog.Infof("Sys info: NumCPU: %v MAXPROC: %v", runtime.NumCPU(), runtime.GOMAXPROCS(0))
 
 	if version == "" {
@@ -108,37 +112,37 @@ func handle() {
 		klog.Fatalf("Bad extra volume labels: %v", err.Error())
 	}
 
-	gceDriver := driver.GetGCEDriver()
-
-	//Initialize GCE Driver
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	//Initialize identity server
+	// Initialize driver
+	gceDriver := driver.GetGCEDriver()
+
+	// Initialize identity server
 	identityServer := driver.NewIdentityServer(gceDriver)
 
-	//Initialize requirements for the controller service
+	// Initialize requirements for the controller service
 	var controllerServer *driver.GCEControllerServer
 	if *runControllerService {
-		cloudProvider, err := gce.CreateCloudProvider(ctx, version, *cloudConfigFilePath)
+		cloudProvider, err := gce.CreateCloudProvider(ctx, version, *cloudConfigFilePath, *computeEndpoint)
 		if err != nil {
 			klog.Fatalf("Failed to get cloud provider: %v", err.Error())
 		}
 		initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond
-		maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Microsecond
+		maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond
 		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration)
 	} else if *cloudConfigFilePath != "" {
 		klog.Warningf("controller service is disabled but cloud config given - it has no effect")
 	}
 
-	//Initialize requirements for the node service
+	// Initialize requirements for the node service
 	var nodeServer *driver.GCENodeServer
 	if *runNodeService {
-		mounter, err := mountmanager.NewSafeMounter()
+		mounter, err := mountmanager.NewSafeMounter(*maxConcurrentFormat, *concurrentFormatTimeout)
 		if err != nil {
 			klog.Fatalf("Failed to get safe mounter: %v", err.Error())
 		}
-		deviceUtils := mountmanager.NewDeviceUtils()
+		deviceUtils := deviceutils.NewDeviceUtils()
 		statter := mountmanager.NewStatter(mounter)
 		meta, err := metadataservice.NewMetadataService()
 		if err != nil {

deploy/common.sh

@@ -19,7 +19,7 @@ function ensure_var(){
 
 function get_needed_roles()
 {
-	echo "roles/compute.storageAdmin roles/iam.serviceAccountUser projects/${PROJECT}/roles/gcp_compute_persistent_disk_csi_driver_custom_role"
+	echo "roles/editor roles/compute.storageAdmin roles/iam.serviceAccountUser projects/${PROJECT}/roles/gcp_compute_persistent_disk_csi_driver_custom_role"
 }
 
 # Installs kustomize in ${PKGDIR}/bin

deploy/kubernetes/base/controller/kustomization.yaml

@@ -6,3 +6,4 @@ resources:
 - cluster_setup.yaml
 - controller.yaml
 - v1_csidriver.yaml
+