Skip to content

Commit

Permalink
Merge pull request #43 from Fedosin/multiaz
Browse files Browse the repository at this point in the history
Bug 1936871: support clouds with multiple availability zones
  • Loading branch information
openshift-merge-robot committed Jun 1, 2021
2 parents cbc5397 + 86a6edc commit 07c1d0e
Show file tree
Hide file tree
Showing 88 changed files with 20,610 additions and 4 deletions.
7 changes: 7 additions & 0 deletions assets/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ data:
use-clouds = true
clouds-file = /etc/kubernetes/secret/clouds.yaml
cloud = openstack
multiaz-cloud.conf: |
[Global]
use-clouds = true
clouds-file = /etc/kubernetes/secret/clouds.yaml
cloud = openstack
[BlockStorage]
ignore-volume-az = yes
kind: ConfigMap
metadata:
name: openstack-cinder-config
Expand Down
4 changes: 3 additions & 1 deletion assets/controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -230,12 +230,14 @@ spec:
items:
- key: cloud.conf
path: cloud.conf
- key: multiaz-cloud.conf
path: multiaz-cloud.conf
- name: cacert
# If present, extract ca-bundle.pem to
# /etc/kubernetes/static-pod-resources/configmaps/cloud-config
# Let the pod start when the ConfigMap does not exist or the certificate
# is not preset there. The certificate file will be created once the
# ConfigMap is created / the cerificate is added to it.
# ConfigMap is created / the certificate is added to it.
configMap:
name: cloud-provider-config
items:
Expand Down
4 changes: 3 additions & 1 deletion assets/node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,13 @@ spec:
items:
- key: cloud.conf
path: cloud.conf
- key: multiaz-cloud.conf
path: multiaz-cloud.conf
- name: cacert
# Extract ca-bundle.pem to /etc/kubernetes/static-pod-resources/configmaps/cloud-config if present.
# Let the pod start when the ConfigMap does not exist or the certificate
# is not preset there. The certificate file will be created once the
# ConfigMap is created / the cerificate is added to it.
# ConfigMap is created / the certificate is added to it.
configMap:
name: cloud-provider-config
items:
Expand Down
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ require (
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/go-bindata/go-bindata v3.1.2+incompatible
github.com/google/gofuzz v1.2.0 // indirect
github.com/gophercloud/gophercloud v0.17.0
github.com/gophercloud/utils v0.0.0-20210323225332-7b186010c04f
github.com/openshift/api v0.0.0-20210331193751-3acddb19d360
github.com/openshift/build-machinery-go v0.0.0-20210209125900-0da259a2c359
github.com/openshift/client-go v0.0.0-20210331195552-cf6c2669e01f
Expand All @@ -14,6 +16,7 @@ require (
github.com/spf13/cobra v1.1.1
github.com/spf13/pflag v1.0.5
golang.org/x/text v0.3.5 // indirect
k8s.io/api v0.21.0-rc.0
k8s.io/apimachinery v0.21.0-rc.0
k8s.io/client-go v0.21.0-rc.0
k8s.io/component-base v0.21.0-rc.0
Expand Down
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,11 @@ github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTV
github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
github.com/gophercloud/gophercloud v0.15.1-0.20210202035223-633d73521055/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4=
github.com/gophercloud/gophercloud v0.17.0 h1:BgVw0saxyeHWH5us/SQe1ltp0GRnytjmOLXDA8pO77E=
github.com/gophercloud/gophercloud v0.17.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4=
github.com/gophercloud/utils v0.0.0-20210323225332-7b186010c04f h1:+SO5iEqu9QjNWL9TfAmOE5u0Uizv1T3jpBuMJfMOVJ0=
github.com/gophercloud/utils v0.0.0-20210323225332-7b186010c04f/go.mod h1:wx8HMD8oQD0Ryhz6+6ykq75PJ79iPyEqYHfwZ4l7OsA=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v0.0.0-20191024121256-f395758b854c/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
Expand Down Expand Up @@ -305,6 +310,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
Expand Down Expand Up @@ -368,6 +374,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
Expand Down Expand Up @@ -546,6 +553,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -829,6 +837,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Expand Down
15 changes: 13 additions & 2 deletions pkg/generated/bindata.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

115 changes: 115 additions & 0 deletions pkg/operator/cloudinfo.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package operator

import (
"fmt"

"github.com/gophercloud/gophercloud"
"github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/availabilityzones"
"github.com/gophercloud/utils/openstack/clientconfig"
azutils "github.com/gophercloud/utils/openstack/compute/v2/availabilityzones"

"k8s.io/klog/v2"
)

// CloudInfo caches data fetched from the user's openstack cloud
type CloudInfo struct {
ComputeZones []string
VolumeZones []string

clients *clients
}

type clients struct {
computeClient *gophercloud.ServiceClient
volumeClient *gophercloud.ServiceClient
}

var ci *CloudInfo

// getCloudInfo fetches and caches metadata from openstack
func getCloudInfo() (*CloudInfo, error) {
var err error

if ci != nil {
return ci, nil
}

ci = &CloudInfo{
clients: &clients{},
}

opts := new(clientconfig.ClientOpts)
opts.Cloud = "openstack"

ci.clients.computeClient, err = clientconfig.NewServiceClient("compute", opts)
if err != nil {
return nil, fmt.Errorf("failed to create a compute client: %w", err)
}

ci.clients.volumeClient, err = clientconfig.NewServiceClient("volume", opts)
if err != nil {
return nil, fmt.Errorf("failed to create a volume client: %w", err)
}

err = ci.collectInfo()
if err != nil {
klog.Warningf("Failed to generate OpenStack cloud info: %v", err)
return nil, nil
}

return ci, nil
}

func (ci *CloudInfo) collectInfo() error {
var err error

ci.ComputeZones, err = ci.getComputeZones()
if err != nil {
return err
}

ci.VolumeZones, err = ci.getVolumeZones()
if err != nil {
return err
}

return nil
}

func (ci *CloudInfo) getComputeZones() ([]string, error) {
zones, err := azutils.ListAvailableAvailabilityZones(ci.clients.computeClient)
if err != nil {
return nil, fmt.Errorf("failed to list compute availability zones: %w", err)
}

if len(zones) == 0 {
return nil, fmt.Errorf("could not find an available compute availability zone")
}

return zones, nil
}

func (ci *CloudInfo) getVolumeZones() ([]string, error) {
allPages, err := availabilityzones.List(ci.clients.volumeClient).AllPages()
if err != nil {
return nil, fmt.Errorf("failed to list volume availability zones: %w", err)
}

availabilityZoneInfo, err := availabilityzones.ExtractAvailabilityZones(allPages)
if err != nil {
return nil, fmt.Errorf("failed to parse response with volume availability zone list: %w", err)
}

if len(availabilityZoneInfo) == 0 {
return nil, fmt.Errorf("could not find an available volume availability zone")
}

var zones []string
for _, zone := range availabilityZoneInfo {
if zone.ZoneState.Available {
zones = append(zones, zone.ZoneName)
}
}

return zones, nil
}
63 changes: 63 additions & 0 deletions pkg/operator/starter.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"time"

appsv1 "k8s.io/api/apps/v1"
"k8s.io/client-go/dynamic"
kubeclient "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
Expand All @@ -30,6 +31,8 @@ const (
operandName = "openstack-cinder-csi-driver"
instanceName = "cinder.csi.openstack.org"
secretName = "openstack-cloud-credentials"

multiAZConfigPath = "/etc/kubernetes/config/multiaz-cloud.conf"
)

func RunOperator(ctx context.Context, controllerConfig *controllercmd.ControllerContext) error {
Expand All @@ -54,6 +57,14 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
return err
}

ci, err := getCloudInfo()
if err != nil {
return fmt.Errorf("couldn't collect info about cloud availability zones: %w", err)
}
// We consider a cloud multiaz when it either have several different zones or compute and volumes
// are different.
isMultiAZDeployment := len(ci.ComputeZones) > 1 || len(ci.VolumeZones) > 1 || ci.ComputeZones[0] != ci.VolumeZones[0]

csiControllerSet := csicontrollerset.NewCSIControllerSet(
operatorClient,
controllerConfig.EventRecorder,
Expand Down Expand Up @@ -102,13 +113,15 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
configInformers,
csidrivercontrollerservicecontroller.WithSecretHashAnnotationHook(defaultNamespace, secretName, secretInformer),
csidrivercontrollerservicecontroller.WithObservedProxyDeploymentHook(),
withCustomConfigDeploymentHook(isMultiAZDeployment),
).WithCSIDriverNodeService(
"OpenStackCinderDriverNodeServiceController",
generated.MustAsset,
"node.yaml",
kubeClient,
kubeInformersForNamespaces.InformersFor(defaultNamespace),
csidrivernodeservicecontroller.WithObservedProxyDaemonSetHook(),
withCustomConfigDaemonSetHook(isMultiAZDeployment),
).WithServiceMonitorController(
"CinderServiceMonitorController",
dynamicClient,
Expand All @@ -132,3 +145,53 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller

return fmt.Errorf("stopped")
}

// withCustomConfigDeploymentHook executes the asset as a template to fill out the parts required
// when using a custom config with controller deployment.
func withCustomConfigDeploymentHook(isMultiAZDeployment bool) csidrivercontrollerservicecontroller.DeploymentHookFunc {
return func(_ *opv1.OperatorSpec, deployment *appsv1.Deployment) error {
if !isMultiAZDeployment {
return nil
}

for i := range deployment.Spec.Template.Spec.Containers {
container := &deployment.Spec.Template.Spec.Containers[i]
if container.Name != "csi-driver" {
continue
}
for i, env := range container.Env {
if env.Name == "CLOUD_CONFIG" {
container.Env[i].Value = multiAZConfigPath
break
}
}
return nil
}
return fmt.Errorf("could not use multiaz config because the csi-driver container is missing from the deployment")
}
}

// withCustomConfigDaemonSetHook executes the asset as a template to fill out the parts required
// when using a custom config with node controller daemonset.
func withCustomConfigDaemonSetHook(isMultiAZDeployment bool) csidrivernodeservicecontroller.DaemonSetHookFunc {
return func(_ *opv1.OperatorSpec, daemonset *appsv1.DaemonSet) error {
if !isMultiAZDeployment {
return nil
}

for i := range daemonset.Spec.Template.Spec.Containers {
container := &daemonset.Spec.Template.Spec.Containers[i]
if container.Name != "csi-driver" {
continue
}
for i, env := range container.Env {
if env.Name == "CLOUD_CONFIG" {
container.Env[i].Value = multiAZConfigPath
break
}
}
return nil
}
return fmt.Errorf("could not use multiaz config because the csi-driver container is missing from the deployment")
}
}
3 changes: 3 additions & 0 deletions vendor/github.com/gophercloud/gophercloud/.gitignore

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 07c1d0e

Please sign in to comment.