From b28c4af3e19ef9bf98e4d3a6e9883b59504d2ee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Wed, 27 Mar 2019 12:34:57 +0100 Subject: [PATCH] ceph: improve upgrade procedure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a cluster is updated with a different image version, this triggers a serialized restart of all the pods. Prior to this commit, no safety check were performed and rook was hoping for the best outcome. Now before doing restarting a daemon we check it can be restarted. Once it's restarted we also check we can pursue with the rest of the platform. For instance, with monitors we check that they are in quorum, for OSD we check that PGs are clean and for MDS we make sure they are all active. Fixes: https://github.com/rook/rook/issues/2889 Signed-off-by: Sébastien Han --- pkg/daemon/ceph/client/status.go | 65 ++++++ pkg/daemon/ceph/client/status_test.go | 14 ++ pkg/daemon/ceph/client/upgrade.go | 270 ++++++++++++++++++++++-- pkg/daemon/ceph/client/upgrade_test.go | 96 ++++++++- pkg/operator/ceph/cluster/mgr/mgr.go | 5 +- pkg/operator/ceph/cluster/mon/mon.go | 30 ++- pkg/operator/ceph/cluster/mon/spec.go | 30 ++- pkg/operator/ceph/cluster/osd/osd.go | 10 +- pkg/operator/ceph/cluster/rbd/mirror.go | 5 +- pkg/operator/ceph/file/mds/mds.go | 5 +- pkg/operator/ceph/nfs/nfs.go | 4 +- pkg/operator/k8sutil/deployment.go | 23 +- pkg/operator/k8sutil/test/deployment.go | 8 +- 13 files changed, 520 insertions(+), 45 deletions(-) diff --git a/pkg/daemon/ceph/client/status.go b/pkg/daemon/ceph/client/status.go index d75f16672cac..a8fccc6064b0 100644 --- a/pkg/daemon/ceph/client/status.go +++ b/pkg/daemon/ceph/client/status.go @@ -52,6 +52,7 @@ type CephStatus struct { } `json:"osdmap"` PgMap PgMap `json:"pgmap"` MgrMap MgrMap `json:"mgrmap"` + Fsmap Fsmap `json:"fsmap"` } type HealthStatus struct { @@ -123,6 +124,23 @@ type PgStateEntry struct { Count int `json:"count"` } +// Fsmap is a struct representing the filesystem map +type Fsmap struct { + Epoch int `json:"epoch"` + ID int `json:"id"` + Up int `json:"up"` + In int `json:"in"` + Max int `json:"max"` + ByRank []struct { + FilesystemID int `json:"filesystem_id"` + Rank int `json:"rank"` + Name string `json:"name"` + Status string `json:"status"` + Gid int `json:"gid"` + } `json:"by_rank"` + UpStandby int `json:"up:standby"` +} + func Status(context *clusterd.Context, clusterName string, debug bool) (CephStatus, error) { args := []string{"status"} cmd := NewCephCommand(context, clusterName, args) @@ -171,3 +189,50 @@ func isClusterClean(status CephStatus) error { return fmt.Errorf("cluster is not fully clean. PGs: %+v", status.PgMap.PgsByState) } + +// getMDSRank returns the rank of a given MDS +func getMDSRank(status CephStatus, clusterName, mdsName string) (int, error) { + // dummy rank + mdsRank := -1000 + for r := range status.Fsmap.ByRank { + if status.Fsmap.ByRank[r].Name == mdsName { + mdsRank = r + } + } + // if the mds is not shown in the map one reason might be because it's in standby + // if not in standby there is something else going wron + if mdsRank < 0 && status.Fsmap.UpStandby < 1 { + // it might seem strange to log an error since this could be a warning too + // it is a warning until we reach the timeout, this should give enough time to the mds to transtion its state + // after the timeout we consider that the mds might be gone or the timeout was not long enough... + return mdsRank, fmt.Errorf("mds %s not found in fsmap, this likely means mdss are transitioning between active and standby states", mdsName) + } + + return mdsRank, nil +} + +// MdsActiveOrStandbyReplay returns wether a given MDS is active or in standby +func MdsActiveOrStandbyReplay(context *clusterd.Context, clusterName, mdsName string) error { + status, err := Status(context, clusterName, false) + if err != nil { + return err + } + + mdsRank, err := getMDSRank(status, clusterName, mdsName) + if err != nil { + return fmt.Errorf("%+v", err) + } + + // this MDS is in standby so let's return immediatly + if mdsRank < 0 { + logger.Infof("mds %s is in standby, nothing to check", mdsName) + return nil + } + + if status.Fsmap.ByRank[mdsRank].Status == "up:active" || status.Fsmap.ByRank[mdsRank].Status == "up:standby-replay" || status.Fsmap.ByRank[mdsRank].Status == "up:standby" { + logger.Infof("mds %s is %s", mdsName, status.Fsmap.ByRank[mdsRank].Status) + return nil + } + + return fmt.Errorf("mds %s is %s, bad state", mdsName, status.Fsmap.ByRank[mdsRank].Status) +} diff --git a/pkg/daemon/ceph/client/status_test.go b/pkg/daemon/ceph/client/status_test.go index d544ab3b1578..b604e72525ab 100644 --- a/pkg/daemon/ceph/client/status_test.go +++ b/pkg/daemon/ceph/client/status_test.go @@ -27,6 +27,11 @@ const ( CephStatusResponseRaw = `{"fsid":"613975f3-3025-4802-9de1-a2280b950e75","health":{"checks":{"OSD_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 osds down"}},"OSD_HOST_DOWN":{"severity":"HEALTH_WARN","summary":{"message":"1 host (1 osds) down"}},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 101 pgs stale"}},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"application not enabled on 1 pool(s)"}}},"status":"HEALTH_WARN","overall_status":"HEALTH_WARN"},"election_epoch":12,"quorum":[0,1,2],"quorum_names":["rook-ceph-mon0","rook-ceph-mon2","rook-ceph-mon1"],"monmap":{"epoch":3,"fsid":"613975f3-3025-4802-9de1-a2280b950e75","modified":"2017-08-11 20:13:02.075679","created":"2017-08-11 20:12:35.314510","features":{"persistent":["kraken","luminous"],"optional":[]},"mons":[{"rank":0,"name":"rook-ceph-mon0","addr":"10.3.0.45:6789/0","public_addr":"10.3.0.45:6789/0"},{"rank":1,"name":"rook-ceph-mon2","addr":"10.3.0.249:6789/0","public_addr":"10.3.0.249:6789/0"},{"rank":2,"name":"rook-ceph-mon1","addr":"10.3.0.252:6789/0","public_addr":"10.3.0.252:6789/0"}]},"osdmap":{"osdmap":{"epoch":17,"num_osds":2,"num_up_osds":1,"num_in_osds":2,"full":false,"nearfull":true,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"stale+active+clean","count":101},{"state_name":"active+clean","count":99}],"num_pgs":200,"num_pools":2,"num_objects":243,"data_bytes":976793635,"bytes_used":13611479040,"bytes_avail":19825307648,"bytes_total":33436786688},"fsmap":{"epoch":1,"by_rank":[]},"mgrmap":{"epoch":3,"active_gid":14111,"active_name":"rook-ceph-mgr0","active_addr":"10.2.73.6:6800/9","available":true,"standbys":[],"modules":["restful","status"],"available_modules":["dashboard","prometheus","restful","status","zabbix"]},"servicemap":{"epoch":1,"modified":"0.000000","services":{}}}` ) +var ( + // this JSON was generated from `ceph status -f json`, using Ceph Luminous 14.2.1 + statusFakeRaw = []byte(`{"fsid":"33bd4615-810a-44bd-bbb5-d2f9d419a531","health":{"checks":{"OLD_CRUSH_TUNABLES":{"severity":"HEALTH_WARN","summary":{"message":"crush map has legacy tunables (require firefly, min is hammer)"}}},"status":"HEALTH_WARN"},"election_epoch":36,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":578,"monmap":{"epoch":4,"fsid":"33bd4615-810a-44bd-bbb5-d2f9d419a531","modified":"2019-06-06T12:33:44.958580Z","created":"2019-06-06T12:03:07.885569Z","min_mon_release":15,"min_mon_release_name":"octopus","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"10.105.83.252:3300","nonce":0},{"type":"v1","addr":"10.105.83.252:6789","nonce":0}]},"addr":"10.105.83.252:6789/0","public_addr":"10.105.83.252:6789/0","priority":0,"weight":0},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"10.107.252.200:3300","nonce":0},{"type":"v1","addr":"10.107.252.200:6789","nonce":0}]},"addr":"10.107.252.200:6789/0","public_addr":"10.107.252.200:6789/0","priority":0,"weight":0},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"10.97.100.135:3300","nonce":0},{"type":"v1","addr":"10.97.100.135:6789","nonce":0}]},"addr":"10.97.100.135:6789/0","public_addr":"10.97.100.135:6789/0","priority":0,"weight":0}]},"osdmap":{"osdmap":{"epoch":36,"num_osds":3,"num_up_osds":3,"num_in_osds":3,"full":false,"nearfull":false,"num_remapped_pgs":0}},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":200}],"num_pgs":200,"num_pools":2,"num_objects":22,"data_bytes":2286,"bytes_used":3237543936,"bytes_avail":90177994752,"bytes_total":93415538688},"fsmap":{"epoch":13,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"myfs-b","status":"up:active","gid":14716}],"up:standby":1},"mgrmap":{"epoch":42,"active_gid":44551,"active_name":"a","active_addrs":{"addrvec":[{"type":"v2","addr":"172.17.0.9:6800","nonce":1},{"type":"v1","addr":"172.17.0.9:6801","nonce":1}]},"active_addr":"172.17.0.9:6801/1","active_change":"2019-06-06T12:34:14.651759+0000","available":true,"standbys":[],"modules":["dashboard","iostat","prometheus","restful","rook"],"available_modules":[{"name":"ansible","can_run":true,"error_string":"","module_options":{"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_url":{"name":"server_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_server":{"name":"verify_server","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"none","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"float","level":"advanced","flags":1,"default_value":"0.01","min":"0","max":"1","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the ratio between the fullest and least-full OSD is below this value then we stop trying to optimize placement.","tags":[],"see_also":[]},"upmap_max_iterations":{"name":"upmap_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimization iterations","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"str","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"str","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"str","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_cephfs":{"name":"FEATURE_TOGGLE_cephfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_iscsi":{"name":"FEATURE_TOGGLE_iscsi","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_mirroring":{"name":"FEATURE_TOGGLE_mirroring","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rbd":{"name":"FEATURE_TOGGLE_rbd","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rgw":{"name":"FEATURE_TOGGLE_rgw","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"str","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"str","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"str","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"str","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"deepsea","can_run":true,"error_string":"","module_options":{"salt_api_eauth":{"name":"salt_api_eauth","type":"str","level":"advanced","flags":0,"default_value":"sharedsecret","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"salt_api_password":{"name":"salt_api_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"salt_api_url":{"name":"salt_api_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"salt_api_username":{"name":"salt_api_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{}},{"name":"iostat","can_run":true,"error_string":"","module_options":{}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator_cli","can_run":true,"error_string":"","module_options":{"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{}},{"name":"restful","can_run":true,"error_string":"","module_options":{"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{}},{"name":"volumes","can_run":true,"error_string":"","module_options":{}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://rook-ceph-mgr-a-755fd477f5-smm7v:8443/","prometheus":"http://rook-ceph-mgr-a-755fd477f5-smm7v:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator_cli","progress","status","volumes"]}},"servicemap":{"epoch":1,"modified":"0.000000","services":{}},"progress_events":{}}`) +) + func TestStatusMarshal(t *testing.T) { var status CephStatus err := json.Unmarshal([]byte(CephStatusResponseRaw), &status) @@ -85,3 +90,12 @@ func TestIsClusterClean(t *testing.T) { err = isClusterClean(status) assert.NotNil(t, err) } + +func TestGetMDSRank(t *testing.T) { + var statusFake CephStatus + json.Unmarshal(statusFakeRaw, &statusFake) + + mdsRankFake, err := getMDSRank(statusFake, "rook-ceph", "myfs-b") + assert.Nil(t, err) + assert.Equal(t, 0, mdsRankFake) +} diff --git a/pkg/daemon/ceph/client/upgrade.go b/pkg/daemon/ceph/client/upgrade.go index 9208f9f70dae..d17e47f03643 100644 --- a/pkg/daemon/ceph/client/upgrade.go +++ b/pkg/daemon/ceph/client/upgrade.go @@ -18,35 +18,70 @@ package client import ( "encoding/json" + "errors" "fmt" + "strings" + "time" "github.com/rook/rook/pkg/clusterd" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/util" ) // CephDaemonsVersions is a structure that can be used to parsed the output of the 'ceph versions' command type CephDaemonsVersions struct { Mon map[string]int `json:"mon,omitempty"` - Osd map[string]int `json:"osd,omitempty"` Mgr map[string]int `json:"mgr,omitempty"` + Osd map[string]int `json:"osd,omitempty"` Mds map[string]int `json:"mds,omitempty"` Overall map[string]int `json:"overall,omitempty"` } -func getCephMonVersionString(context *clusterd.Context) (string, error) { - output, err := context.Executor.ExecuteCommandWithOutput(false, "", "ceph", "version") +var ( + // we don't perform any checks on these daemons + // they don't have any "ok-to-stop" command implemented + daemonNoCheck = []string{"mgr", "rgw", "rbdmirror"} +) + +func getCephMonVersionString(context *clusterd.Context, clusterName string) (string, error) { + args := []string{"version"} + command, args := FinalizeCephCommandArgs("ceph", args, context.ConfigDir, clusterName) + + output, err := context.Executor.ExecuteCommandWithOutput(false, "", command, args...) if err != nil { - return "", fmt.Errorf("failed to run ceph version: %+v", err) + return "", fmt.Errorf("failed to run ceph version. %+v", err) } logger.Debug(output) return output, nil } -func getCephVersionsString(context *clusterd.Context) (string, error) { - output, err := context.Executor.ExecuteCommandWithOutput(false, "", "ceph", "versions") +func getCephVersionsString(context *clusterd.Context, clusterName string) (string, error) { + args := []string{"versions"} + command, args := FinalizeCephCommandArgs("ceph", args, context.ConfigDir, clusterName) + + output, err := context.Executor.ExecuteCommandWithOutput(false, "", command, args...) if err != nil { - return "", fmt.Errorf("failed to run ceph versions: %+v", err) + return "", fmt.Errorf("failed to run ceph versions. %+v", err) + } + logger.Debug(output) + + return output, nil +} + +func getCephDaemonVersionString(context *clusterd.Context, deployment, clusterName string) (string, error) { + daemonName, err := findDaemonName(deployment) + if err != nil { + return "", fmt.Errorf("%+v", err) + } + daemonID := findDaemonID(deployment) + daemon := daemonName + "." + daemonID + + args := []string{"tell", daemon, "version"} + command, args := FinalizeCephCommandArgs("ceph", args, context.ConfigDir, clusterName) + output, err := context.Executor.ExecuteCommandWithOutput(false, "", command, args...) + if err != nil { + return "", fmt.Errorf("failed to run ceph tell. %+v", err) } logger.Debug(output) @@ -54,10 +89,26 @@ func getCephVersionsString(context *clusterd.Context) (string, error) { } // GetCephMonVersion reports the Ceph version of all the monitors, or at least a majority with quorum -func GetCephMonVersion(context *clusterd.Context) (*cephver.CephVersion, error) { - output, err := getCephMonVersionString(context) +func GetCephMonVersion(context *clusterd.Context, clusterName string) (*cephver.CephVersion, error) { + output, err := getCephMonVersionString(context, clusterName) + if err != nil { + return nil, fmt.Errorf("failed to run ceph version. %+v", err) + } + logger.Debug(output) + + v, err := cephver.ExtractCephVersion(output) + if err != nil { + return nil, fmt.Errorf("failed to extract ceph version. %+v", err) + } + + return v, nil +} + +// GetCephDaemonVersion reports the Ceph version of a particular daemon +func GetCephDaemonVersion(context *clusterd.Context, deployment, clusterName string) (*cephver.CephVersion, error) { + output, err := getCephDaemonVersionString(context, deployment, clusterName) if err != nil { - return nil, fmt.Errorf("failed to run ceph version: %+v", err) + return nil, fmt.Errorf("failed to run ceph tell. %+v", err) } logger.Debug(output) @@ -70,10 +121,10 @@ func GetCephMonVersion(context *clusterd.Context) (*cephver.CephVersion, error) } // GetCephVersions reports the Ceph version of each daemon in the cluster -func GetCephVersions(context *clusterd.Context) (*CephDaemonsVersions, error) { - output, err := getCephVersionsString(context) +func GetCephVersions(context *clusterd.Context, clusterName string) (*CephDaemonsVersions, error) { + output, err := getCephVersionsString(context, clusterName) if err != nil { - return nil, fmt.Errorf("failed to run ceph versions: %+v", err) + return nil, fmt.Errorf("failed to run ceph versions. %+v", err) } logger.Debug(output) @@ -90,7 +141,7 @@ func GetCephVersions(context *clusterd.Context) (*CephDaemonsVersions, error) { func EnableMessenger2(context *clusterd.Context) error { _, err := context.Executor.ExecuteCommandWithOutput(false, "", "ceph", "mon", "enable-msgr2") if err != nil { - return fmt.Errorf("failed to enable msgr2 protocol: %+v", err) + return fmt.Errorf("failed to enable msgr2 protocol. %+v", err) } logger.Infof("successfully enabled msgr2 protocol") @@ -104,6 +155,197 @@ func EnableNautilusOSD(context *clusterd.Context) error { return fmt.Errorf("failed to disallow pre-nautilus osds and enable all new nautilus-only functionality: %+v", err) } logger.Infof("successfully disallowed pre-nautilus osds and enabled all new nautilus-only functionality") + return nil +} + +// OkToStop determines if it's ok to stop an upgrade +func OkToStop(context *clusterd.Context, namespace, deployment, clusterName string, cephVersion cephver.CephVersion) error { + daemonName, err := findDaemonName(deployment) + if err != nil { + logger.Warningf("%+v", err) + return nil + } + + // The ok-to-stop command for mon and mds landed on 14.2.1 + // so we return nil if that Ceph version is not satisfied + if !cephVersion.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 1}) { + if daemonName != "osd" { + return nil + } + } + + versions, err := GetCephVersions(context, clusterName) + if err != nil { + return fmt.Errorf("failed to get ceph daemons versions. %+v", err) + } + + switch daemonName { + // Trying to handle the case where a **single** mon is deployed and an upgrade is called + case "mon": + // if len(versions.Mon) > 1, we have different Ceph versions for some monitor(s) + // this is fine running the upgrade we can run the upgrade checks + if len(versions.Mon) == 1 { + // now trying to parse and find how many mons are presents + // if we have less than 3 mons we skip the check and do best-effort + // we do less than 3 because during the initial bootstrap the mon sequence is updated too + // so running running the check on 2/3 mon fails + // versions.Mon looks like this map[ceph version 15.0.0-12-g6c8fb92 (6c8fb920cb1d862f36ee852ed849a15f9a50bd68) octopus (dev):1] + // now looping over a single element since we can't address the key directly (we don't know its name) + for _, monCount := range versions.Mon { + if monCount < 3 { + logger.Infof("the cluster has less than 3 monitors, not performing upgrade check, running in best-effort") + return nil + } + } + } + // Trying to handle the case where a **single** osd is deployed and an upgrade is called + case "osd": + // if len(versions.Osd) > 1, we have different Ceph versions for some osd(s) + // this is fine running the upgrade we can run the upgrade checks + if len(versions.Osd) == 1 { + // now trying to parse and find how many osds are presents + // if we have less than 3 osds we skip the check and do best-effort + for _, osdCount := range versions.Osd { + if osdCount < 3 { + logger.Infof("the cluster has less than 3 OSDs, not performing upgrade check, running in best-effort") + return nil + } + } + } + } + // we don't implement any checks for mon, rgw and rbdmirror since: + // - mon: the is done in the monitor code since it ensures all the mons are always in quorum before continuing + // - rgw: the pod spec has a liveness probe so if the pod successfully start + // - rbdmirror: you can chain as many as you want like mdss but there is no ok-to-stop logic yet + err = okToStopDaemon(context, deployment, clusterName) + if err != nil { + return fmt.Errorf("failed to check if %s was ok to stop. %+v", deployment, err) + } return nil } + +// OkToContinue determines if it's ok to continue an upgrade +func OkToContinue(context *clusterd.Context, namespace, deployment string, cephVersion cephver.CephVersion) error { + daemonName, err := findDaemonName(deployment) + if err != nil { + logger.Warningf("%+v", err) + return nil + } + + // the mon case is handled directly in the deployment where the mon checks for quorum + switch daemonName { + case "osd": + err := okToContinueOSDDaemon(context, namespace) + if err != nil { + return fmt.Errorf("failed to check if %s was ok to continue. %+v", deployment, err) + } + case "mds": + err := okToContinueMDSDaemon(context, namespace, deployment) + if err != nil { + return fmt.Errorf("failed to check if %s was ok to continue. %+v", deployment, err) + } + } + + return nil +} + +func okToStopDaemon(context *clusterd.Context, deployment, clusterName string) error { + daemonID := findDaemonID(deployment) + daemonName, err := findDaemonName(deployment) + if err != nil { + logger.Warningf("%+v", err) + return nil + } + + if !stringInSlice(daemonName, daemonNoCheck) { + args := []string{daemonName, "ok-to-stop", daemonID} + command, args := FinalizeCephCommandArgs("ceph", args, context.ConfigDir, clusterName) + + output, err := context.Executor.ExecuteCommandWithOutput(false, "", command, args...) + if err != nil { + return fmt.Errorf("deployment %s cannot be stopped. %+v", deployment, err) + } + logger.Debugf("deployment %s is ok to be updated. %s", deployment, output) + } + + logger.Debugf("deployment %s is ok to be updated.", deployment) + + return nil +} + +// okToContinueOSDDaemon determines whether it's fine to go to the next osd during an upgrade +// This basically makes sure all the PGs have settled +func okToContinueOSDDaemon(context *clusterd.Context, namespace string) error { + // Reconciliating PGs should not take too long so let's wait up to 10 minutes + err := util.Retry(10, 60*time.Second, func() error { + return IsClusterClean(context, namespace) + }) + if err != nil { + return err + } + + return nil +} + +// okToContinueMDSDaemon determines whether it's fine to go to the next mds during an upgrade +// mostly a placeholder function for the future but since we have standby mds this shouldn't be needed +func okToContinueMDSDaemon(context *clusterd.Context, namespace, deployment string) error { + daemonName, err := findDaemonName(deployment) + if err != nil { + logger.Warningf("%+v", err) + return nil + } + + // wait for the MDS to be active again or in standby-replay + err = util.Retry(10, 15*time.Second, func() error { + return MdsActiveOrStandbyReplay(context, namespace, daemonName) + }) + if err != nil { + return err + } + + return nil +} + +func findDaemonID(deployment string) string { + daemonTrimPrefixSplit := strings.Split(deployment, "-") + return daemonTrimPrefixSplit[len(daemonTrimPrefixSplit)-1] +} + +func findDaemonName(deployment string) (string, error) { + err := errors.New("could not find daemon name, is this a new daemon?") + + if strings.Contains(deployment, "mds") { + return "mds", nil + } + if strings.Contains(deployment, "rgw") { + return "rgw", nil + } + if strings.Contains(deployment, "mon") { + return "mon", nil + } + if strings.Contains(deployment, "osd") { + return "osd", nil + } + if strings.Contains(deployment, "mgr") { + return "mgr", nil + } + if strings.Contains(deployment, "nfs") { + return "nfs", nil + } + if strings.Contains(deployment, "rbdmirror") { + return "rbdmirror", nil + } + + return "", fmt.Errorf("%+v from deployment %s", err, deployment) +} + +func stringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} diff --git a/pkg/daemon/ceph/client/upgrade_test.go b/pkg/daemon/ceph/client/upgrade_test.go index af2ede7393aa..8bc33c091453 100644 --- a/pkg/daemon/ceph/client/upgrade_test.go +++ b/pkg/daemon/ceph/client/upgrade_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/rook/rook/pkg/clusterd" + cephver "github.com/rook/rook/pkg/operator/ceph/version" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" ) @@ -28,12 +29,11 @@ func TestGetCephMonVersionString(t *testing.T) { executor := &exectest.MockExecutor{} executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { assert.Equal(t, "version", args[0]) - assert.Equal(t, 1, len(args)) return "", nil } context := &clusterd.Context{Executor: executor} - _, err := getCephMonVersionString(context) + _, err := getCephMonVersionString(context, "rook-ceph") assert.Nil(t, err) } @@ -41,12 +41,26 @@ func TestGetCephMonVersionsString(t *testing.T) { executor := &exectest.MockExecutor{} executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { assert.Equal(t, "versions", args[0]) - assert.Equal(t, 1, len(args)) return "", nil } context := &clusterd.Context{Executor: executor} - _, err := getCephVersionsString(context) + _, err := getCephVersionsString(context, "rook-ceph") + assert.Nil(t, err) +} + +func TestGetCephDaemonVersionString(t *testing.T) { + executor := &exectest.MockExecutor{} + deployment := "rook-ceph-mds-a" + executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { + assert.Equal(t, "tell", args[0]) + assert.Equal(t, "mds.a", args[1]) + assert.Equal(t, "version", args[2]) + return "", nil + } + context := &clusterd.Context{Executor: executor} + + _, err := getCephDaemonVersionString(context, deployment, "rook-ceph") assert.Nil(t, err) } @@ -55,7 +69,6 @@ func TestEnableMessenger2(t *testing.T) { executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { assert.Equal(t, "mon", args[0]) assert.Equal(t, "enable-msgr2", args[1]) - assert.Equal(t, 2, len(args)) return "", nil } context := &clusterd.Context{Executor: executor} @@ -69,8 +82,6 @@ func TestEnableNautilusOSD(t *testing.T) { executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { assert.Equal(t, "osd", args[0]) assert.Equal(t, "require-osd-release", args[1]) - assert.Equal(t, "nautilus", args[2]) - assert.Equal(t, 3, len(args)) return "", nil } context := &clusterd.Context{Executor: executor} @@ -78,3 +89,74 @@ func TestEnableNautilusOSD(t *testing.T) { err := EnableNautilusOSD(context) assert.Nil(t, err) } + +func TestOkToStopDaemon(t *testing.T) { + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { + assert.Equal(t, "mon", args[0]) + assert.Equal(t, "ok-to-stop", args[1]) + assert.Equal(t, "a", args[2]) + return "", nil + } + context := &clusterd.Context{Executor: executor} + + deployment := "rook-ceph-mon-a" + err := okToStopDaemon(context, deployment, "rook-ceph") + assert.Nil(t, err) +} + +func TestFindDaemonName(t *testing.T) { + n, err := findDaemonName("rook-ceph-mon-a") + assert.Nil(t, err) + assert.Equal(t, "mon", n) + n, err = findDaemonName("rook-ceph-osd-0") + assert.Nil(t, err) + assert.Equal(t, "osd", n) + n, err = findDaemonName("rook-ceph-rgw-my-store-a") + assert.Nil(t, err) + assert.Equal(t, "rgw", n) + n, err = findDaemonName("rook-ceph-mds-myfs-a") + assert.Nil(t, err) + assert.Equal(t, "mds", n) + n, err = findDaemonName("rook-ceph-mgr-a") + assert.Nil(t, err) + assert.Equal(t, "mgr", n) + _, err = findDaemonName("rook-ceph-unknown-a") + assert.NotNil(t, err) +} + +func TestFindDaemonID(t *testing.T) { + id := findDaemonID("rook-ceph-mon-a") + assert.Equal(t, "a", id) + id = findDaemonID("rook-ceph-osd-0") + assert.Equal(t, "0", id) + id = findDaemonID("rook-ceph-rgw-my-super-store-a") + assert.Equal(t, "a", id) + id = findDaemonID("rook-ceph-mds-my-wonderful-fs-a") + assert.Equal(t, "a", id) + id = findDaemonID("rook-ceph-mgr-a") + assert.Equal(t, "a", id) + id = findDaemonID("rook.ceph.mgr.a") + assert.NotEqual(t, "a", id) +} + +func TestOkToContinue(t *testing.T) { + executor := &exectest.MockExecutor{} + context := &clusterd.Context{Executor: executor} + v := cephver.Nautilus + + err := OkToContinue(context, "rook-ceph", "rook-ceph-mon-a", v) // mon is not checked on ok-to-continue so nil is expected + assert.Nil(t, err) +} + +func TestOkToStop(t *testing.T) { + executor := &exectest.MockExecutor{} + context := &clusterd.Context{Executor: executor} + v := cephver.Nautilus + + err := OkToStop(context, "rook-ceph", "rook-ceph-mon-a", "rook-ceph", v) + assert.Nil(t, err) + + err = OkToStop(context, "rook-ceph", "rook-ceph-mds-a", "rook-ceph", v) + assert.Nil(t, err) +} diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 720dafa59765..78d29f131fca 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -26,6 +26,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" @@ -96,7 +97,7 @@ func New( } } -var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // Start begins the process of running a cluster of Ceph mgrs. func (c *Cluster) Start() error { @@ -141,7 +142,7 @@ func (c *Cluster) Start() error { return fmt.Errorf("failed to create mgr deployment %s. %+v", resourceName, err) } logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName) - if _, err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil { + if err := updateDeploymentAndWait(c.context, d, c.Namespace, c.clusterInfo.Name, c.clusterInfo.CephVersion); err != nil { return fmt.Errorf("failed to update mgr deployment %s. %+v", resourceName, err) } } diff --git a/pkg/operator/ceph/cluster/mon/mon.go b/pkg/operator/ceph/cluster/mon/mon.go index 79119b7724ed..fa8898dd2027 100644 --- a/pkg/operator/ceph/cluster/mon/mon.go +++ b/pkg/operator/ceph/cluster/mon/mon.go @@ -219,12 +219,12 @@ func (c *Cluster) startMons(targetCount int) error { // Enable Ceph messenger 2 protocol on Nautilus if c.clusterInfo.CephVersion.IsAtLeastNautilus() { - v, err := client.GetCephMonVersion(c.context) + v, err := client.GetCephMonVersion(c.context, c.clusterInfo.Name) if err != nil { return fmt.Errorf("failed to get ceph mon version. %+v", err) } if v.IsAtLeastNautilus() { - versions, err := client.GetCephVersions(c.context) + versions, err := client.GetCephVersions(c.context, c.clusterInfo.Name) if err != nil { return fmt.Errorf("failed to get ceph daemons versions. %+v", err) } @@ -426,9 +426,28 @@ func (c *Cluster) startDeployments(mons []*monConfig, requireAllInQuorum bool) e if err != nil { return fmt.Errorf("failed to create mon %s. %+v", mons[i].DaemonName, err) } + // For the initial deployment (first creation) it's expected to not have all the monitors in quorum + // However, in an event of an update, it's crucial to proceed monitors by monitors + // At the end of the method we perform one last check where all the monitors must be in quorum + requireAllInQuorum := false + err = c.waitForMonsToJoin(mons, requireAllInQuorum) + if err != nil { + return fmt.Errorf("failed to check mon quorum %s. %+v", mons[i].DaemonName, err) + } } logger.Infof("mons created: %d", len(mons)) + // Final verification that **all** mons are in quorum + // Do not proceed if one monitor is still syncing + // Only do this when monitors versions are different so we don't block the orchestration if a mon is down. + versions, err := client.GetCephVersions(c.context, c.clusterInfo.Name) + if err != nil { + logger.Warningf("failed to get ceph daemons versions. %+v, this likely means there is no cluster yet.", err) + } else { + if len(versions.Mon) != 1 { + requireAllInQuorum = true + } + } return c.waitForMonsToJoin(mons, requireAllInQuorum) } @@ -504,9 +523,10 @@ func (c *Cluster) saveMonConfig() error { return nil } -var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +var updateDeploymentAndWait = UpdateCephDeploymentAndWait func (c *Cluster) startMon(m *monConfig, hostname string) error { + d := c.makeDeployment(m, hostname) logger.Debugf("Starting mon: %+v", d.Name) _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(d) @@ -515,7 +535,7 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error { return fmt.Errorf("failed to create mon deployment %s. %+v", m.ResourceName, err) } logger.Infof("deployment for mon %s already exists. updating if needed", m.ResourceName) - if _, err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil { + if err := updateDeploymentAndWait(c.context, d, c.Namespace, c.clusterInfo.Name, c.clusterInfo.CephVersion); err != nil { return fmt.Errorf("failed to update mon deployment %s. %+v", m.ResourceName, err) } } @@ -528,7 +548,7 @@ func waitForQuorumWithMons(context *clusterd.Context, clusterName string, mons [ // wait for monitors to establish quorum retryCount := 0 - retryMax := 20 + retryMax := 30 for { retryCount++ if retryCount > retryMax { diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index e12fc8b401ee..155e9d4ddc6b 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -21,9 +21,11 @@ import ( "os" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" - opspec "github.com/rook/rook/pkg/operator/ceph/spec" + cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -213,3 +215,29 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { return container } + +// UpdateCephDeploymentAndWait verifies a deployment can be stopped or continued +func UpdateCephDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace, clusterName string, cephVersion cephver.CephVersion) error { + callback := func(action string) error { + logger.Infof("checking if we can %s the deployment %s", action, deployment.Name) + + if action == "stop" { + err := client.OkToStop(context, namespace, deployment.Name, clusterName, cephVersion) + if err != nil { + return fmt.Errorf("failed to check if we can %s the deployment %s: %+v", action, deployment.Name, err) + } + } + + if action == "continue" { + err := client.OkToContinue(context, namespace, deployment.Name, cephVersion) + if err != nil { + return fmt.Errorf("failed to check if we can %s the deployment %s: %+v", action, deployment.Name, err) + } + } + + return nil + } + + _, err := k8sutil.UpdateDeploymentAndWait(context, deployment, namespace, callback) + return err +} diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 5ef63ce86d61..0e6c2ddb5847 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" osdconfig "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" cephver "github.com/rook/rook/pkg/operator/ceph/version" @@ -40,7 +41,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-osd") +var ( + logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-osd") + updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait +) const ( appName = "rook-ceph-osd" @@ -191,7 +195,7 @@ func (c *Cluster) Start() error { // The following block is used to apply any command(s) required by an upgrade // The block below handles the upgrade from Mimic to Nautilus. if c.clusterInfo.CephVersion.IsAtLeastNautilus() { - versions, err := client.GetCephVersions(c.context) + versions, err := client.GetCephVersions(c.context, c.clusterInfo.Name) if err != nil { logger.Warningf("failed to get ceph daemons versions. this likely means there are no osds yet. %+v", err) } else { @@ -321,7 +325,7 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig continue } logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID) - if _, err = k8sutil.UpdateDeploymentAndWait(c.context, dp, c.Namespace); err != nil { + if err = updateDeploymentAndWait(c.context, dp, c.Namespace, c.clusterInfo.Name, c.clusterInfo.CephVersion); err != nil { config.addError(fmt.Sprintf("failed to update osd deployment %d. %+v", osd.ID, err)) } } diff --git a/pkg/operator/ceph/cluster/rbd/mirror.go b/pkg/operator/ceph/cluster/rbd/mirror.go index be3805633356..b5884f401267 100644 --- a/pkg/operator/ceph/cluster/rbd/mirror.go +++ b/pkg/operator/ceph/cluster/rbd/mirror.go @@ -25,6 +25,7 @@ import ( rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" @@ -87,7 +88,7 @@ func New( } } -var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // Start begins the process of running rbd mirroring daemons. func (m *Mirroring) Start() error { @@ -119,7 +120,7 @@ func (m *Mirroring) Start() error { return fmt.Errorf("failed to create %s deployment. %+v", resourceName, err) } logger.Infof("deployment for rbd-mirror %s already exists. updating if needed", resourceName) - if _, err := updateDeploymentAndWait(m.context, d, m.Namespace); err != nil { + if err := updateDeploymentAndWait(m.context, d, m.Namespace, m.ClusterInfo.Name, m.ClusterInfo.CephVersion); err != nil { // fail could be an issue updating label selector (immutable), so try del and recreate logger.Debugf("updateDeploymentAndWait failed for rbd-mirror %s. Attempting del-and-recreate. %+v", resourceName, err) err = m.context.Clientset.AppsV1().Deployments(m.Namespace).Delete(d.Name, &metav1.DeleteOptions{}) diff --git a/pkg/operator/ceph/file/mds/mds.go b/pkg/operator/ceph/file/mds/mds.go index 35e2ffe02994..ba6cc2ea6791 100644 --- a/pkg/operator/ceph/file/mds/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -27,6 +27,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" cephver "github.com/rook/rook/pkg/operator/ceph/version" @@ -94,7 +95,7 @@ func NewCluster( } // UpdateDeploymentAndWait can be overridden for unit tests. Do not alter this for runtime operation. -var UpdateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +var UpdateDeploymentAndWait = mon.UpdateCephDeploymentAndWait // Start starts or updates a Ceph mds cluster in Kubernetes. func (c *Cluster) Start() error { @@ -155,7 +156,7 @@ func (c *Cluster) Start() error { // keyring must be generated before update-and-wait since no keyring will prevent the // deployment from reaching ready state if createErr != nil && errors.IsAlreadyExists(createErr) { - if _, err = UpdateDeploymentAndWait(c.context, d, c.fs.Namespace); err != nil { + if err = UpdateDeploymentAndWait(c.context, d, c.fs.Namespace, c.clusterInfo.Name, c.clusterInfo.CephVersion); err != nil { return fmt.Errorf("failed to update mds deployment %s. %+v", d.Name, err) } } diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index 5f1deed3b73e..38c1ff5165a4 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -36,7 +36,7 @@ const ( ganeshaRadosGraceCmd = "ganesha-rados-grace" ) -var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +var updateDeploymentAndWait = opmon.UpdateCephDeploymentAndWait // Create the ganesha server func (c *CephNFSController) upCephNFS(n cephv1.CephNFS, oldActive int) error { @@ -68,7 +68,7 @@ func (c *CephNFSController) upCephNFS(n cephv1.CephNFS, oldActive int) error { return fmt.Errorf("failed to create ganesha deployment. %+v", err) } logger.Infof("ganesha deployment %s already exists. updating if needed", deployment.Name) - if _, err := updateDeploymentAndWait(c.context, deployment, n.Namespace); err != nil { + if err := updateDeploymentAndWait(c.context, deployment, n.Namespace, c.clusterInfo.Name, c.clusterInfo.CephVersion); err != nil { return fmt.Errorf("failed to update ganesha deployment %s. %+v", deployment.Name, err) } } else { diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go index c5244f52ccc1..c76c70f48c09 100644 --- a/pkg/operator/k8sutil/deployment.go +++ b/pkg/operator/k8sutil/deployment.go @@ -21,8 +21,9 @@ import ( "time" "github.com/rook/rook/pkg/clusterd" - "k8s.io/api/apps/v1" + "github.com/rook/rook/pkg/util" apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -36,6 +37,7 @@ func GetDeploymentImage(clientset kubernetes.Interface, namespace, name, contain return GetDeploymentSpecImage(clientset, *d, container, false) } +// GetDeploymentSpecImage returns the image name from the spec func GetDeploymentSpecImage(clientset kubernetes.Interface, d apps.Deployment, container string, initContainer bool) (string, error) { image, err := GetSpecContainerImage(d.Spec.Template.Spec, container, initContainer) if err != nil { @@ -47,12 +49,21 @@ func GetDeploymentSpecImage(clientset kubernetes.Interface, d apps.Deployment, c // UpdateDeploymentAndWait updates a deployment and waits until it is running to return. It will // error if the deployment does not exist to be updated or if it takes too long. -func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*v1.Deployment, error) { +func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace string, verifyCallback func(action string) error) (*v1.Deployment, error) { original, err := context.Clientset.AppsV1().Deployments(namespace).Get(deployment.Name, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err) } + // Let's verify the deployment can be stopped + // retry for 5 times, every minute + err = util.Retry(5, 60*time.Second, func() error { + return verifyCallback("stop") + }) + if err != nil { + return nil, fmt.Errorf("failed to check if deployment %s can be updated: %+v", deployment.Name, err) + } + logger.Infof("updating deployment %s", deployment.Name) if _, err := context.Clientset.AppsV1().Deployments(namespace).Update(deployment); err != nil { return nil, fmt.Errorf("failed to update deployment %s. %+v", deployment.Name, err) @@ -73,9 +84,15 @@ func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deploym } if d.Status.ObservedGeneration != original.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 { logger.Infof("finished waiting for updated deployment %s", d.Name) + + // Now we check if we can go to the next daemon + err = verifyCallback("continue") + if err != nil { + return nil, fmt.Errorf("failed to check if deployment %s can be updated: %+v", deployment.Name, err) + } + return d, nil } - logger.Debugf("deployment %s status=%+v", d.Name, d.Status) time.Sleep(time.Duration(sleepTime) * time.Second) } diff --git a/pkg/operator/k8sutil/test/deployment.go b/pkg/operator/k8sutil/test/deployment.go index e4d5c5b8ef7d..b3b4fd3369fd 100644 --- a/pkg/operator/k8sutil/test/deployment.go +++ b/pkg/operator/k8sutil/test/deployment.go @@ -2,8 +2,8 @@ package test import ( "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/ceph/version" apps "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // UpdateDeploymentAndWaitStub returns a stub replacement for the UpdateDeploymentAndWait function @@ -15,13 +15,13 @@ import ( // returns a pointer to this slice which the calling func may use to verify the expected contents of // deploymentsUpdated based on expected behavior. func UpdateDeploymentAndWaitStub() ( - stubFunc func(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*apps.Deployment, error), + stubFunc func(context *clusterd.Context, deployment *apps.Deployment, namespace, clusterName string, cephVersion version.CephVersion) error, deploymentsUpdated *[]*apps.Deployment, ) { deploymentsUpdated = &[]*apps.Deployment{} - stubFunc = func(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*apps.Deployment, error) { + stubFunc = func(context *clusterd.Context, deployment *apps.Deployment, namespace, clusterName string, cephVersion version.CephVersion) error { *deploymentsUpdated = append(*deploymentsUpdated, deployment) - return &apps.Deployment{ObjectMeta: metav1.ObjectMeta{UID: "stub-deployment-uid"}}, nil + return nil } return stubFunc, deploymentsUpdated }