diff --git a/go.sum b/go.sum index 6c0a38a3f..62d0f273e 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -195,6 +196,7 @@ github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -208,6 +210,7 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= @@ -234,6 +237,7 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= @@ -249,6 +253,7 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -280,6 +285,7 @@ github.com/jessevdk/go-assets-builder v0.0.0-20130903091706-b8483521738f/go.mod github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= @@ -334,6 +340,7 @@ github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/f github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -363,6 +370,7 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= @@ -397,6 +405,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -480,7 +489,9 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -501,6 +512,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -522,6 +534,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -541,6 +554,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEha golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -571,6 +585,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -590,6 +605,7 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -607,6 +623,7 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -618,6 +635,9 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -708,19 +728,29 @@ k8s.io/apiserver v0.19.8/go.mod h1:+OJE9rJCT99Qr9DYITQDCKDLxFLVi5zA8nI9KqjGshk= k8s.io/client-go v0.19.8 h1:rcb2BrXb1HUBiBCoP3m/9Q2VZIMWhZUAmH49EmAyRUA= k8s.io/client-go v0.19.8/go.mod h1:5Op2bSbK+COBz8mwH62rrRgqhA9wOcORkWZ03+GL0Ow= k8s.io/component-base v0.19.8/go.mod h1:cvHAT4oGxKsfcnnm0hMp3JkEMxAt5s6le943V796FXM= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= diff --git a/main.go b/main.go index de07c800b..ae4bf6cdd 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package main @@ -32,6 +33,8 @@ import ( "strings" "time" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + "github.com/arangodb/kube-arangodb/pkg/operator/scope" "github.com/arangodb/kube-arangodb/pkg/deployment/features" @@ -114,6 +117,10 @@ var ( singleMode bool scope string } + timeouts struct { + k8s time.Duration + arangoD time.Duration + } chaosOptions struct { allowed bool } @@ -143,7 +150,8 @@ func init() { f.BoolVar(&chaosOptions.allowed, "chaos.allowed", false, "Set to allow chaos in deployments. Only activated when allowed and enabled in deployment") f.BoolVar(&operatorOptions.singleMode, "mode.single", false, "Enable single mode in Operator. WARNING: There should be only one replica of Operator, otherwise Operator can take unexpected actions") f.StringVar(&operatorOptions.scope, "scope", scope.DefaultScope.String(), "Define scope on which Operator works. Legacy - pre 1.1.0 scope with limited cluster access") - + f.DurationVar(&timeouts.k8s, "timeout.k8s", time.Second*3, "The request timeout to the kubernetes") + f.DurationVar(&timeouts.arangoD, "timeout.arangod", time.Second*10, "The request timeout to the ArangoDB") features.Init(&cmdMain) } @@ -168,6 +176,8 @@ func cmdMainRun(cmd *cobra.Command, args []string) { ip := os.Getenv(constants.EnvOperatorPodIP) deploymentApi.DefaultImage = operatorOptions.arangoImage + k8sutil.SetRequestTimeout(timeouts.k8s) + arangod.SetRequestTimeout(timeouts.arangoD) // Prepare log service var err error diff --git a/pkg/backup/handlers/arango/backup/arango_client_impl.go b/pkg/backup/handlers/arango/backup/arango_client_impl.go index 40a552b8b..7a47f74a0 100644 --- a/pkg/backup/handlers/arango/backup/arango_client_impl.go +++ b/pkg/backup/handlers/arango/backup/arango_client_impl.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Lars Maier +// Author Tomasz Mielech // package backup @@ -128,9 +129,10 @@ func (ac *arangoClientBackupImpl) Get(backupID driver.BackupID) (driver.BackupMe } } -func (ac *arangoClientBackupImpl) getCredentialsFromSecret(secretName string) (interface{}, error) { - - token, err := k8sutil.GetTokenSecret(ac.kubecli.CoreV1().Secrets(ac.backup.Namespace), secretName) +func (ac *arangoClientBackupImpl) getCredentialsFromSecret(ctx context.Context, secretName string) (interface{}, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + token, err := k8sutil.GetTokenSecret(ctxChild, ac.kubecli.CoreV1().Secrets(ac.backup.Namespace), secretName) if err != nil { return nil, err } @@ -152,7 +154,7 @@ func (ac *arangoClientBackupImpl) Upload(backupID driver.BackupID) (driver.Backu return "", errors.Newf("upload was called but no upload spec was given") } - cred, err := ac.getCredentialsFromSecret(uploadSpec.CredentialsSecretName) + cred, err := ac.getCredentialsFromSecret(ctx, uploadSpec.CredentialsSecretName) if err != nil { return "", err } @@ -169,7 +171,7 @@ func (ac *arangoClientBackupImpl) Download(backupID driver.BackupID) (driver.Bac return "", errors.Newf("Download was called but not download spec was given") } - cred, err := ac.getCredentialsFromSecret(downloadSpec.CredentialsSecretName) + cred, err := ac.getCredentialsFromSecret(ctx, downloadSpec.CredentialsSecretName) if err != nil { return "", err } diff --git a/pkg/deployment/access_package.go b/pkg/deployment/access_package.go index 43c77056c..f67987f26 100644 --- a/pkg/deployment/access_package.go +++ b/pkg/deployment/access_package.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -46,7 +47,7 @@ const ( // createAccessPackages creates a arangosync access packages specified // in spec.sync.externalAccess.accessPackageSecretNames. -func (d *Deployment) createAccessPackages() error { +func (d *Deployment) createAccessPackages(ctx context.Context) error { log := d.deps.Log spec := d.apiObject.Spec secrets := d.deps.KubeCli.CoreV1().Secrets(d.GetNamespace()) @@ -60,13 +61,15 @@ func (d *Deployment) createAccessPackages() error { apNameMap := make(map[string]struct{}) for _, apSecretName := range spec.Sync.ExternalAccess.AccessPackageSecretNames { apNameMap[apSecretName] = struct{}{} - if err := d.ensureAccessPackage(apSecretName); err != nil { + if err := d.ensureAccessPackage(ctx, apSecretName); err != nil { return errors.WithStack(err) } } // Remove all access packages that we did build, but are no longer needed - secretList, err := secrets.List(context.Background(), metav1.ListOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + secretList, err := secrets.List(ctxChild, metav1.ListOptions{}) if err != nil { log.Debug().Err(err).Msg("Failed to list secrets") return errors.WithStack(err) @@ -77,9 +80,12 @@ func (d *Deployment) createAccessPackages() error { // Secret is an access package if _, wanted := apNameMap[secret.GetName()]; !wanted { // We found an obsolete access package secret. Remove it. - if err := secrets.Delete(context.Background(), secret.GetName(), metav1.DeleteOptions{ - Preconditions: &metav1.Preconditions{UID: &secret.UID}, - }); err != nil && !k8sutil.IsNotFound(err) { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return secrets.Delete(ctxChild, secret.GetName(), metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &secret.UID}, + }) + }) + if err != nil && !k8sutil.IsNotFound(err) { // Not serious enough to stop everything now, just log and create an event log.Warn().Err(err).Msg("Failed to remove obsolete access package secret") d.CreateEvent(k8sutil.NewErrorEvent("Access Package cleanup failed", err, d.apiObject)) @@ -98,20 +104,29 @@ func (d *Deployment) createAccessPackages() error { // ensureAccessPackage creates an arangosync access package with given name // it is does not already exist. -func (d *Deployment) ensureAccessPackage(apSecretName string) error { +func (d *Deployment) ensureAccessPackage(ctx context.Context, apSecretName string) error { log := d.deps.Log ns := d.GetNamespace() secrets := d.deps.KubeCli.CoreV1().Secrets(ns) spec := d.apiObject.Spec - if _, err := secrets.Get(context.Background(), apSecretName, metav1.GetOptions{}); err == nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Get(ctxChild, apSecretName, metav1.GetOptions{}) + return err + }) + if err == nil { // Secret already exists return nil + } else if !k8sutil.IsNotFound(err) { + log.Debug().Err(err).Str("name", apSecretName).Msg("Failed to get arangosync access package secret") + return errors.WithStack(err) } // Fetch client authentication CA clientAuthSecretName := spec.Sync.Authentication.GetClientCASecretName() - clientAuthCert, clientAuthKey, _, err := k8sutil.GetCASecret(secrets, clientAuthSecretName, nil) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + clientAuthCert, clientAuthKey, _, err := k8sutil.GetCASecret(ctxChild, secrets, clientAuthSecretName, nil) if err != nil { log.Debug().Err(err).Msg("Failed to get client-auth CA secret") return errors.WithStack(err) @@ -119,7 +134,7 @@ func (d *Deployment) ensureAccessPackage(apSecretName string) error { // Fetch TLS CA public key tlsCASecretName := spec.Sync.TLS.GetCASecretName() - tlsCACert, err := k8sutil.GetCACertficateSecret(secrets, tlsCASecretName) + tlsCACert, err := k8sutil.GetCACertficateSecret(ctx, secrets, tlsCASecretName) if err != nil { log.Debug().Err(err).Msg("Failed to get TLS CA secret") return errors.WithStack(err) @@ -205,7 +220,11 @@ func (d *Deployment) ensureAccessPackage(apSecretName string) error { } // Attach secret to owner secret.SetOwnerReferences(append(secret.GetOwnerReferences(), d.apiObject.AsOwner())) - if _, err := secrets.Create(context.Background(), secret, metav1.CreateOptions{}); err != nil { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Create(ctxChild, secret, metav1.CreateOptions{}) + return err + }) + if err != nil { // Failed to create secret log.Debug().Err(err).Str("secret-name", apSecretName).Msg("Failed to create access package Secret") return errors.WithStack(err) diff --git a/pkg/deployment/chaos/context.go b/pkg/deployment/chaos/context.go index 72d97bd6e..ef3723253 100644 --- a/pkg/deployment/chaos/context.go +++ b/pkg/deployment/chaos/context.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,14 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package chaos import ( + "context" + v1 "k8s.io/api/core/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -34,7 +37,7 @@ type Context interface { GetSpec() api.DeploymentSpec // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - DeletePod(podName string) error + DeletePod(ctx context.Context, podName string) error // GetOwnedPods returns a list of all pods owned by the deployment. - GetOwnedPods() ([]v1.Pod, error) + GetOwnedPods(ctx context.Context) ([]v1.Pod, error) } diff --git a/pkg/deployment/chaos/monkey.go b/pkg/deployment/chaos/monkey.go index e7ff95f54..329c4b26f 100644 --- a/pkg/deployment/chaos/monkey.go +++ b/pkg/deployment/chaos/monkey.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package chaos import ( + "context" "math/rand" "time" @@ -50,6 +52,9 @@ func NewMonkey(log zerolog.Logger, context Context) *Monkey { // Run the monkey until the given channel is closed. func (m Monkey) Run(stopCh <-chan struct{}) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for { spec := m.context.GetSpec() if spec.Chaos.IsEnabled() { @@ -57,7 +62,7 @@ func (m Monkey) Run(stopCh <-chan struct{}) { chance := float64(spec.Chaos.GetKillPodProbability()) / 100.0 if rand.Float64() < chance { // Let's introduce pod chaos - if err := m.killRandomPod(); err != nil { + if err := m.killRandomPod(ctx); err != nil { log.Info().Err(err).Msg("Failed to kill random pod") } } @@ -74,8 +79,8 @@ func (m Monkey) Run(stopCh <-chan struct{}) { } // killRandomPod fetches all owned pods and tries to kill one. -func (m Monkey) killRandomPod() error { - pods, err := m.context.GetOwnedPods() +func (m Monkey) killRandomPod(ctx context.Context) error { + pods, err := m.context.GetOwnedPods(ctx) if err != nil { return errors.WithStack(err) } @@ -85,7 +90,7 @@ func (m Monkey) killRandomPod() error { } p := pods[rand.Intn(len(pods))] m.log.Info().Str("pod-name", p.GetName()).Msg("Killing pod") - if err := m.context.DeletePod(p.GetName()); err != nil { + if err := m.context.DeletePod(ctx, p.GetName()); err != nil { return errors.WithStack(err) } return nil diff --git a/pkg/deployment/cleanup.go b/pkg/deployment/cleanup.go index 86472357f..d23726177 100644 --- a/pkg/deployment/cleanup.go +++ b/pkg/deployment/cleanup.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -25,26 +26,30 @@ package deployment import ( "context" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" - core "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) // removePodFinalizers removes all finalizers from all pods owned by us. -func (d *Deployment) removePodFinalizers(cachedStatus inspectorInterface.Inspector) error { +func (d *Deployment) removePodFinalizers(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { log := d.deps.Log kubecli := d.GetKubeCli() if err := cachedStatus.IteratePods(func(pod *core.Pod) error { - if err := k8sutil.RemovePodFinalizers(log, kubecli, pod, pod.GetFinalizers(), true); err != nil { + if err := k8sutil.RemovePodFinalizers(ctx, log, kubecli, pod, pod.GetFinalizers(), true); err != nil { log.Warn().Err(err).Msg("Failed to remove pod finalizers") return err } - if err := kubecli.CoreV1().Pods(pod.GetNamespace()).Delete(context.Background(), pod.GetName(), meta.DeleteOptions{ + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + if err := kubecli.CoreV1().Pods(pod.GetNamespace()).Delete(ctxChild, pod.GetName(), meta.DeleteOptions{ GracePeriodSeconds: util.NewInt64(1), }); err != nil { if !k8sutil.IsNotFound(err) { @@ -61,12 +66,12 @@ func (d *Deployment) removePodFinalizers(cachedStatus inspectorInterface.Inspect } // removePVCFinalizers removes all finalizers from all PVCs owned by us. -func (d *Deployment) removePVCFinalizers(cachedStatus inspectorInterface.Inspector) error { +func (d *Deployment) removePVCFinalizers(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { log := d.deps.Log kubecli := d.GetKubeCli() if err := cachedStatus.IteratePersistentVolumeClaims(func(pvc *core.PersistentVolumeClaim) error { - if err := k8sutil.RemovePVCFinalizers(log, kubecli, pvc, pvc.GetFinalizers(), true); err != nil { + if err := k8sutil.RemovePVCFinalizers(ctx, log, kubecli, pvc, pvc.GetFinalizers(), true); err != nil { log.Warn().Err(err).Msg("Failed to remove PVC finalizers") return err } diff --git a/pkg/deployment/cluster_scaling_integration.go b/pkg/deployment/cluster_scaling_integration.go index 42b3e2a89..3cb9ffe9a 100644 --- a/pkg/deployment/cluster_scaling_integration.go +++ b/pkg/deployment/cluster_scaling_integration.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -80,14 +81,14 @@ func (ci *clusterScalingIntegration) SendUpdateToCluster(spec api.DeploymentSpec // checkScalingCluster checks if inspection // returns true if inspection occurred -func (ci *clusterScalingIntegration) checkScalingCluster(expectSuccess bool) bool { +func (ci *clusterScalingIntegration) checkScalingCluster(ctx context.Context, expectSuccess bool) bool { ci.scaleEnabled.mutex.Lock() defer ci.scaleEnabled.mutex.Unlock() if !ci.scaleEnabled.enabled { // Check if it is possible to turn on scaling without any issue status, _ := ci.depl.GetStatus() - if status.Plan.IsEmpty() && ci.setNumberOfServers() == nil { + if status.Plan.IsEmpty() && ci.setNumberOfServers(ctx) == nil { // Scaling should be enabled because there is no Plan. // It can happen when the enabling action fails ci.scaleEnabled.enabled = true @@ -100,8 +101,6 @@ func (ci *clusterScalingIntegration) checkScalingCluster(expectSuccess bool) boo } // Update cluster with our state - ctx := context.Background() - //expectSuccess := *goodInspections > 0 || time.Since(start) > maxClusterBootstrapTime safeToAskCluster, err := ci.updateClusterServerCount(ctx, expectSuccess) if err != nil { if expectSuccess { @@ -124,10 +123,13 @@ func (ci *clusterScalingIntegration) checkScalingCluster(expectSuccess bool) boo func (ci *clusterScalingIntegration) ListenForClusterEvents(stopCh <-chan struct{}) { start := time.Now() goodInspections := 0 + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for { expectSuccess := goodInspections > 0 || time.Since(start) > maxClusterBootstrapTime - if ci.checkScalingCluster(expectSuccess) { + if ci.checkScalingCluster(ctx, expectSuccess) { goodInspections++ } @@ -144,11 +146,17 @@ func (ci *clusterScalingIntegration) ListenForClusterEvents(stopCh <-chan struct // Perform a single inspection of the cluster func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context, expectSuccess bool) error { log := ci.log - c, err := ci.depl.clientCache.GetDatabase(ctx) + + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := ci.depl.clientCache.GetDatabase(ctxChild) if err != nil { return errors.WithStack(err) } - req, err := arangod.GetNumberOfServers(ctx, c.Connection()) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + req, err := arangod.GetNumberOfServers(ctxChild, c.Connection()) if err != nil { if expectSuccess { log.Debug().Err(err).Msg("Failed to get number of servers") @@ -191,7 +199,9 @@ func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context, expectS } // Let's update the spec apiObject := ci.depl.apiObject - current, err := ci.depl.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(apiObject.Namespace).Get(context.Background(), apiObject.Name, metav1.GetOptions{}) + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + current, err := ci.depl.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(apiObject.Namespace).Get(ctxChild, apiObject.Name, metav1.GetOptions{}) if err != nil { return errors.WithStack(err) } @@ -211,7 +221,7 @@ func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context, expectS // Restore original spec in cluster ci.SendUpdateToCluster(current.Spec) } else { - if err := ci.depl.updateCRSpec(*newSpec); err != nil { + if err := ci.depl.updateCRSpec(ctx, *newSpec); err != nil { log.Warn().Err(err).Msg("Failed to update current deployment") return errors.WithStack(err) } @@ -288,12 +298,11 @@ func (ci *clusterScalingIntegration) GetLastNumberOfServers() arangod.NumberOfSe } // DisableScalingCluster disables scaling DBservers and coordinators -func (ci *clusterScalingIntegration) DisableScalingCluster() error { +func (ci *clusterScalingIntegration) DisableScalingCluster(ctx context.Context) error { ci.scaleEnabled.mutex.Lock() defer ci.scaleEnabled.mutex.Unlock() // Turn off scaling DBservers and coordinators in arangoDB for the UI - ctx := context.Background() if err := ci.depl.SetNumberOfServers(ctx, nil, nil); err != nil { return errors.WithStack(err) } @@ -303,7 +312,7 @@ func (ci *clusterScalingIntegration) DisableScalingCluster() error { } // EnableScalingCluster enables scaling DBservers and coordinators -func (ci *clusterScalingIntegration) EnableScalingCluster() error { +func (ci *clusterScalingIntegration) EnableScalingCluster(ctx context.Context) error { ci.scaleEnabled.mutex.Lock() defer ci.scaleEnabled.mutex.Unlock() @@ -311,15 +320,14 @@ func (ci *clusterScalingIntegration) EnableScalingCluster() error { return nil } - if err := ci.setNumberOfServers(); err != nil { + if err := ci.setNumberOfServers(ctx); err != nil { return errors.WithStack(err) } ci.scaleEnabled.enabled = true return nil } -func (ci *clusterScalingIntegration) setNumberOfServers() error { - ctx := context.Background() +func (ci *clusterScalingIntegration) setNumberOfServers(ctx context.Context) error { spec := ci.depl.GetSpec() numOfCoordinators := spec.Coordinators.GetCount() numOfDBServers := spec.DBServers.GetCount() diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index 2a45dff56..867a68d83 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -66,9 +67,14 @@ import ( v1 "k8s.io/api/core/v1" ) +var _ resources.Context = &Deployment{} + // GetBackup receives information about a backup resource -func (d *Deployment) GetBackup(backup string) (*backupApi.ArangoBackup, error) { - return d.deps.DatabaseCRCli.BackupV1().ArangoBackups(d.Namespace()).Get(context.Background(), backup, meta.GetOptions{}) +func (d *Deployment) GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + return d.deps.DatabaseCRCli.BackupV1().ArangoBackups(d.Namespace()).Get(ctxChild, backup, meta.GetOptions{}) } // GetAPIObject returns the deployment as k8s object. @@ -147,14 +153,14 @@ func (d *Deployment) getStatus() (api.DeploymentStatus, int32) { // updates the resources in k8s. // If the given last version does not match the actual last version of the status object, // an error is returned. -func (d *Deployment) UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error { +func (d *Deployment) UpdateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error { d.status.mutex.Lock() defer d.status.mutex.Unlock() - return d.updateStatus(status, lastVersion, force...) + return d.updateStatus(ctx, status, lastVersion, force...) } -func (d *Deployment) updateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error { +func (d *Deployment) updateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error { if d.status.version != lastVersion { // Status is obsolete d.deps.Log.Error(). @@ -165,14 +171,14 @@ func (d *Deployment) updateStatus(status api.DeploymentStatus, lastVersion int32 } d.status.version++ d.status.last = *status.DeepCopy() - if err := d.updateCRStatus(force...); err != nil { + if err := d.updateCRStatus(ctx, force...); err != nil { return errors.WithStack(err) } return nil } // UpdateMember updates the deployment status wrt the given member. -func (d *Deployment) UpdateMember(member api.MemberStatus) error { +func (d *Deployment) UpdateMember(ctx context.Context, member api.MemberStatus) error { status, lastVersion := d.GetStatus() _, group, found := status.Members.ElementByID(member.ID) if !found { @@ -181,7 +187,7 @@ func (d *Deployment) UpdateMember(member api.MemberStatus) error { if err := status.Members.Update(member, group); err != nil { return errors.WithStack(err) } - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(ctx, status, lastVersion); err != nil { d.deps.Log.Debug().Err(err).Msg("Updating CR status failed") return errors.WithStack(err) } @@ -324,7 +330,7 @@ func (d *Deployment) GetSyncServerClient(ctx context.Context, group api.ServerGr ns := d.apiObject.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := d.apiObject.Spec.Sync.Monitoring.GetTokenSecretName() - monitoringToken, err := k8sutil.GetTokenSecret(secrets, secretName) + monitoringToken, err := k8sutil.GetTokenSecret(ctx, secrets, secretName) if err != nil { log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync monitoring secret") return nil, errors.WithStack(err) @@ -355,7 +361,7 @@ func (d *Deployment) GetSyncServerClient(ctx context.Context, group api.ServerGr // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. -func (d *Deployment) CreateMember(group api.ServerGroup, id string) (string, error) { +func (d *Deployment) CreateMember(ctx context.Context, group api.ServerGroup, id string) (string, error) { log := d.deps.Log status, lastVersion := d.GetStatus() id, err := createMember(log, &status, group, id, d.apiObject) @@ -364,7 +370,7 @@ func (d *Deployment) CreateMember(group api.ServerGroup, id string) (string, err return "", errors.WithStack(err) } // Save added member - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(ctx, status, lastVersion); err != nil { log.Debug().Err(err).Msg("Updating CR status failed") return "", errors.WithStack(err) } @@ -375,16 +381,22 @@ func (d *Deployment) CreateMember(group api.ServerGroup, id string) (string, err } // GetPod returns pod. -func (d *Deployment) GetPod(podName string) (*v1.Pod, error) { - return d.deps.KubeCli.CoreV1().Pods(d.GetNamespace()).Get(context.Background(), podName, meta.GetOptions{}) +func (d *Deployment) GetPod(ctx context.Context, podName string) (*v1.Pod, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + return d.deps.KubeCli.CoreV1().Pods(d.GetNamespace()).Get(ctxChild, podName, meta.GetOptions{}) } // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. -func (d *Deployment) DeletePod(podName string) error { +func (d *Deployment) DeletePod(ctx context.Context, podName string) error { log := d.deps.Log ns := d.apiObject.GetNamespace() - if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(context.Background(), podName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return d.deps.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, meta.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pod", podName).Msg("Failed to remove pod") return errors.WithStack(err) } @@ -393,13 +405,16 @@ func (d *Deployment) DeletePod(podName string) error { // CleanupPod deletes a given pod with force and explicit UID. // If the pod does not exist, the error is ignored. -func (d *Deployment) CleanupPod(p *v1.Pod) error { +func (d *Deployment) CleanupPod(ctx context.Context, p *v1.Pod) error { log := d.deps.Log podName := p.GetName() ns := p.GetNamespace() options := meta.NewDeleteOptions(0) options.Preconditions = meta.NewUIDPreconditions(string(p.GetUID())) - if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(context.Background(), podName, *options); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return d.deps.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, *options) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pod", podName).Msg("Failed to cleanup pod") return errors.WithStack(err) } @@ -408,18 +423,23 @@ func (d *Deployment) CleanupPod(p *v1.Pod) error { // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. -func (d *Deployment) RemovePodFinalizers(podName string) error { +func (d *Deployment) RemovePodFinalizers(ctx context.Context, podName string) error { log := d.deps.Log ns := d.GetNamespace() kubecli := d.deps.KubeCli - p, err := kubecli.CoreV1().Pods(ns).Get(context.Background(), podName, meta.GetOptions{}) + + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + p, err := kubecli.CoreV1().Pods(ns).Get(ctxChild, podName, meta.GetOptions{}) if err != nil { if k8sutil.IsNotFound(err) { return nil } return errors.WithStack(err) } - if err := k8sutil.RemovePodFinalizers(log, d.deps.KubeCli, p, p.GetFinalizers(), true); err != nil { + + err = k8sutil.RemovePodFinalizers(ctx, log, d.deps.KubeCli, p, p.GetFinalizers(), true) + if err != nil { return errors.WithStack(err) } return nil @@ -427,10 +447,13 @@ func (d *Deployment) RemovePodFinalizers(podName string) error { // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. -func (d *Deployment) DeletePvc(pvcName string) error { +func (d *Deployment) DeletePvc(ctx context.Context, pvcName string) error { log := d.deps.Log ns := d.apiObject.GetNamespace() - if err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(ns).Delete(context.Background(), pvcName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return d.deps.KubeCli.CoreV1().PersistentVolumeClaims(ns).Delete(ctxChild, pvcName, meta.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pvc", pvcName).Msg("Failed to remove pvc") return errors.WithStack(err) } @@ -439,8 +462,11 @@ func (d *Deployment) DeletePvc(pvcName string) error { // UpdatePvc updated a persistent volume claim in the namespace // of the deployment. If the pvc does not exist, the error is ignored. -func (d *Deployment) UpdatePvc(pvc *v1.PersistentVolumeClaim) error { - _, err := d.GetKubeCli().CoreV1().PersistentVolumeClaims(d.GetNamespace()).Update(context.Background(), pvc, meta.UpdateOptions{}) +func (d *Deployment) UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := d.GetKubeCli().CoreV1().PersistentVolumeClaims(d.GetNamespace()).Update(ctxChild, pvc, meta.UpdateOptions{}) + return err + }) if err == nil { return nil } @@ -471,8 +497,11 @@ func (d *Deployment) GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) { } // GetPvc gets a PVC by the given name, in the samespace of the deployment. -func (d *Deployment) GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) { - pvc, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).Get(context.Background(), pvcName, meta.GetOptions{}) +func (d *Deployment) GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + pvc, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).Get(ctxChild, pvcName, meta.GetOptions{}) if err != nil { log.Debug().Err(err).Str("pvc-name", pvcName).Msg("Failed to get PVC") return nil, errors.WithStack(err) @@ -495,10 +524,13 @@ func (d *Deployment) GetTLSKeyfile(group api.ServerGroup, member api.MemberStatu // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. -func (d *Deployment) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error { +func (d *Deployment) DeleteTLSKeyfile(ctx context.Context, group api.ServerGroup, member api.MemberStatus) error { secretName := k8sutil.CreateTLSKeyfileSecretName(d.apiObject.GetName(), group.AsRole(), member.ID) ns := d.apiObject.GetNamespace() - if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(context.Background(), secretName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return d.deps.KubeCli.CoreV1().Secrets(ns).Delete(ctxChild, secretName, meta.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { return errors.WithStack(err) } return nil @@ -524,12 +556,12 @@ func (d *Deployment) InvalidateSyncStatus() { d.resources.InvalidateSyncStatus() } -func (d *Deployment) DisableScalingCluster() error { - return d.clusterScalingIntegration.DisableScalingCluster() +func (d *Deployment) DisableScalingCluster(ctx context.Context) error { + return d.clusterScalingIntegration.DisableScalingCluster(ctx) } -func (d *Deployment) EnableScalingCluster() error { - return d.clusterScalingIntegration.EnableScalingCluster() +func (d *Deployment) EnableScalingCluster(ctx context.Context) error { + return d.clusterScalingIntegration.EnableScalingCluster(ctx) } // GetAgencyPlan returns agency plan @@ -546,8 +578,8 @@ func (d *Deployment) GetAgencyData(ctx context.Context, i interface{}, keyParts return err } -func (d *Deployment) RenderPodForMember(cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) { - return d.resources.RenderPodForMember(cachedStatus, spec, status, memberID, imageInfo) +func (d *Deployment) RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) { + return d.resources.RenderPodForMember(ctx, cachedStatus, spec, status, memberID, imageInfo) } func (d *Deployment) SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) { @@ -562,7 +594,7 @@ func (d *Deployment) GetArangoImage() string { return d.config.ArangoImage } -func (d *Deployment) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error { +func (d *Deployment) WithStatusUpdate(ctx context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error { d.status.mutex.Lock() defer d.status.mutex.Unlock() @@ -574,7 +606,7 @@ func (d *Deployment) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, return nil } - return d.updateStatus(status, version, force...) + return d.updateStatus(ctx, status, version, force...) } func (d *Deployment) SecretsInterface() k8sutil.SecretInterface { @@ -585,8 +617,11 @@ func (d *Deployment) GetName() string { return d.apiObject.GetName() } -func (d *Deployment) GetOwnedPods() ([]v1.Pod, error) { - pods, err := d.GetKubeCli().CoreV1().Pods(d.apiObject.GetNamespace()).List(context.Background(), meta.ListOptions{}) +func (d *Deployment) GetOwnedPods(ctx context.Context) ([]v1.Pod, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + pods, err := d.GetKubeCli().CoreV1().Pods(d.apiObject.GetNamespace()).List(ctxChild, meta.ListOptions{}) if err != nil { return nil, err } diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 7ae06a836..a0a3b5b2f 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -222,24 +223,24 @@ func (d *Deployment) run() { if d.GetPhase() == api.DeploymentPhaseNone { // Create service monitor if d.haveServiceMonitorCRD { - if err := d.resources.EnsureServiceMonitor(); err != nil { + if err := d.resources.EnsureServiceMonitor(context.TODO()); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Failed to create service monitor", err, d.GetAPIObject())) } } // Create members - if err := d.createInitialMembers(d.apiObject); err != nil { + if err := d.createInitialMembers(context.TODO(), d.apiObject); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Failed to create initial members", err, d.GetAPIObject())) } // Create Pod Disruption Budgets - if err := d.resources.EnsurePDBs(); err != nil { + if err := d.resources.EnsurePDBs(context.TODO()); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Failed to create pdbs", err, d.GetAPIObject())) } status, lastVersion := d.GetStatus() status.Phase = api.DeploymentPhaseRunning - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(context.TODO(), status, lastVersion); err != nil { log.Warn().Err(err).Msg("update initial CR status failed") } log.Info().Msg("start running...") @@ -257,10 +258,10 @@ func (d *Deployment) run() { } // Remove finalizers from created resources log.Info().Msg("Deployment removed, removing finalizers to prevent orphaned resources") - if err := d.removePodFinalizers(cachedStatus); err != nil { + if err := d.removePodFinalizers(context.TODO(), cachedStatus); err != nil { log.Warn().Err(err).Msg("Failed to remove Pod finalizers") } - if err := d.removePVCFinalizers(cachedStatus); err != nil { + if err := d.removePVCFinalizers(context.TODO(), cachedStatus); err != nil { log.Warn().Err(err).Msg("Failed to remove PVC finalizers") } // We're being stopped. @@ -284,7 +285,7 @@ func (d *Deployment) run() { d.lookForServiceMonitorCRD() case <-d.updateDeploymentTrigger.Done(): inspectionInterval = minInspectionInterval - if err := d.handleArangoDeploymentUpdatedEvent(); err != nil { + if err := d.handleArangoDeploymentUpdatedEvent(context.TODO()); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Failed to handle deployment update", err, d.GetAPIObject())) } @@ -298,11 +299,13 @@ func (d *Deployment) run() { } // handleArangoDeploymentUpdatedEvent is called when the deployment is updated by the user. -func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { +func (d *Deployment) handleArangoDeploymentUpdatedEvent(ctx context.Context) error { log := d.deps.Log.With().Str("deployment", d.apiObject.GetName()).Logger() // Get the most recent version of the deployment from the API server - current, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(context.Background(), d.apiObject.GetName(), metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + current, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(ctxChild, d.apiObject.GetName(), metav1.GetOptions{}) if err != nil { log.Debug().Err(err).Msg("Failed to get current version of deployment from API server") if k8sutil.IsNotFound(err) { @@ -328,7 +331,7 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { if err := newAPIObject.Spec.Validate(); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Validation failed", err, d.apiObject)) // Try to reset object - if err := d.updateCRSpec(d.apiObject.Spec, true); err != nil { + if err := d.updateCRSpec(ctx, d.apiObject.Spec, true); err != nil { log.Error().Err(err).Msg("Restore original spec failed") d.CreateEvent(k8sutil.NewErrorEvent("Restore original failed", err, d.apiObject)) } @@ -342,7 +345,7 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { } // Save updated spec - if err := d.updateCRSpec(newAPIObject.Spec, true); err != nil { + if err := d.updateCRSpec(ctx, newAPIObject.Spec, true); err != nil { return errors.WithStack(errors.Newf("failed to update ArangoDeployment spec: %v", err)) } // Save updated accepted spec @@ -354,7 +357,7 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { status.ForceStatusReload = nil } status.AcceptedSpec = newAPIObject.Spec.DeepCopy() - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(errors.Newf("failed to update ArangoDeployment status: %v", err)) } } @@ -377,7 +380,7 @@ func (d *Deployment) CreateEvent(evt *k8sutil.Event) { } // Update the status of the API object from the internal status -func (d *Deployment) updateCRStatus(force ...bool) error { +func (d *Deployment) updateCRStatus(ctx context.Context, force ...bool) error { if len(force) == 0 || !force[0] { if d.apiObject.Status.Equal(d.status.last) { // Nothing has changed @@ -396,7 +399,14 @@ func (d *Deployment) updateCRStatus(force ...bool) error { if update.GetDeletionTimestamp() == nil { ensureFinalizers(update) } - newAPIObject, err := depls.Update(context.Background(), update, metav1.UpdateOptions{}) + + var newAPIObject *api.ArangoDeployment + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + newAPIObject, err = depls.Update(ctxChild, update, metav1.UpdateOptions{}) + + return err + }) if err == nil { // Update internal object d.apiObject = newAPIObject @@ -406,7 +416,13 @@ func (d *Deployment) updateCRStatus(force ...bool) error { // API object may have been changed already, // Reload api object and try again var current *api.ArangoDeployment - current, err = depls.Get(context.Background(), update.GetName(), metav1.GetOptions{}) + + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + current, err = depls.Get(ctxChild, update.GetName(), metav1.GetOptions{}) + + return err + }) if err == nil { update = current.DeepCopy() continue @@ -422,7 +438,7 @@ func (d *Deployment) updateCRStatus(force ...bool) error { // Update the spec part of the API object (d.apiObject) // to the given object, while preserving the status. // On success, d.apiObject is updated. -func (d *Deployment) updateCRSpec(newSpec api.DeploymentSpec, force ...bool) error { +func (d *Deployment) updateCRSpec(ctx context.Context, newSpec api.DeploymentSpec, force ...bool) error { if len(force) == 0 || !force[0] { if d.apiObject.Spec.Equal(&newSpec) { @@ -440,7 +456,13 @@ func (d *Deployment) updateCRSpec(newSpec api.DeploymentSpec, force ...bool) err update.Spec = newSpec update.Status = d.status.last ns := d.apiObject.GetNamespace() - newAPIObject, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Update(context.Background(), update, metav1.UpdateOptions{}) + var newAPIObject *api.ArangoDeployment + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + newAPIObject, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Update(ctxChild, update, metav1.UpdateOptions{}) + + return err + }) if err == nil { // Update internal object d.apiObject = newAPIObject @@ -450,7 +472,13 @@ func (d *Deployment) updateCRSpec(newSpec api.DeploymentSpec, force ...bool) err // API object may have been changed already, // Reload api object and try again var current *api.ArangoDeployment - current, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Get(context.Background(), update.GetName(), metav1.GetOptions{}) + + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + current, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Get(ctxChild, update.GetName(), metav1.GetOptions{}) + + return err + }) if err == nil { update = current.DeepCopy() continue @@ -504,12 +532,17 @@ func (d *Deployment) lookForServiceMonitorCRD() { // SetNumberOfServers adjust number of DBservers and coordinators in arangod func (d *Deployment) SetNumberOfServers(ctx context.Context, noCoordinators, noDBServers *int) error { - c, err := d.clientCache.GetDatabase(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := d.clientCache.GetDatabase(ctxChild) if err != nil { return errors.WithStack(err) } - err = arangod.SetNumberOfServers(ctx, c.Connection(), noCoordinators, noDBServers) + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return arangod.SetNumberOfServers(ctxChild, c.Connection(), noCoordinators, noDBServers) + }) + if err != nil { return errors.WithStack(err) } @@ -520,7 +553,7 @@ func (d *Deployment) getArangoDeployment() *api.ArangoDeployment { return d.apiObject } -func (d *Deployment) ApplyPatch(p ...patch.Item) error { +func (d *Deployment) ApplyPatch(ctx context.Context, p ...patch.Item) error { parser := patch.Patch(p) data, err := parser.Marshal() @@ -530,7 +563,9 @@ func (d *Deployment) ApplyPatch(p ...patch.Item) error { c := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()) - depl, err := c.Patch(context.Background(), d.apiObject.GetName(), types.JSONPatchType, data, metav1.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + depl, err := c.Patch(ctxChild, d.apiObject.GetName(), types.JSONPatchType, data, metav1.PatchOptions{}) if err != nil { return err } diff --git a/pkg/deployment/deployment_finalizers.go b/pkg/deployment/deployment_finalizers.go index 62f4b84e6..8229e0beb 100644 --- a/pkg/deployment/deployment_finalizers.go +++ b/pkg/deployment/deployment_finalizers.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -56,7 +57,9 @@ func (d *Deployment) runDeploymentFinalizers(ctx context.Context, cachedStatus i var removalList []string depls := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.GetNamespace()) - updated, err := depls.Get(context.Background(), d.apiObject.GetName(), metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + updated, err := depls.Get(ctxChild, d.apiObject.GetName(), metav1.GetOptions{}) if err != nil { return errors.WithStack(err) } @@ -73,7 +76,7 @@ func (d *Deployment) runDeploymentFinalizers(ctx context.Context, cachedStatus i } // Remove finalizers (if needed) if len(removalList) > 0 { - if err := removeDeploymentFinalizers(log, d.deps.DatabaseCRCli, updated, removalList); err != nil { + if err := removeDeploymentFinalizers(ctx, log, d.deps.DatabaseCRCli, updated, removalList); err != nil { log.Debug().Err(err).Msg("Failed to update ArangoDeployment (to remove finalizers)") return errors.WithStack(err) } @@ -83,11 +86,11 @@ func (d *Deployment) runDeploymentFinalizers(ctx context.Context, cachedStatus i // inspectRemoveChildFinalizers checks the finalizer condition for remove-child-finalizers. // It returns nil if the finalizer can be removed. -func (d *Deployment) inspectRemoveChildFinalizers(ctx context.Context, log zerolog.Logger, depl *api.ArangoDeployment, cachedStatus inspectorInterface.Inspector) error { - if err := d.removePodFinalizers(cachedStatus); err != nil { +func (d *Deployment) inspectRemoveChildFinalizers(ctx context.Context, _ zerolog.Logger, _ *api.ArangoDeployment, cachedStatus inspectorInterface.Inspector) error { + if err := d.removePodFinalizers(ctx, cachedStatus); err != nil { return errors.WithStack(err) } - if err := d.removePVCFinalizers(cachedStatus); err != nil { + if err := d.removePVCFinalizers(ctx, cachedStatus); err != nil { return errors.WithStack(err) } @@ -95,10 +98,14 @@ func (d *Deployment) inspectRemoveChildFinalizers(ctx context.Context, log zerol } // removeDeploymentFinalizers removes the given finalizers from the given PVC. -func removeDeploymentFinalizers(log zerolog.Logger, cli versioned.Interface, depl *api.ArangoDeployment, finalizers []string) error { +func removeDeploymentFinalizers(ctx context.Context, log zerolog.Logger, cli versioned.Interface, + depl *api.ArangoDeployment, finalizers []string) error { depls := cli.DatabaseV1().ArangoDeployments(depl.GetNamespace()) getFunc := func() (metav1.Object, error) { - result, err := depls.Get(context.Background(), depl.GetName(), metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + result, err := depls.Get(ctxChild, depl.GetName(), metav1.GetOptions{}) if err != nil { return nil, errors.WithStack(err) } @@ -106,7 +113,10 @@ func removeDeploymentFinalizers(log zerolog.Logger, cli versioned.Interface, dep } updateFunc := func(updated metav1.Object) error { updatedDepl := updated.(*api.ArangoDeployment) - result, err := depls.Update(context.Background(), updatedDepl, metav1.UpdateOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + result, err := depls.Update(ctxChild, updatedDepl, metav1.UpdateOptions{}) if err != nil { return errors.WithStack(err) } diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 3f6e3ee72..f74be1e8a 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -45,10 +46,33 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + timeoutReconciliationPerNode = time.Second * 20 +) + var ( inspectDeploymentDurationGauges = metrics.MustRegisterGaugeVec(metricsComponent, "inspect_deployment_duration", "Amount of time taken by a single inspection of a deployment (in sec)", metrics.DeploymentName) ) +// getReconciliationTimeout gets timeout for the reconciliation loop. +// The whole reconciliation loop timeout depends on the number of nodes but not less then one minute. +func (d *Deployment) getReconciliationTimeout() (time.Duration, error) { + ctx, cancel := context.WithTimeout(context.TODO(), k8sutil.GetRequestTimeout()) + defer cancel() + + nodes, err := d.GetKubeCli().CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return 0, errors.Wrapf(err, "Unable to get nodes") + } + + if timeout := timeoutReconciliationPerNode * time.Duration(len(nodes.Items)); timeout > time.Minute { + return timeout, nil + } + + // The minimum timeout for the reconciliation loop. + return time.Minute, nil +} + // inspectDeployment inspects the entire deployment, creates // a plan to update if needed and inspects underlying resources. // This function should be called when: @@ -59,13 +83,22 @@ var ( func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval { log := d.deps.Log start := time.Now() + + timeout, err := d.getReconciliationTimeout() + if err != nil { + log.Error().Err(err).Msg("Unable to get nodes") + return minInspectionInterval // Retry ASAP + } + + ctxReconciliation, cancelReconciliation := context.WithTimeout(context.Background(), timeout) + defer cancelReconciliation() defer func() { d.deps.Log.Info().Msgf("Inspect loop took %s", time.Since(start)) }() nextInterval := lastInterval hasError := false - ctx := context.Background() + deploymentName := d.apiObject.GetName() defer metrics.SetDuration(inspectDeploymentDurationGauges.WithLabelValues(deploymentName), start) @@ -76,7 +109,12 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval } // Check deployment still exists - updated, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(context.Background(), deploymentName, metav1.GetOptions{}) + var updated *api.ArangoDeployment + err = k8sutil.RunWithTimeout(ctxReconciliation, func(ctxChild context.Context) error { + var err error + updated, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(ctxChild, deploymentName, metav1.GetOptions{}) + return err + }) if k8sutil.IsNotFound(err) { // Deployment is gone log.Info().Msg("Deployment is gone") @@ -84,7 +122,7 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval return nextInterval } else if updated != nil && updated.GetDeletionTimestamp() != nil { // Deployment is marked for deletion - if err := d.runDeploymentFinalizers(ctx, cachedStatus); err != nil { + if err := d.runDeploymentFinalizers(ctxReconciliation, cachedStatus); err != nil { hasError = true d.CreateEvent(k8sutil.NewErrorEvent("ArangoDeployment finalizer inspection failed", err, d.apiObject)) } @@ -105,7 +143,8 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval d.apiObject = updated - if inspectNextInterval, err := d.inspectDeploymentWithError(ctx, nextInterval, cachedStatus); err != nil { + inspectNextInterval, err := d.inspectDeploymentWithError(ctxReconciliation, nextInterval, cachedStatus) + if err != nil { if !operatorErrors.IsReconcile(err) { nextInterval = inspectNextInterval hasError = true @@ -129,7 +168,8 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval return nextInterval.ReduceTo(maxInspectionInterval) } -func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterval util.Interval, cachedStatus inspectorInterface.Inspector) (nextInterval util.Interval, inspectError error) { +func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterval util.Interval, + cachedStatus inspectorInterface.Inspector) (nextInterval util.Interval, inspectError error) { t := time.Now() d.SetCachedStatus(cachedStatus) @@ -152,7 +192,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } else { condition, exists := status.Conditions.Get(api.ConditionTypeUpToDate) if checksum != status.AppliedVersion && (!exists || condition.IsTrue()) { - if err = d.updateCondition(api.ConditionTypeUpToDate, false, "Spec Changed", "Spec Object changed. Waiting until plan will be applied"); err != nil { + if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, false, "Spec Changed", "Spec Object changed. Waiting until plan will be applied"); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition") } @@ -161,26 +201,26 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } // Cleanup terminated pods on the beginning of loop - if x, err := d.resources.CleanupTerminatedPods(cachedStatus); err != nil { + if x, err := d.resources.CleanupTerminatedPods(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Pod cleanup failed") } else { nextInterval = nextInterval.ReduceTo(x) } - if err := d.resources.EnsureArangoMembers(cachedStatus); err != nil { + if err := d.resources.EnsureArangoMembers(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "ArangoMember creation failed") } - if err := d.resources.EnsureServices(cachedStatus); err != nil { + if err := d.resources.EnsureServices(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Service creation failed") } - if err := d.resources.EnsureSecrets(d.deps.Log, cachedStatus); err != nil { + if err := d.resources.EnsureSecrets(ctx, d.deps.Log, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Secret creation failed") } // Inspect secret hashes - if err := d.resources.ValidateSecretHashes(cachedStatus); err != nil { + if err := d.resources.ValidateSecretHashes(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Secret hash validation failed") } @@ -195,7 +235,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } // Ensure we have image info - if retrySoon, exists, err := d.ensureImages(d.apiObject); err != nil { + if retrySoon, exists, err := d.ensureImages(ctx, d.apiObject); err != nil { return minInspectionInterval, errors.Wrapf(err, "Image detection failed") } else if retrySoon || !exists { return minInspectionInterval, nil @@ -215,16 +255,16 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } // Check members for resilience - if err := d.resilience.CheckMemberFailure(); err != nil { + if err := d.resilience.CheckMemberFailure(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "Member failure detection failed") } // Immediate actions - if err := d.reconciler.CheckDeployment(); err != nil { + if err := d.reconciler.CheckDeployment(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "Reconciler immediate actions failed") } - if interval, err := d.ensureResources(nextInterval, cachedStatus); err != nil { + if interval, err := d.ensureResources(ctx, nextInterval, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Reconciler resource recreation failed") } else { nextInterval = interval @@ -232,11 +272,11 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva // Create scale/update plan if _, ok := d.apiObject.Annotations[deployment.ArangoDeploymentPlanCleanAnnotation]; ok { - if err := d.ApplyPatch(patch.ItemRemove(patch.NewPath("metadata", "annotations", deployment.ArangoDeploymentPlanCleanAnnotation))); err != nil { + if err := d.ApplyPatch(ctx, patch.ItemRemove(patch.NewPath("metadata", "annotations", deployment.ArangoDeploymentPlanCleanAnnotation))); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to create remove annotation patch") } - if err := d.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := d.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.Plan = nil return true }, true); err != nil { @@ -249,7 +289,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } if d.apiObject.Status.Plan.IsEmpty() && status.AppliedVersion != checksum { - if err := d.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := d.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.AppliedVersion = checksum return true }); err != nil { @@ -259,7 +299,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva return minInspectionInterval, nil } else if status.AppliedVersion == checksum { if !status.Plan.IsEmpty() && status.Conditions.IsTrue(api.ConditionTypeUpToDate) { - if err = d.updateCondition(api.ConditionTypeUpToDate, false, "Plan is not empty", "There are pending operations in plan"); err != nil { + if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, false, "Plan is not empty", "There are pending operations in plan"); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition") } @@ -267,7 +307,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } if status.Plan.IsEmpty() && !status.Conditions.IsTrue(api.ConditionTypeUpToDate) { - if err = d.updateCondition(api.ConditionTypeUpToDate, true, "Spec is Up To Date", "Spec is Up To Date"); err != nil { + if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, true, "Spec is Up To Date", "Spec is Up To Date"); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition") } @@ -285,17 +325,17 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } // Create access packages - if err := d.createAccessPackages(); err != nil { + if err := d.createAccessPackages(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "AccessPackage creation failed") } // Inspect deployment for obsolete members - if err := d.resources.CleanupRemovedMembers(); err != nil { + if err := d.resources.CleanupRemovedMembers(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "Removed member cleanup failed") } // At the end of the inspect, we cleanup terminated pods. - if x, err := d.resources.CleanupTerminatedPods(cachedStatus); err != nil { + if x, err := d.resources.CleanupTerminatedPods(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Pod cleanup failed") } else { nextInterval = nextInterval.ReduceTo(x) @@ -304,31 +344,31 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva return } -func (d *Deployment) ensureResources(lastInterval util.Interval, cachedStatus inspectorInterface.Inspector) (util.Interval, error) { +func (d *Deployment) ensureResources(ctx context.Context, lastInterval util.Interval, cachedStatus inspectorInterface.Inspector) (util.Interval, error) { // Ensure all resources are created if d.haveServiceMonitorCRD { - if err := d.resources.EnsureServiceMonitor(); err != nil { + if err := d.resources.EnsureServiceMonitor(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "Service monitor creation failed") } } - if err := d.resources.EnsurePVCs(cachedStatus); err != nil { + if err := d.resources.EnsurePVCs(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "PVC creation failed") } - if err := d.resources.EnsurePods(cachedStatus); err != nil { + if err := d.resources.EnsurePods(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Pod creation failed") } - if err := d.resources.EnsurePDBs(); err != nil { + if err := d.resources.EnsurePDBs(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "PDB creation failed") } - if err := d.resources.EnsureAnnotations(cachedStatus); err != nil { + if err := d.resources.EnsureAnnotations(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Annotation update failed") } - if err := d.resources.EnsureLabels(cachedStatus); err != nil { + if err := d.resources.EnsureLabels(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Labels update failed") } @@ -345,9 +385,9 @@ func (d *Deployment) triggerCRDInspection() { d.inspectCRDTrigger.Trigger() } -func (d *Deployment) updateCondition(conditionType api.ConditionType, status bool, reason, message string) error { +func (d *Deployment) updateCondition(ctx context.Context, conditionType api.ConditionType, status bool, reason, message string) error { d.deps.Log.Info().Str("condition", string(conditionType)).Bool("status", status).Str("reason", reason).Str("message", message).Msg("Updated condition") - if err := d.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := d.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { return s.Conditions.Update(conditionType, status, reason, message) }); err != nil { return errors.Wrapf(err, "Unable to update condition") diff --git a/pkg/deployment/deployment_pod_sync_test.go b/pkg/deployment/deployment_pod_sync_test.go index 5b3ed53bb..f63c8bb64 100644 --- a/pkg/deployment/deployment_pod_sync_test.go +++ b/pkg/deployment/deployment_pod_sync_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -227,7 +227,7 @@ func TestEnsurePod_Sync_Master(t *testing.T) { testCase.createTestPodData(deployment, api.ServerGroupSyncMasters, firstSyncMaster) name := testCase.ArangoDeployment.Spec.Sync.Monitoring.GetTokenSecretName() - auth, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + auth, err := k8sutil.GetTokenSecret(context.Background(), deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) require.NoError(t, err) testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( @@ -306,7 +306,7 @@ func TestEnsurePod_Sync_Master(t *testing.T) { testCase.createTestPodData(deployment, api.ServerGroupSyncMasters, firstSyncMaster) name := testCase.ArangoDeployment.Spec.Sync.Monitoring.GetTokenSecretName() - auth, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + auth, err := k8sutil.GetTokenSecret(context.Background(), deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) require.NoError(t, err) testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( @@ -408,7 +408,7 @@ func TestEnsurePod_Sync_Worker(t *testing.T) { testCase.createTestPodData(deployment, api.ServerGroupSyncWorkers, firstSyncWorker) name := testCase.ArangoDeployment.Spec.Sync.Monitoring.GetTokenSecretName() - auth, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + auth, err := k8sutil.GetTokenSecret(context.Background(), deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) require.NoError(t, err) testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( diff --git a/pkg/deployment/deployment_run_test.go b/pkg/deployment/deployment_run_test.go index f205b0c9a..e273af240 100644 --- a/pkg/deployment/deployment_run_test.go +++ b/pkg/deployment/deployment_run_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package deployment @@ -66,7 +67,7 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { for { cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) require.NoError(t, err) - err = d.resources.EnsureSecrets(log.Logger, cache) + err = d.resources.EnsureSecrets(context.Background(), log.Logger, cache) if err == nil { break } @@ -143,7 +144,7 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { // Act cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) require.NoError(t, err) - err = d.resources.EnsurePods(cache) + err = d.resources.EnsurePods(context.Background(), cache) // Assert if testCase.ExpectedError != nil { diff --git a/pkg/deployment/deployment_suite_test.go b/pkg/deployment/deployment_suite_test.go index 07b288cdb..183729a26 100644 --- a/pkg/deployment/deployment_suite_test.go +++ b/pkg/deployment/deployment_suite_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package deployment import ( + "context" "fmt" "io/ioutil" "os" @@ -102,7 +104,7 @@ func createTestLifecycle() *core.Lifecycle { func createTestToken(deployment *Deployment, testCase *testCaseStruct, paths []string) (string, error) { name := testCase.ArangoDeployment.Spec.Authentication.GetJWTSecretName() - s, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + s, err := k8sutil.GetTokenSecret(context.Background(), deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) if err != nil { return "", err } diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index a0463d273..4763eec31 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -80,7 +81,7 @@ type imagesBuilder struct { // ensureImages creates pods needed to detect ImageID for specified images. // Returns: retrySoon, error -func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, bool, error) { +func (d *Deployment) ensureImages(ctx context.Context, apiObject *api.ArangoDeployment) (bool, bool, error) { status, lastVersion := d.GetStatus() ib := imagesBuilder{ APIObject: apiObject, @@ -89,13 +90,13 @@ func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, bool, Log: d.deps.Log, KubeCli: d.deps.KubeCli, UpdateCRStatus: func(status api.DeploymentStatus) error { - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } return nil }, } - ctx := context.Background() + retrySoon, exists, err := ib.Run(ctx) if err != nil { return retrySoon, exists, errors.WithStack(err) @@ -134,12 +135,18 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima Logger() // Check if pod exists - if pod, err := ib.KubeCli.CoreV1().Pods(ns).Get(context.Background(), podName, metav1.GetOptions{}); err == nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + pod, err := ib.KubeCli.CoreV1().Pods(ns).Get(ctxChild, podName, metav1.GetOptions{}) + if err == nil { // Pod found if k8sutil.IsPodFailed(pod) { // Wait some time before deleting the pod if time.Now().After(pod.GetCreationTimestamp().Add(30 * time.Second)) { - if err := ib.KubeCli.CoreV1().Pods(ns).Delete(context.Background(), podName, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return ib.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, metav1.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete Image ID Pod") return false, nil } @@ -167,7 +174,9 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima log.Warn().Err(err).Msg("Failed to create Image ID Pod client") return true, nil } - v, err := client.Version(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + v, err := client.Version(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to fetch version from Image ID Pod") return true, nil @@ -176,7 +185,10 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima enterprise := strings.ToLower(v.License) == "enterprise" // We have all the info we need now, kill the pod and store the image info. - if err := ib.KubeCli.CoreV1().Pods(ns).Delete(context.Background(), podName, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return ib.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, metav1.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete Image ID Pod") return true, nil } @@ -213,13 +225,17 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima apiObject: ib.APIObject, } - pod, err := resources.RenderArangoPod(ib.APIObject, role, id, podName, args, &imagePod) + pod, err = resources.RenderArangoPod(ib.APIObject, role, id, podName, args, &imagePod) if err != nil { log.Debug().Err(err).Msg("Failed to render image ID pod") return true, errors.WithStack(err) } - if _, err := resources.CreateArangoPod(ib.KubeCli, ib.APIObject, ib.Spec, api.ServerGroupImageDiscovery, pod); err != nil { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := resources.CreateArangoPod(ctxChild, ib.KubeCli, ib.APIObject, ib.Spec, api.ServerGroupImageDiscovery, pod) + return err + }) + if err != nil { log.Debug().Err(err).Msg("Failed to create image ID pod") return true, errors.WithStack(err) } diff --git a/pkg/deployment/images_test.go b/pkg/deployment/images_test.go index 85e4666e7..6b6763a6c 100644 --- a/pkg/deployment/images_test.go +++ b/pkg/deployment/images_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -331,7 +331,7 @@ func TestEnsureImages(t *testing.T) { require.NoError(t, err) // Act - retrySoon, _, err := d.ensureImages(d.apiObject) + retrySoon, _, err := d.ensureImages(context.Background(), d.apiObject) // Assert assert.EqualValues(t, testCase.RetrySoon, retrySoon) diff --git a/pkg/deployment/members.go b/pkg/deployment/members.go index 982c699d0..5cf36eb92 100644 --- a/pkg/deployment/members.go +++ b/pkg/deployment/members.go @@ -23,6 +23,7 @@ package deployment import ( + "context" "strings" "github.com/arangodb/kube-arangodb/pkg/util/errors" @@ -37,7 +38,7 @@ import ( // createInitialMembers creates all members needed for the initial state of the deployment. // Note: This does not create any pods of PVCs -func (d *Deployment) createInitialMembers(apiObject *api.ArangoDeployment) error { +func (d *Deployment) createInitialMembers(ctx context.Context, apiObject *api.ArangoDeployment) error { log := d.deps.Log log.Debug().Msg("creating initial members...") @@ -59,7 +60,7 @@ func (d *Deployment) createInitialMembers(apiObject *api.ArangoDeployment) error // Save status log.Debug().Msg("saving initial members...") - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } // Save events diff --git a/pkg/deployment/pod/encryption.go b/pkg/deployment/pod/encryption.go index 0cf4c8eff..d5fd9d11c 100644 --- a/pkg/deployment/pod/encryption.go +++ b/pkg/deployment/pod/encryption.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package pod @@ -67,8 +68,11 @@ func GroupEncryptionSupported(mode api.DeploymentMode, group api.ServerGroup) bo } } -func GetEncryptionKey(secrets k8sutil.SecretInterface, name string) (string, []byte, bool, error) { - keyfile, err := secrets.Get(context.Background(), name, meta.GetOptions{}) +func GetEncryptionKey(ctx context.Context, secrets k8sutil.SecretInterface, name string) (string, []byte, bool, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + keyfile, err := secrets.Get(ctxChild, name, meta.GetOptions{}) if err != nil { if k8sutil.IsNotFound(err) { return "", nil, false, nil diff --git a/pkg/deployment/reconcile/action_add_member.go b/pkg/deployment/reconcile/action_add_member.go index 28e9fbdb1..29a6bdd6a 100644 --- a/pkg/deployment/reconcile/action_add_member.go +++ b/pkg/deployment/reconcile/action_add_member.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -64,7 +65,7 @@ type actionAddMember struct { // Returns true if the action is completely finished, false in case // the start time needs to be recorded and a ready condition needs to be checked. func (a *actionAddMember) Start(ctx context.Context) (bool, error) { - newID, err := a.actionCtx.CreateMember(a.action.Group, a.action.MemberID) + newID, err := a.actionCtx.CreateMember(ctx, a.action.Group, a.action.MemberID) if err != nil { log.Debug().Err(err).Msg("Failed to create member") return false, errors.WithStack(err) @@ -72,14 +73,14 @@ func (a *actionAddMember) Start(ctx context.Context) (bool, error) { a.newMemberID = newID if _, ok := a.action.Params[api.ActionTypeWaitForMemberUp.String()]; ok { - a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.Plan = append(s.Plan, api.NewAction(api.ActionTypeWaitForMemberInSync, a.action.Group, newID, "Wait for member in sync after creation")) return true }) } if _, ok := a.action.Params[api.ActionTypeWaitForMemberInSync.String()]; ok { - a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.Plan = append(s.Plan, api.NewAction(api.ActionTypeWaitForMemberInSync, a.action.Group, newID, "Wait for member in sync after creation")) return true }) diff --git a/pkg/deployment/reconcile/action_backup_restore.go b/pkg/deployment/reconcile/action_backup_restore.go index b9145ea75..3fddd3f0e 100644 --- a/pkg/deployment/reconcile/action_backup_restore.go +++ b/pkg/deployment/reconcile/action_backup_restore.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,9 +27,10 @@ import ( "context" "github.com/arangodb/go-driver" + "github.com/rs/zerolog" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" - "github.com/rs/zerolog" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" ) func init() { @@ -64,12 +66,14 @@ func (a actionBackupRestore) Start(ctx context.Context) (bool, error) { return true, nil } - dbc, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + dbc, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { return false, err } - backupResource, err := a.actionCtx.GetBackup(*spec.RestoreFrom) + backupResource, err := a.actionCtx.GetBackup(ctx, *spec.RestoreFrom) if err != nil { a.log.Error().Err(err).Msg("Unable to find backup") return true, nil @@ -80,7 +84,7 @@ func (a actionBackupRestore) Start(ctx context.Context) (bool, error) { return true, nil } - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { result := &api.DeploymentRestoreResult{ RequestedFrom: spec.GetRestoreFrom(), } @@ -94,12 +98,13 @@ func (a actionBackupRestore) Start(ctx context.Context) (bool, error) { return false, err } + // The below action can take a while so the full parent timeout context is used. restoreError := dbc.Backup().Restore(ctx, driver.BackupID(backupResource.Status.Backup.ID), nil) if restoreError != nil { a.log.Error().Err(restoreError).Msg("Restore failed") } - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { result := &api.DeploymentRestoreResult{ RequestedFrom: spec.GetRestoreFrom(), } diff --git a/pkg/deployment/reconcile/action_backup_restore_clean.go b/pkg/deployment/reconcile/action_backup_restore_clean.go index 3958a2968..74a26a2e7 100644 --- a/pkg/deployment/reconcile/action_backup_restore_clean.go +++ b/pkg/deployment/reconcile/action_backup_restore_clean.go @@ -50,7 +50,7 @@ type actionBackupRestoreClean struct { } func (a actionBackupRestoreClean) Start(ctx context.Context) (bool, error) { - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.Restore == nil { return false } diff --git a/pkg/deployment/reconcile/action_bootstrap_set_password.go b/pkg/deployment/reconcile/action_bootstrap_set_password.go index 4847c9b0c..720aab826 100644 --- a/pkg/deployment/reconcile/action_bootstrap_set_password.go +++ b/pkg/deployment/reconcile/action_bootstrap_set_password.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -27,6 +28,8 @@ import ( "crypto/rand" "encoding/hex" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/go-driver" @@ -66,14 +69,15 @@ func (a actionBootstrapSetPassword) Start(ctx context.Context) (bool, error) { a.log.Warn().Msgf("User does not exist in password hashes") return true, nil } else { - ctx, c := context.WithTimeout(context.Background(), a.Timeout(spec)) - defer c() - if password, err := a.setUserPassword(ctx, user, secret.Get()); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, a.Timeout(spec)) + defer cancel() + + if password, err := a.setUserPassword(ctxChild, user, secret.Get()); err != nil { return false, err } else { passwordSha := util.SHA256FromString(password) - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.SecretHashes == nil { s.SecretHashes = &api.SecretHashes{} } @@ -99,30 +103,44 @@ func (a actionBootstrapSetPassword) Start(ctx context.Context) (bool, error) { func (a actionBootstrapSetPassword) setUserPassword(ctx context.Context, user, secret string) (string, error) { a.log.Debug().Msgf("Bootstrapping user %s, secret %s", user, secret) - client, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + client, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { return "", errors.WithStack(err) } - password, err := a.ensureUserPasswordSecret(user, secret) + password, err := a.ensureUserPasswordSecret(ctx, user, secret) if err != nil { return "", errors.WithStack(err) } // Obtain the user - if u, err := client.User(context.Background(), user); driver.IsNotFound(err) { - _, err := client.CreateUser(context.Background(), user, &driver.UserOptions{Password: password}) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if u, err := client.User(ctxChild, user); err != nil { + if !driver.IsNotFound(err) { + return "", err + } + + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := client.CreateUser(ctxChild, user, &driver.UserOptions{Password: password}) + return err + }) + return password, errors.WithStack(err) - } else if err == nil { - return password, errors.WithStack(u.Update(context.Background(), driver.UserOptions{ - Password: password, - })) } else { - return "", err + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return u.Update(ctxChild, driver.UserOptions{ + Password: password, + }) + }) + + return password, errors.WithStack(err) } } -func (a actionBootstrapSetPassword) ensureUserPasswordSecret(user, secret string) (string, error) { +func (a actionBootstrapSetPassword) ensureUserPasswordSecret(ctx context.Context, user, secret string) (string, error) { cache := a.actionCtx.GetCachedStatus() if auth, ok := cache.Secret(secret); !ok { @@ -134,7 +152,8 @@ func (a actionBootstrapSetPassword) ensureUserPasswordSecret(user, secret string token := hex.EncodeToString(tokenData) owner := a.actionCtx.GetAPIObject().AsOwner() - if err := k8sutil.CreateBasicAuthSecret(a.actionCtx.SecretsInterface(), secret, user, token, &owner); err != nil { + err := k8sutil.CreateBasicAuthSecret(ctx, a.actionCtx.SecretsInterface(), secret, user, token, &owner) + if err != nil { return "", err } diff --git a/pkg/deployment/reconcile/action_bootstrap_update.go b/pkg/deployment/reconcile/action_bootstrap_update.go index 0b92a33e6..33d528c4d 100644 --- a/pkg/deployment/reconcile/action_bootstrap_update.go +++ b/pkg/deployment/reconcile/action_bootstrap_update.go @@ -51,7 +51,7 @@ type actionBootstrapUpdate struct { } func (a actionBootstrapUpdate) Start(ctx context.Context) (bool, error) { - if err := a.actionCtx.WithStatusUpdate(func(status *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(status *api.DeploymentStatus) bool { if errMessage, ok := a.action.GetParam("error"); ok { status.Conditions.Update(api.ConditionTypeBootstrapCompleted, true, "Bootstrap failed", fmt.Sprintf("%s", errMessage)) status.Conditions.Update(api.ConditionTypeBootstrapSucceded, false, "Bootstrap failed", fmt.Sprintf("%s", errMessage)) diff --git a/pkg/deployment/reconcile/action_cleanout_member.go b/pkg/deployment/reconcile/action_cleanout_member.go index 69f8c3af7..e2847b5ab 100644 --- a/pkg/deployment/reconcile/action_cleanout_member.go +++ b/pkg/deployment/reconcile/action_cleanout_member.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -64,19 +65,28 @@ func (a *actionCleanoutMember) Start(ctx context.Context) (bool, error) { return true, nil } log := a.log - c, err := a.actionCtx.GetDatabaseClient(ctx) + + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return false, errors.WithStack(err) } - cluster, err := c.Cluster(ctx) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + cluster, err := c.Cluster(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") return false, errors.WithStack(err) } + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() var jobID string - ctx = driver.WithJobIDResponse(ctx, &jobID) - if err := cluster.CleanOutServer(ctx, a.action.MemberID); err != nil { + ctxJobID := driver.WithJobIDResponse(ctxChild, &jobID) + if err := cluster.CleanOutServer(ctxJobID, a.action.MemberID); err != nil { log.Debug().Err(err).Msg("Failed to cleanout member") return false, errors.WithStack(err) } @@ -84,7 +94,7 @@ func (a *actionCleanoutMember) Start(ctx context.Context) (bool, error) { // Update status m.Phase = api.MemberPhaseCleanOut m.CleanoutJobID = jobID - if a.actionCtx.UpdateMember(m); err != nil { + if a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } return false, nil @@ -103,17 +113,26 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e if !m.IsInitialized { return true, false, nil } - c, err := a.actionCtx.GetDatabaseClient(ctx) + + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to create database client") return false, false, errors.WithStack(err) } - cluster, err := c.Cluster(ctx) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + cluster, err := c.Cluster(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") return false, false, errors.WithStack(err) } - cleanedOut, err := cluster.IsCleanedOut(ctx, a.action.MemberID) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + cleanedOut, err := cluster.IsCleanedOut(ctxChild, a.action.MemberID) if err != nil { log.Debug().Err(err).Msg("IsCleanedOut failed") return false, false, errors.WithStack(err) @@ -122,17 +141,25 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e // We're not done yet, check job status log.Debug().Msg("IsCleanedOut returned false") - c, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to create database client") return false, false, errors.WithStack(err) } - agency, err := a.actionCtx.GetAgency(ctx) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + agency, err := a.actionCtx.GetAgency(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return false, false, errors.WithStack(err) } - jobStatus, err := arangod.CleanoutServerJobStatus(ctx, m.CleanoutJobID, c, agency) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, m.CleanoutJobID, c, agency) if err != nil { log.Debug().Err(err).Msg("Failed to fetch cleanout job status") return false, false, errors.WithStack(err) @@ -142,7 +169,7 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e // Revert cleanout state m.Phase = api.MemberPhaseCreated m.CleanoutJobID = "" - if a.actionCtx.UpdateMember(m); err != nil { + if a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } return false, true, nil @@ -151,7 +178,7 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e } // Cleanout completed if m.Conditions.Update(api.ConditionTypeCleanedOut, true, "CleanedOut", "") { - if a.actionCtx.UpdateMember(m); err != nil { + if a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } } diff --git a/pkg/deployment/reconcile/action_cluster_member_cleanup.go b/pkg/deployment/reconcile/action_cluster_member_cleanup.go index 411450a69..eb7447aa7 100644 --- a/pkg/deployment/reconcile/action_cluster_member_cleanup.go +++ b/pkg/deployment/reconcile/action_cluster_member_cleanup.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,6 +26,8 @@ package reconcile import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -68,17 +71,23 @@ func (a *actionClusterMemberCleanup) Start(ctx context.Context) (bool, error) { func (a *actionClusterMemberCleanup) start(ctx context.Context) error { id := driver.ServerID(a.MemberID()) - c, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { return err } - cluster, err := c.Cluster(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + cluster, err := c.Cluster(ctxChild) if err != nil { return err } - health, err := cluster.Health(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + health, err := cluster.Health(ctxChild) if err != nil { return err } @@ -87,9 +96,7 @@ func (a *actionClusterMemberCleanup) start(ctx context.Context) error { return nil } - if err := cluster.RemoveServer(ctx, id); err != nil { - return err - } - - return nil + return arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return cluster.RemoveServer(ctxChild, id) + }) } diff --git a/pkg/deployment/reconcile/action_context.go b/pkg/deployment/reconcile/action_context.go index 9364516c7..ce413becc 100644 --- a/pkg/deployment/reconcile/action_context.go +++ b/pkg/deployment/reconcile/action_context.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -71,33 +72,33 @@ type ActionContext interface { GetMemberStatusByID(id string) (api.MemberStatus, bool) // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. - CreateMember(group api.ServerGroup, id string) (string, error) + CreateMember(ctx context.Context, group api.ServerGroup, id string) (string, error) // UpdateMember updates the deployment status wrt the given member. - UpdateMember(member api.MemberStatus) error + UpdateMember(ctx context.Context, member api.MemberStatus) error // RemoveMemberByID removes a member with given id. - RemoveMemberByID(id string) error + RemoveMemberByID(ctx context.Context, id string) error // GetPod returns pod. - GetPod(podName string) (*v1.Pod, error) + GetPod(ctx context.Context, podName string) (*v1.Pod, error) // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - DeletePod(podName string) error + DeletePod(ctx context.Context, podName string) error // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. - DeletePvc(pvcName string) error + DeletePvc(ctx context.Context, pvcName string) error // GetPvc returns PVC info about PVC with given name in the namespace // of the deployment. - GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) + GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) // UpdatePvc update PVC with given name in the namespace // of the deployment. - UpdatePvc(pvc *v1.PersistentVolumeClaim) error + UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - RemovePodFinalizers(podName string) error + RemovePodFinalizers(ctx context.Context, podName string) error // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. - DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error + DeleteTLSKeyfile(ctx context.Context, group api.ServerGroup, member api.MemberStatus) error // DeleteTLSCASecret removes the Secret containing the TLS CA certificate. - DeleteTLSCASecret() error + DeleteTLSCASecret(ctx context.Context) error // GetImageInfo returns the image info for an image with given name. // Returns: (info, infoFound) GetImageInfo(imageName string) (api.ImageInfo, bool) @@ -106,7 +107,7 @@ type ActionContext interface { GetCurrentImageInfo() (api.ImageInfo, bool) // SetCurrentImage changes the CurrentImage field in the deployment // status to the given image. - SetCurrentImage(imageInfo api.ImageInfo) error + SetCurrentImage(ctx context.Context, imageInfo api.ImageInfo) error // GetDeploymentHealth returns a copy of the latest known state of cluster health GetDeploymentHealth() (driver.ClusterHealth, error) // GetShardSyncStatus returns true if all shards are in sync @@ -118,16 +119,16 @@ type ActionContext interface { // GetStatus returns a copy of the status GetStatus() api.DeploymentStatus // DisableScalingCluster disables scaling DBservers and coordinators - DisableScalingCluster() error + DisableScalingCluster(ctx context.Context) error // EnableScalingCluster enables scaling DBservers and coordinators - EnableScalingCluster() error + EnableScalingCluster(ctx context.Context) error // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken - UpdateClusterCondition(conditionType api.ConditionType, status bool, reason, message string) error + UpdateClusterCondition(ctx context.Context, conditionType api.ConditionType, status bool, reason, message string) error SecretsInterface() k8sutil.SecretInterface // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken - WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error + WithStatusUpdate(ctx context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error // GetBackup receives information about a backup resource - GetBackup(backup string) (*backupApi.ArangoBackup, error) + GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) // GetName receives information about a deployment name GetName() string // GetNameget current cached state of deployment @@ -166,12 +167,12 @@ func (ac *actionContext) GetStatus() api.DeploymentStatus { return *s } -func (ac *actionContext) GetBackup(backup string) (*backupApi.ArangoBackup, error) { - return ac.context.GetBackup(backup) +func (ac *actionContext) GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) { + return ac.context.GetBackup(ctx, backup) } -func (ac *actionContext) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error { - return ac.context.WithStatusUpdate(action, force...) +func (ac *actionContext) WithStatusUpdate(ctx context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error { + return ac.context.WithStatusUpdate(ctx, action, force...) } func (ac *actionContext) SecretsInterface() k8sutil.SecretInterface { @@ -182,8 +183,8 @@ func (ac *actionContext) GetShardSyncStatus() bool { return ac.context.GetShardSyncStatus() } -func (ac *actionContext) UpdateClusterCondition(conditionType api.ConditionType, status bool, reason, message string) error { - return ac.context.WithStatusUpdate(func(s *api.DeploymentStatus) bool { +func (ac *actionContext) UpdateClusterCondition(ctx context.Context, conditionType api.ConditionType, status bool, reason, message string) error { + return ac.context.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { return s.Conditions.Update(conditionType, status, reason, message) }) } @@ -192,16 +193,16 @@ func (ac *actionContext) GetAPIObject() k8sutil.APIObject { return ac.context.GetAPIObject() } -func (ac *actionContext) UpdatePvc(pvc *v1.PersistentVolumeClaim) error { - return ac.context.UpdatePvc(pvc) +func (ac *actionContext) UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error { + return ac.context.UpdatePvc(ctx, pvc) } func (ac *actionContext) CreateEvent(evt *k8sutil.Event) { ac.context.CreateEvent(evt) } -func (ac *actionContext) GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) { - return ac.context.GetPvc(pvcName) +func (ac *actionContext) GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) { + return ac.context.GetPvc(ctx, pvcName) } // Gets the specified mode of deployment @@ -276,8 +277,8 @@ func (ac *actionContext) GetMemberStatusByID(id string) (api.MemberStatus, bool) // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. -func (ac *actionContext) CreateMember(group api.ServerGroup, id string) (string, error) { - result, err := ac.context.CreateMember(group, id) +func (ac *actionContext) CreateMember(ctx context.Context, group api.ServerGroup, id string) (string, error) { + result, err := ac.context.CreateMember(ctx, group, id) if err != nil { return "", errors.WithStack(err) } @@ -285,7 +286,7 @@ func (ac *actionContext) CreateMember(group api.ServerGroup, id string) (string, } // UpdateMember updates the deployment status wrt the given member. -func (ac *actionContext) UpdateMember(member api.MemberStatus) error { +func (ac *actionContext) UpdateMember(ctx context.Context, member api.MemberStatus) error { status, lastVersion := ac.context.GetStatus() _, group, found := status.Members.ElementByID(member.ID) if !found { @@ -294,7 +295,7 @@ func (ac *actionContext) UpdateMember(member api.MemberStatus) error { if err := status.Members.Update(member, group); err != nil { return errors.WithStack(err) } - if err := ac.context.UpdateStatus(status, lastVersion); err != nil { + if err := ac.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Debug().Err(err).Msg("Updating CR status failed") return errors.WithStack(err) } @@ -302,7 +303,7 @@ func (ac *actionContext) UpdateMember(member api.MemberStatus) error { } // RemoveMemberByID removes a member with given id. -func (ac *actionContext) RemoveMemberByID(id string) error { +func (ac *actionContext) RemoveMemberByID(ctx context.Context, id string) error { status, lastVersion := ac.context.GetStatus() _, group, found := status.Members.ElementByID(id) if !found { @@ -313,15 +314,15 @@ func (ac *actionContext) RemoveMemberByID(id string) error { return errors.WithStack(err) } // Save removed member - if err := ac.context.UpdateStatus(status, lastVersion); err != nil { + if err := ac.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } return nil } // GetPod returns pod. -func (ac *actionContext) GetPod(podName string) (*v1.Pod, error) { - if pod, err := ac.context.GetPod(podName); err != nil { +func (ac *actionContext) GetPod(ctx context.Context, podName string) (*v1.Pod, error) { + if pod, err := ac.context.GetPod(ctx, podName); err != nil { return nil, errors.WithStack(err) } else { return pod, nil @@ -330,8 +331,8 @@ func (ac *actionContext) GetPod(podName string) (*v1.Pod, error) { // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. -func (ac *actionContext) DeletePod(podName string) error { - if err := ac.context.DeletePod(podName); err != nil { +func (ac *actionContext) DeletePod(ctx context.Context, podName string) error { + if err := ac.context.DeletePod(ctx, podName); err != nil { return errors.WithStack(err) } return nil @@ -339,8 +340,8 @@ func (ac *actionContext) DeletePod(podName string) error { // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. -func (ac *actionContext) DeletePvc(pvcName string) error { - if err := ac.context.DeletePvc(pvcName); err != nil { +func (ac *actionContext) DeletePvc(ctx context.Context, pvcName string) error { + if err := ac.context.DeletePvc(ctx, pvcName); err != nil { return errors.WithStack(err) } return nil @@ -348,8 +349,8 @@ func (ac *actionContext) DeletePvc(pvcName string) error { // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. -func (ac *actionContext) RemovePodFinalizers(podName string) error { - if err := ac.context.RemovePodFinalizers(podName); err != nil { +func (ac *actionContext) RemovePodFinalizers(ctx context.Context, podName string) error { + if err := ac.context.RemovePodFinalizers(ctx, podName); err != nil { return errors.WithStack(err) } return nil @@ -357,15 +358,15 @@ func (ac *actionContext) RemovePodFinalizers(podName string) error { // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. -func (ac *actionContext) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error { - if err := ac.context.DeleteTLSKeyfile(group, member); err != nil { +func (ac *actionContext) DeleteTLSKeyfile(ctx context.Context, group api.ServerGroup, member api.MemberStatus) error { + if err := ac.context.DeleteTLSKeyfile(ctx, group, member); err != nil { return errors.WithStack(err) } return nil } // DeleteTLSCASecret removes the Secret containing the TLS CA certificate. -func (ac *actionContext) DeleteTLSCASecret() error { +func (ac *actionContext) DeleteTLSCASecret(ctx context.Context) error { spec := ac.context.GetSpec().TLS if !spec.IsSecure() { return nil @@ -378,7 +379,7 @@ func (ac *actionContext) DeleteTLSCASecret() error { status, lastVersion := ac.context.GetStatus() if status.SecretHashes != nil { status.SecretHashes.TLSCA = "" - if err := ac.context.UpdateStatus(status, lastVersion); err != nil { + if err := ac.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } } @@ -410,8 +411,8 @@ func (ac *actionContext) GetCurrentImageInfo() (api.ImageInfo, bool) { // SetCurrentImage changes the CurrentImage field in the deployment // status to the given image. -func (ac *actionContext) SetCurrentImage(imageInfo api.ImageInfo) error { - return ac.context.WithStatusUpdate(func(s *api.DeploymentStatus) bool { +func (ac *actionContext) SetCurrentImage(ctx context.Context, imageInfo api.ImageInfo) error { + return ac.context.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.CurrentImage == nil || s.CurrentImage.Image != imageInfo.Image { s.CurrentImage = &imageInfo return true @@ -426,11 +427,11 @@ func (ac *actionContext) InvalidateSyncStatus() { } // DisableScalingCluster disables scaling DBservers and coordinators -func (ac *actionContext) DisableScalingCluster() error { - return ac.context.DisableScalingCluster() +func (ac *actionContext) DisableScalingCluster(ctx context.Context) error { + return ac.context.DisableScalingCluster(ctx) } // EnableScalingCluster enables scaling DBservers and coordinators -func (ac *actionContext) EnableScalingCluster() error { - return ac.context.EnableScalingCluster() +func (ac *actionContext) EnableScalingCluster(ctx context.Context) error { + return ac.context.EnableScalingCluster(ctx) } diff --git a/pkg/deployment/reconcile/action_disable_scaling_cluster.go b/pkg/deployment/reconcile/action_disable_scaling_cluster.go index d2c8094a6..e7493cf70 100644 --- a/pkg/deployment/reconcile/action_disable_scaling_cluster.go +++ b/pkg/deployment/reconcile/action_disable_scaling_cluster.go @@ -54,7 +54,7 @@ type actionDisableScalingCluster struct { // Start disables scaling DBservers and coordinators func (a *actionDisableScalingCluster) Start(ctx context.Context) (bool, error) { - err := a.actionCtx.DisableScalingCluster() + err := a.actionCtx.DisableScalingCluster(ctx) if err != nil { return false, err } diff --git a/pkg/deployment/reconcile/action_enable_scaling_cluster.go b/pkg/deployment/reconcile/action_enable_scaling_cluster.go index 5f05ca763..178c4357d 100644 --- a/pkg/deployment/reconcile/action_enable_scaling_cluster.go +++ b/pkg/deployment/reconcile/action_enable_scaling_cluster.go @@ -54,7 +54,7 @@ type actionEnableScalingCluster struct { // Start enables scaling DBservers and coordinators func (a *actionEnableScalingCluster) Start(ctx context.Context) (bool, error) { - err := a.actionCtx.EnableScalingCluster() + err := a.actionCtx.EnableScalingCluster(ctx) if err != nil { return false, err } diff --git a/pkg/deployment/reconcile/action_encryption_add.go b/pkg/deployment/reconcile/action_encryption_add.go index c8df1c277..db051f952 100644 --- a/pkg/deployment/reconcile/action_encryption_add.go +++ b/pkg/deployment/reconcile/action_encryption_add.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,8 @@ import ( "context" "encoding/base64" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/arangodb/kube-arangodb/pkg/util/errors" @@ -85,7 +88,7 @@ func (a *encryptionKeyAddAction) Start(ctx context.Context) (bool, error) { secret = s } - sha, d, exists, err := pod.GetEncryptionKey(a.actionCtx.SecretsInterface(), secret) + sha, d, exists, err := pod.GetEncryptionKey(ctx, a.actionCtx.SecretsInterface(), secret) if err != nil { a.log.Error().Err(err).Msgf("Unable to fetch current encryption key") return true, nil @@ -104,7 +107,10 @@ func (a *encryptionKeyAddAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { return false, err } diff --git a/pkg/deployment/reconcile/action_encryption_propagated.go b/pkg/deployment/reconcile/action_encryption_propagated.go index 45aad362a..8a41c815a 100644 --- a/pkg/deployment/reconcile/action_encryption_propagated.go +++ b/pkg/deployment/reconcile/action_encryption_propagated.go @@ -56,7 +56,7 @@ func (a *encryptionKeyPropagatedAction) Start(ctx context.Context) (bool, error) propagatedFlagBool := propagatedFlag == conditionTrue - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.Hashes.Encryption.Propagated != propagatedFlagBool { s.Hashes.Encryption.Propagated = propagatedFlagBool return true diff --git a/pkg/deployment/reconcile/action_encryption_refresh.go b/pkg/deployment/reconcile/action_encryption_refresh.go index d829cf795..ae2fde2a6 100644 --- a/pkg/deployment/reconcile/action_encryption_refresh.go +++ b/pkg/deployment/reconcile/action_encryption_refresh.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,11 +26,14 @@ package reconcile import ( "context" + "github.com/rs/zerolog" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/client" "github.com/arangodb/kube-arangodb/pkg/deployment/pod" - "github.com/rs/zerolog" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) func init() { @@ -54,21 +58,26 @@ func (a *encryptionKeyRefreshAction) Start(ctx context.Context) (bool, error) { } func (a *encryptionKeyRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error) { - keyfolder, err := a.actionCtx.SecretsInterface().Get(ctx, pod.GetEncryptionFolderSecretName(a.actionCtx.GetName()), meta.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + keyfolder, err := a.actionCtx.SecretsInterface().Get(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetName()), meta.GetOptions{}) if err != nil { a.log.Err(err).Msgf("Unable to fetch encryption folder") return true, false, nil } - c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := a.actionCtx.GetServerClient(ctxChild, a.action.Group, a.action.MemberID) if err != nil { a.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil } client := client.NewClient(c.Connection()) - - e, err := client.RefreshEncryption(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + e, err := client.RefreshEncryption(ctxChild) if err != nil { a.log.Warn().Err(err).Msg("Unable to refresh encryption") return true, false, nil diff --git a/pkg/deployment/reconcile/action_encryption_remove.go b/pkg/deployment/reconcile/action_encryption_remove.go index e8eb0f1f6..7e80ae06c 100644 --- a/pkg/deployment/reconcile/action_encryption_remove.go +++ b/pkg/deployment/reconcile/action_encryption_remove.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -81,7 +82,10 @@ func (a *encryptionKeyRemoveAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) diff --git a/pkg/deployment/reconcile/action_encryption_status_update.go b/pkg/deployment/reconcile/action_encryption_status_update.go index 4782a1f4e..63ecdbf08 100644 --- a/pkg/deployment/reconcile/action_encryption_status_update.go +++ b/pkg/deployment/reconcile/action_encryption_status_update.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,6 +26,8 @@ package reconcile import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -57,7 +60,10 @@ func (a *encryptionKeyStatusUpdateAction) Start(ctx context.Context) (bool, erro return true, nil } - f, err := a.actionCtx.SecretsInterface().Get(ctx, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), meta.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + f, err := a.actionCtx.SecretsInterface().Get(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), meta.GetOptions{}) if err != nil { a.log.Error().Err(err).Msgf("Unable to get folder info") return true, nil @@ -65,7 +71,7 @@ func (a *encryptionKeyStatusUpdateAction) Start(ctx context.Context) (bool, erro keyHashes := secretKeysToListWithPrefix("sha256:", f) - if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err = a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if len(keyHashes) == 0 { if s.Hashes.Encryption.Keys != nil { s.Hashes.Encryption.Keys = nil diff --git a/pkg/deployment/reconcile/action_helper.go b/pkg/deployment/reconcile/action_helper.go index 5a4441980..8ce21d571 100644 --- a/pkg/deployment/reconcile/action_helper.go +++ b/pkg/deployment/reconcile/action_helper.go @@ -43,14 +43,14 @@ type actionEmptyCheckProgress struct { // CheckProgress define optional check progress for action // Returns: ready, abort, error. -func (e actionEmptyCheckProgress) CheckProgress(ctx context.Context) (bool, bool, error) { +func (e actionEmptyCheckProgress) CheckProgress(_ context.Context) (bool, bool, error) { return true, false, nil } type actionEmptyStart struct { } -func (e actionEmptyStart) Start(ctx context.Context) (bool, error) { +func (e actionEmptyStart) Start(_ context.Context) (bool, error) { return false, nil } diff --git a/pkg/deployment/reconcile/action_idle.go b/pkg/deployment/reconcile/action_idle.go index 9fa513841..c54d7ab08 100644 --- a/pkg/deployment/reconcile/action_idle.go +++ b/pkg/deployment/reconcile/action_idle.go @@ -55,6 +55,6 @@ type actionIdle struct { // Start performs the start of the action. // Returns true if the action is completely finished, false in case // the start time needs to be recorded and a ready condition needs to be checked. -func (a *actionIdle) Start(ctx context.Context) (bool, error) { +func (a *actionIdle) Start(_ context.Context) (bool, error) { return true, nil } diff --git a/pkg/deployment/reconcile/action_jwt_add.go b/pkg/deployment/reconcile/action_jwt_add.go index 5ecfe52d9..03859700b 100644 --- a/pkg/deployment/reconcile/action_jwt_add.go +++ b/pkg/deployment/reconcile/action_jwt_add.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -116,7 +117,10 @@ func (a *jwtAddAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) diff --git a/pkg/deployment/reconcile/action_jwt_clean.go b/pkg/deployment/reconcile/action_jwt_clean.go index 169c9d58e..cdc675316 100644 --- a/pkg/deployment/reconcile/action_jwt_clean.go +++ b/pkg/deployment/reconcile/action_jwt_clean.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -107,7 +108,10 @@ func (a *jwtCleanAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) diff --git a/pkg/deployment/reconcile/action_jwt_propagated.go b/pkg/deployment/reconcile/action_jwt_propagated.go index ffc56aa34..a39e1bdb1 100644 --- a/pkg/deployment/reconcile/action_jwt_propagated.go +++ b/pkg/deployment/reconcile/action_jwt_propagated.go @@ -62,7 +62,7 @@ func (a *jwtPropagatedAction) Start(ctx context.Context) (bool, error) { propagatedFlagBool := propagatedFlag == conditionTrue - if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err = a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.Hashes.JWT.Propagated != propagatedFlagBool { s.Hashes.JWT.Propagated = propagatedFlagBool return true diff --git a/pkg/deployment/reconcile/action_jwt_refresh.go b/pkg/deployment/reconcile/action_jwt_refresh.go index d69533273..4a9792dae 100644 --- a/pkg/deployment/reconcile/action_jwt_refresh.go +++ b/pkg/deployment/reconcile/action_jwt_refresh.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -28,6 +29,7 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/client" "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/rs/zerolog" ) @@ -58,12 +60,17 @@ func (a *jwtRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error return true, false, nil } - c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := a.actionCtx.GetServerClient(ctxChild, a.action.Group, a.action.MemberID) if err != nil { a.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil } - if invalid, err := isMemberJWTTokenInvalid(ctx, client.NewClient(c.Connection()), folder.Data, true); err != nil { + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if invalid, err := isMemberJWTTokenInvalid(ctxChild, client.NewClient(c.Connection()), folder.Data, true); err != nil { a.log.Warn().Err(err).Msg("Error while getting JWT Status") return true, false, nil } else if invalid { diff --git a/pkg/deployment/reconcile/action_jwt_set_active.go b/pkg/deployment/reconcile/action_jwt_set_active.go index 17245d98c..823ffdcc9 100644 --- a/pkg/deployment/reconcile/action_jwt_set_active.go +++ b/pkg/deployment/reconcile/action_jwt_set_active.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -118,7 +119,10 @@ func (a *jwtSetActiveAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) diff --git a/pkg/deployment/reconcile/action_jwt_status_update.go b/pkg/deployment/reconcile/action_jwt_status_update.go index 7cf948c0b..79e4793ea 100644 --- a/pkg/deployment/reconcile/action_jwt_status_update.go +++ b/pkg/deployment/reconcile/action_jwt_status_update.go @@ -106,7 +106,7 @@ func (a *jwtStatusUpdateAction) Start(ctx context.Context) (bool, error) { keySha := fmt.Sprintf("sha256:%s", util.SHA256(key)) - if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err = a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.Hashes.JWT.Passive != nil { s.Hashes.JWT.Passive = nil return true @@ -131,7 +131,7 @@ func (a *jwtStatusUpdateAction) Start(ctx context.Context) (bool, error) { return true, nil } - if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) (update bool) { + if err = a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) (update bool) { activeKeyData, active := f.Data[pod.ActiveJWTKey] activeKeyShort := util.SHA256(activeKeyData) activeKey := fmt.Sprintf("sha256:%s", activeKeyShort) diff --git a/pkg/deployment/reconcile/action_maintenance_disable.go b/pkg/deployment/reconcile/action_maintenance_disable.go index 4e7e53ce9..988fdde0d 100644 --- a/pkg/deployment/reconcile/action_maintenance_disable.go +++ b/pkg/deployment/reconcile/action_maintenance_disable.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -27,6 +28,7 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/agency" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/rs/zerolog" ) @@ -57,13 +59,18 @@ func (a *actionDisableMaintenance) Start(ctx context.Context) (bool, error) { return true, nil } - client, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + client, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { a.log.Error().Err(err).Msgf("Unable to get agency client") return true, nil } - if err := agency.SetMaintenanceMode(ctx, client, false); err != nil { + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return agency.SetMaintenanceMode(ctxChild, client, false) + }) + if err != nil { a.log.Error().Err(err).Msgf("Unable to disable maintenance") return true, nil } diff --git a/pkg/deployment/reconcile/action_maintenance_enable.go b/pkg/deployment/reconcile/action_maintenance_enable.go index ceb3d1b68..1110b074a 100644 --- a/pkg/deployment/reconcile/action_maintenance_enable.go +++ b/pkg/deployment/reconcile/action_maintenance_enable.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,6 +26,8 @@ package reconcile import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/agency" "github.com/rs/zerolog" @@ -57,13 +60,18 @@ func (a *actionEnableMaintenance) Start(ctx context.Context) (bool, error) { return true, nil } - client, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + client, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { a.log.Error().Err(err).Msgf("Unable to get agency client") return true, nil } - if err := agency.SetMaintenanceMode(ctx, client, true); err != nil { + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return agency.SetMaintenanceMode(ctxChild, client, true) + }) + if err != nil { a.log.Error().Err(err).Msgf("Unable to set maintenance") return true, nil } diff --git a/pkg/deployment/reconcile/action_mark_to_remove_member.go b/pkg/deployment/reconcile/action_mark_to_remove_member.go index f02a6808a..8bea24d1a 100644 --- a/pkg/deployment/reconcile/action_mark_to_remove_member.go +++ b/pkg/deployment/reconcile/action_mark_to_remove_member.go @@ -54,7 +54,7 @@ func (a *actionMarkToRemove) Start(ctx context.Context) (bool, error) { return true, nil } - return true, a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + return true, a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { member, group, ok := s.Members.ElementByID(a.action.MemberID) if !ok { return false diff --git a/pkg/deployment/reconcile/action_pvc_resize.go b/pkg/deployment/reconcile/action_pvc_resize.go index 4d547bfd6..89a34b868 100644 --- a/pkg/deployment/reconcile/action_pvc_resize.go +++ b/pkg/deployment/reconcile/action_pvc_resize.go @@ -71,7 +71,7 @@ func (a *actionPVCResize) Start(ctx context.Context) (bool, error) { return true, nil } - pvc, err := a.actionCtx.GetPvc(m.PersistentVolumeClaimName) + pvc, err := a.actionCtx.GetPvc(ctx, m.PersistentVolumeClaimName) if err != nil { if apiErrors.IsNotFound(err) { return true, nil @@ -92,14 +92,14 @@ func (a *actionPVCResize) Start(ctx context.Context) (bool, error) { cmp := volumeSize.Cmp(requestedSize) if cmp < 0 { pvc.Spec.Resources.Requests[core.ResourceStorage] = requestedSize - if err := a.actionCtx.UpdatePvc(pvc); err != nil { + if err := a.actionCtx.UpdatePvc(ctx, pvc); err != nil { return false, err } return false, nil } else if cmp > 0 { if groupSpec.GetVolumeAllowShrink() && group == api.ServerGroupDBServers { - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.Plan = append(s.Plan, api.NewAction(api.ActionTypeMarkToRemoveMember, group, m.ID)) return true }); err != nil { @@ -129,7 +129,7 @@ func (a *actionPVCResize) CheckProgress(ctx context.Context) (bool, bool, error) return true, false, nil } - pvc, err := a.actionCtx.GetPvc(m.PersistentVolumeClaimName) + pvc, err := a.actionCtx.GetPvc(ctx, m.PersistentVolumeClaimName) if err != nil { if apiErrors.IsNotFound(err) { return true, false, nil diff --git a/pkg/deployment/reconcile/action_pvc_resized.go b/pkg/deployment/reconcile/action_pvc_resized.go index 00a51ae2f..db901b375 100644 --- a/pkg/deployment/reconcile/action_pvc_resized.go +++ b/pkg/deployment/reconcile/action_pvc_resized.go @@ -67,7 +67,7 @@ func (a *actionPVCResized) CheckProgress(ctx context.Context) (bool, bool, error return true, false, nil } - pvc, err := a.actionCtx.GetPvc(m.PersistentVolumeClaimName) + pvc, err := a.actionCtx.GetPvc(ctx, m.PersistentVolumeClaimName) if err != nil { if apiErrors.IsNotFound(err) { return true, false, nil diff --git a/pkg/deployment/reconcile/action_recreate_member.go b/pkg/deployment/reconcile/action_recreate_member.go index 18deb38d3..a3268ff48 100644 --- a/pkg/deployment/reconcile/action_recreate_member.go +++ b/pkg/deployment/reconcile/action_recreate_member.go @@ -66,7 +66,7 @@ func (a *actionRecreateMember) Start(ctx context.Context) (bool, error) { return false, errors.Newf("expecting member to be present in list, but it is not") } - _, err := a.actionCtx.GetPvc(m.PersistentVolumeClaimName) + _, err := a.actionCtx.GetPvc(ctx, m.PersistentVolumeClaimName) if err != nil { if kubeErrors.IsNotFound(err) { return false, errors.Newf("PVC is missing %s. Members won't be recreated without old PV", m.PersistentVolumeClaimName) @@ -80,7 +80,7 @@ func (a *actionRecreateMember) Start(ctx context.Context) (bool, error) { m.Phase = api.MemberPhaseNone } - if err = a.actionCtx.UpdateMember(m); err != nil { + if err = a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } diff --git a/pkg/deployment/reconcile/action_remove_member.go b/pkg/deployment/reconcile/action_remove_member.go index 3734e3ce2..5959458f3 100644 --- a/pkg/deployment/reconcile/action_remove_member.go +++ b/pkg/deployment/reconcile/action_remove_member.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -68,11 +69,16 @@ func (a *actionRemoveMember) Start(ctx context.Context) (bool, error) { } // For safety, remove from cluster if a.action.Group == api.ServerGroupCoordinators || a.action.Group == api.ServerGroupDBServers { - client, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + client, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { return false, errors.WithStack(err) } - if err := arangod.RemoveServerFromCluster(ctx, client.Connection(), driver.ServerID(m.ID)); err != nil { + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if err := arangod.RemoveServerFromCluster(ctxChild, client.Connection(), driver.ServerID(m.ID)); err != nil { if !driver.IsNotFound(err) && !driver.IsPreconditionFailed(err) { a.log.Err(err).Str("member-id", m.ID).Msgf("Failed to remove server from cluster") // ignore this error, maybe all coordinators are failed and no connction to cluster is possible @@ -100,17 +106,17 @@ func (a *actionRemoveMember) Start(ctx context.Context) (bool, error) { } } // Remove the pod (if any) - if err := a.actionCtx.DeletePod(m.PodName); err != nil { + if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil { return false, errors.WithStack(err) } // Remove the pvc (if any) if m.PersistentVolumeClaimName != "" { - if err := a.actionCtx.DeletePvc(m.PersistentVolumeClaimName); err != nil { + if err := a.actionCtx.DeletePvc(ctx, m.PersistentVolumeClaimName); err != nil { return false, errors.WithStack(err) } } // Remove member - if err := a.actionCtx.RemoveMemberByID(a.action.MemberID); err != nil { + if err := a.actionCtx.RemoveMemberByID(ctx, a.action.MemberID); err != nil { return false, errors.WithStack(err) } // Check that member has been removed diff --git a/pkg/deployment/reconcile/action_resign_leadership.go b/pkg/deployment/reconcile/action_resign_leadership.go index 2f1744f89..0f372136d 100644 --- a/pkg/deployment/reconcile/action_resign_leadership.go +++ b/pkg/deployment/reconcile/action_resign_leadership.go @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -68,7 +69,9 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) { return true, nil } - client, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + client, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { log.Error().Err(err).Msgf("Unable to get client") return true, errors.WithStack(err) @@ -76,14 +79,18 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) { switch group { case api.ServerGroupDBServers: - cluster, err := client.Cluster(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + cluster, err := client.Cluster(ctxChild) if err != nil { log.Error().Err(err).Msgf("Unable to get cluster client") return true, errors.WithStack(err) } var jobID string - jobCtx := driver.WithJobIDResponse(ctx, &jobID) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + jobCtx := driver.WithJobIDResponse(ctxChild, &jobID) log.Debug().Msg("Temporary shutdown, resign leadership") if err := cluster.ResignServer(jobCtx, m.ID); err != nil { log.Debug().Err(err).Msg("Failed to resign server") @@ -92,7 +99,7 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) { m.CleanoutJobID = jobID - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return true, errors.WithStack(err) } @@ -112,19 +119,25 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool, return true, false, nil } - agency, err := a.actionCtx.GetAgency(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + agency, err := a.actionCtx.GetAgency(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return false, false, errors.WithStack(err) } - c, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := a.actionCtx.GetDatabaseClient(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return false, false, errors.WithStack(err) } - jobStatus, err := arangod.CleanoutServerJobStatus(ctx, m.CleanoutJobID, c, agency) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, m.CleanoutJobID, c, agency) if err != nil { if driver.IsNotFound(err) { log.Debug().Err(err).Msg("Job not found, but proceeding") @@ -136,7 +149,7 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool, if jobStatus.IsFailed() { m.CleanoutJobID = "" - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } log.Error().Msg("Resign server job failed") @@ -145,7 +158,7 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool, if jobStatus.IsFinished() { m.CleanoutJobID = "" - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } return true, false, nil diff --git a/pkg/deployment/reconcile/action_rotate_member.go b/pkg/deployment/reconcile/action_rotate_member.go index c9f51f3bc..6ed715ad7 100644 --- a/pkg/deployment/reconcile/action_rotate_member.go +++ b/pkg/deployment/reconcile/action_rotate_member.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -72,7 +73,7 @@ func (a *actionRotateMember) Start(ctx context.Context) (bool, error) { // Update status m.Phase = api.MemberPhaseRotating - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } return false, nil @@ -96,7 +97,7 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, bool, err } // Pod is terminated, we can now remove it - if err := a.actionCtx.DeletePod(m.PodName); err != nil { + if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil { if !k8sutil.IsNotFound(err) { return false, false, errors.WithStack(err) } @@ -105,7 +106,7 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, bool, err m.Phase = api.MemberPhaseNone m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations. m.CleanoutJobID = "" - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } return true, false, nil diff --git a/pkg/deployment/reconcile/action_rotate_start_member.go b/pkg/deployment/reconcile/action_rotate_start_member.go index 73e0188db..259d201db 100644 --- a/pkg/deployment/reconcile/action_rotate_start_member.go +++ b/pkg/deployment/reconcile/action_rotate_start_member.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -72,7 +73,7 @@ func (a *actionRotateStartMember) Start(ctx context.Context) (bool, error) { // Update status m.Phase = api.MemberPhaseRotateStart - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } return false, nil @@ -96,7 +97,7 @@ func (a *actionRotateStartMember) CheckProgress(ctx context.Context) (bool, bool } // Pod is terminated, we can now remove it - if err := a.actionCtx.DeletePod(m.PodName); err != nil { + if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil { if !k8sutil.IsNotFound(err) { return false, false, errors.WithStack(err) } diff --git a/pkg/deployment/reconcile/action_rotate_stop_member.go b/pkg/deployment/reconcile/action_rotate_stop_member.go index 3cc86c331..a155e1047 100644 --- a/pkg/deployment/reconcile/action_rotate_stop_member.go +++ b/pkg/deployment/reconcile/action_rotate_stop_member.go @@ -67,7 +67,7 @@ func (a *actionRotateStopMember) Start(ctx context.Context) (bool, error) { m.Phase = api.MemberPhaseNone m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations. m.CleanoutJobID = "" - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } return false, nil diff --git a/pkg/deployment/reconcile/action_set_current_image.go b/pkg/deployment/reconcile/action_set_current_image.go index 816612d1c..aed10d7f0 100644 --- a/pkg/deployment/reconcile/action_set_current_image.go +++ b/pkg/deployment/reconcile/action_set_current_image.go @@ -73,7 +73,7 @@ func (a *setCurrentMemberImageAction) CheckProgress(ctx context.Context) (bool, return true, false, nil } - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { m, g, found := s.Members.ElementByID(a.action.MemberID) if !found { log.Error().Msg("No such member") diff --git a/pkg/deployment/reconcile/action_shutdown_member.go b/pkg/deployment/reconcile/action_shutdown_member.go index 06ff2dde6..f3cf49321 100644 --- a/pkg/deployment/reconcile/action_shutdown_member.go +++ b/pkg/deployment/reconcile/action_shutdown_member.go @@ -70,7 +70,7 @@ func (a *actionShutdownMember) Start(ctx context.Context) (bool, error) { // Update status m.Phase = api.MemberPhaseShuttingDown - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } return false, nil diff --git a/pkg/deployment/reconcile/action_tls_ca_append.go b/pkg/deployment/reconcile/action_tls_ca_append.go index 7eab3151f..7e60e830f 100644 --- a/pkg/deployment/reconcile/action_tls_ca_append.go +++ b/pkg/deployment/reconcile/action_tls_ca_append.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -115,7 +116,10 @@ func (a *appendTLSCACertificateAction) Start(ctx context.Context) (bool, error) return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) diff --git a/pkg/deployment/reconcile/action_tls_ca_clean.go b/pkg/deployment/reconcile/action_tls_ca_clean.go index 5615bdd97..c5651c7c1 100644 --- a/pkg/deployment/reconcile/action_tls_ca_clean.go +++ b/pkg/deployment/reconcile/action_tls_ca_clean.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -117,7 +118,11 @@ func (a *cleanTLSCACertificateAction) Start(ctx context.Context) (bool, error) { } a.log.Info().Msgf("Removing key %s from truststore", certChecksum) - _, err = a.actionCtx.SecretsInterface().Patch(ctx, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) + + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) diff --git a/pkg/deployment/reconcile/action_tls_ca_renew.go b/pkg/deployment/reconcile/action_tls_ca_renew.go index 772a40c64..b7ebb2514 100644 --- a/pkg/deployment/reconcile/action_tls_ca_renew.go +++ b/pkg/deployment/reconcile/action_tls_ca_renew.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -54,8 +55,11 @@ func (a *renewTLSCACertificateAction) Start(ctx context.Context) (bool, error) { return true, nil } - s := a.actionCtx.SecretsInterface() - if err := s.Delete(ctx, a.actionCtx.GetSpec().TLS.GetCASecretName(), meta.DeleteOptions{}); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + s := a.actionCtx.SecretsInterface() + return s.Delete(ctxChild, a.actionCtx.GetSpec().TLS.GetCASecretName(), meta.DeleteOptions{}) + }) + if err != nil { if !k8sutil.IsNotFound(err) { a.log.Warn().Err(err).Msgf("Unable to clean cert %s", a.actionCtx.GetSpec().TLS.GetCASecretName()) return true, nil diff --git a/pkg/deployment/reconcile/action_tls_keyfile_clean.go b/pkg/deployment/reconcile/action_tls_keyfile_clean.go index 54f174164..00f4da853 100644 --- a/pkg/deployment/reconcile/action_tls_keyfile_clean.go +++ b/pkg/deployment/reconcile/action_tls_keyfile_clean.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -60,7 +61,7 @@ func (a *cleanTLSKeyfileCertificateAction) Start(ctx context.Context) (bool, err return true, nil } - if err := a.actionCtx.DeleteTLSKeyfile(a.action.Group, member); err != nil { + if err := a.actionCtx.DeleteTLSKeyfile(ctx, a.action.Group, member); err != nil { a.log.Warn().Err(err).Msgf("Unable to remove keyfile") if !k8sutil.IsNotFound(err) { return false, err diff --git a/pkg/deployment/reconcile/action_tls_keyfile_refresh.go b/pkg/deployment/reconcile/action_tls_keyfile_refresh.go index a8c0f38af..d15eec61c 100644 --- a/pkg/deployment/reconcile/action_tls_keyfile_refresh.go +++ b/pkg/deployment/reconcile/action_tls_keyfile_refresh.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -27,6 +28,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/deployment/client" "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" @@ -52,7 +54,9 @@ type refreshTLSKeyfileCertificateAction struct { } func (a *refreshTLSKeyfileCertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) { - c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := a.actionCtx.GetServerClient(ctxChild, a.action.Group, a.action.MemberID) if err != nil { a.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil @@ -74,7 +78,9 @@ func (a *refreshTLSKeyfileCertificateAction) CheckProgress(ctx context.Context) client := client.NewClient(c.Connection()) - e, err := client.RefreshTLS(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + e, err := client.RefreshTLS(ctxChild) if err != nil { a.log.Warn().Err(err).Msg("Unable to refresh TLS") return true, false, nil diff --git a/pkg/deployment/reconcile/action_tls_propagated.go b/pkg/deployment/reconcile/action_tls_propagated.go index d2f2fa7a4..0add6b289 100644 --- a/pkg/deployment/reconcile/action_tls_propagated.go +++ b/pkg/deployment/reconcile/action_tls_propagated.go @@ -56,7 +56,7 @@ func (a *tlsPropagatedAction) Start(ctx context.Context) (bool, error) { propagatedFlagBool := propagatedFlag == conditionTrue - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.Hashes.TLS.Propagated != propagatedFlagBool { s.Hashes.TLS.Propagated = propagatedFlagBool return true diff --git a/pkg/deployment/reconcile/action_tls_sni_update.go b/pkg/deployment/reconcile/action_tls_sni_update.go index 8b61b3e0f..f5ff7dc71 100644 --- a/pkg/deployment/reconcile/action_tls_sni_update.go +++ b/pkg/deployment/reconcile/action_tls_sni_update.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,7 @@ import ( "context" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/rs/zerolog" ) @@ -68,13 +70,17 @@ func (t *tlsSNIUpdate) CheckProgress(ctx context.Context) (bool, bool, error) { return true, false, nil } - c, err := t.actionCtx.GetServerClient(ctx, t.action.Group, t.action.MemberID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := t.actionCtx.GetServerClient(ctxChild, t.action.Group, t.action.MemberID) if err != nil { t.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil } - if ok, err := compareTLSSNIConfig(ctx, c.Connection(), fetchedSecrets, true); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if ok, err := compareTLSSNIConfig(ctxChild, c.Connection(), fetchedSecrets, true); err != nil { t.log.Warn().Err(err).Msg("Unable to compare TLS config") return true, false, nil } else { diff --git a/pkg/deployment/reconcile/action_tls_status_update.go b/pkg/deployment/reconcile/action_tls_status_update.go index c5a604d1b..22bae8d17 100644 --- a/pkg/deployment/reconcile/action_tls_status_update.go +++ b/pkg/deployment/reconcile/action_tls_status_update.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -27,6 +28,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/deployment/resources" "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" meta "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -56,7 +58,9 @@ func (a *tlsKeyStatusUpdateAction) Start(ctx context.Context) (bool, error) { return true, nil } - f, err := a.actionCtx.SecretsInterface().Get(ctx, resources.GetCASecretName(a.actionCtx.GetAPIObject()), meta.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + f, err := a.actionCtx.SecretsInterface().Get(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), meta.GetOptions{}) if err != nil { a.log.Error().Err(err).Msgf("Unable to get folder info") return true, nil @@ -64,7 +68,7 @@ func (a *tlsKeyStatusUpdateAction) Start(ctx context.Context) (bool, error) { keyHashes := secretKeysToListWithPrefix("sha256:", f) - if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err = a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { r := false if len(keyHashes) == 1 { if s.Hashes.TLS.CA == nil || *s.Hashes.TLS.CA != keyHashes[0] { diff --git a/pkg/deployment/reconcile/action_upgrade_current_image.go b/pkg/deployment/reconcile/action_upgrade_current_image.go index 8cfa138ee..bfcb0a5c8 100644 --- a/pkg/deployment/reconcile/action_upgrade_current_image.go +++ b/pkg/deployment/reconcile/action_upgrade_current_image.go @@ -71,7 +71,7 @@ func (a *setCurrentImageAction) CheckProgress(ctx context.Context) (bool, bool, if !found { return false, false, nil } - if err := a.actionCtx.SetCurrentImage(imageInfo); err != nil { + if err := a.actionCtx.SetCurrentImage(ctx, imageInfo); err != nil { return false, false, errors.WithStack(err) } log.Info().Str("image", a.action.Image).Str("to", imageInfo.Image).Msg("Changed current main image") diff --git a/pkg/deployment/reconcile/action_upgrade_member.go b/pkg/deployment/reconcile/action_upgrade_member.go index d70807255..5a0cd679e 100644 --- a/pkg/deployment/reconcile/action_upgrade_member.go +++ b/pkg/deployment/reconcile/action_upgrade_member.go @@ -62,7 +62,7 @@ func (a *actionUpgradeMember) Start(ctx context.Context) (bool, error) { } // Set AutoUpgrade condition m.Conditions.Update(api.ConditionTypeAutoUpgrade, true, "Upgrading", "AutoUpgrade on first restart") - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } @@ -112,7 +112,7 @@ func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, bool, er m.Image = m.OldImage.DeepCopy() } - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, true, nil } @@ -141,7 +141,7 @@ func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, bool, er if !m.OldImage.Equal(m.Image) && isUpgrading { m.OldImage = m.Image.DeepCopy() } - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } return isUpgrading, false, nil diff --git a/pkg/deployment/reconcile/action_wait_for_member_up.go b/pkg/deployment/reconcile/action_wait_for_member_up.go index 6f4d2b069..4f78b1676 100644 --- a/pkg/deployment/reconcile/action_wait_for_member_up.go +++ b/pkg/deployment/reconcile/action_wait_for_member_up.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,7 @@ import ( "context" "time" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/errors" driver "github.com/arangodb/go-driver" @@ -75,22 +77,26 @@ func (a *actionWaitForMemberUp) CheckProgress(ctx context.Context) (bool, bool, a.log.Debug().Msg("Member in failed phase") return true, false, nil } + + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if a.action.Group.IsArangosync() { - return a.checkProgressArangoSync(ctx) + return a.checkProgressArangoSync(ctxChild) } switch a.actionCtx.GetMode() { case api.DeploymentModeSingle: - return a.checkProgressSingle(ctx) + return a.checkProgressSingle(ctxChild) case api.DeploymentModeActiveFailover: if a.action.Group == api.ServerGroupAgents { - return a.checkProgressAgent(ctx) + return a.checkProgressAgent(ctxChild) } - return a.checkProgressSingleInActiveFailover(ctx) + return a.checkProgressSingleInActiveFailover(ctxChild) default: if a.action.Group == api.ServerGroupAgents { - return a.checkProgressAgent(ctx) + return a.checkProgressAgent(ctxChild) } - return a.checkProgressCluster(ctx) + return a.checkProgressCluster(ctxChild) } } @@ -98,6 +104,7 @@ func (a *actionWaitForMemberUp) CheckProgress(ctx context.Context) (bool, bool, // of a single server. func (a *actionWaitForMemberUp) checkProgressSingle(ctx context.Context) (bool, bool, error) { log := a.log + c, err := a.actionCtx.GetDatabaseClient(ctx) if err != nil { log.Debug().Err(err).Msg("Failed to create database client") diff --git a/pkg/deployment/reconcile/context.go b/pkg/deployment/reconcile/context.go index 333c0c71e..d00041227 100644 --- a/pkg/deployment/reconcile/context.go +++ b/pkg/deployment/reconcile/context.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -25,18 +26,16 @@ package reconcile import ( "context" - inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" - - "github.com/arangodb/kube-arangodb/pkg/util/arangod/conn" - - backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" - "github.com/arangodb/arangosync-client/client" driver "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/agency" + v1 "k8s.io/api/core/v1" + + backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/arangod/conn" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" - v1 "k8s.io/api/core/v1" + inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" ) // Context provides methods to the reconcile package. @@ -49,9 +48,9 @@ type Context interface { GetStatus() (api.DeploymentStatus, int32) // UpdateStatus replaces the status of the deployment with the given status and // updates the resources in k8s. - UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error + UpdateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error // UpdateMember updates the deployment status wrt the given member. - UpdateMember(member api.MemberStatus) error + UpdateMember(ctx context.Context, member api.MemberStatus) error // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), // creating one if needed. GetDatabaseClient(ctx context.Context) (driver.Client, error) @@ -70,29 +69,29 @@ type Context interface { // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. // Returns ID, error - CreateMember(group api.ServerGroup, id string) (string, error) + CreateMember(ctx context.Context, group api.ServerGroup, id string) (string, error) // GetPod returns pod. - GetPod(podName string) (*v1.Pod, error) + GetPod(ctx context.Context, podName string) (*v1.Pod, error) // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - DeletePod(podName string) error + DeletePod(ctx context.Context, podName string) error // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. - DeletePvc(pvcName string) error + DeletePvc(ctx context.Context, pvcName string) error // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - RemovePodFinalizers(podName string) error + RemovePodFinalizers(ctx context.Context, podName string) error // UpdatePvc update PVC with given name in the namespace // of the deployment. - UpdatePvc(pvc *v1.PersistentVolumeClaim) error + UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error // GetPvc gets a PVC by the given name, in the samespace of the deployment. - GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) + GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) // GetTLSKeyfile returns the keyfile encoded TLS certificate+key for // the given member. GetTLSKeyfile(group api.ServerGroup, member api.MemberStatus) (string, error) // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. - DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error + DeleteTLSKeyfile(ctx context.Context, group api.ServerGroup, member api.MemberStatus) error // DeleteSecret removes the Secret with given name. // If the secret does not exist, the error is ignored. DeleteSecret(secretName string) error @@ -103,21 +102,21 @@ type Context interface { // InvalidateSyncStatus resets the sync state to false and triggers an inspection InvalidateSyncStatus() // DisableScalingCluster disables scaling DBservers and coordinators - DisableScalingCluster() error + DisableScalingCluster(ctx context.Context) error // EnableScalingCluster enables scaling DBservers and coordinators - EnableScalingCluster() error + EnableScalingCluster(ctx context.Context) error // GetAgencyData object for key path GetAgencyData(ctx context.Context, i interface{}, keyParts ...string) error // Renders Pod definition for member - RenderPodForMember(cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) + RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) // SelectImage select currently used image by pod SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken - WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error + WithStatusUpdate(ctx context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error // SecretsInterface return secret interface SecretsInterface() k8sutil.SecretInterface // GetBackup receives information about a backup resource - GetBackup(backup string) (*backupApi.ArangoBackup, error) + GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) // GetName receives deployment name GetName() string // GetAuthentication return authentication for members diff --git a/pkg/deployment/reconcile/helper_shutdown.go b/pkg/deployment/reconcile/helper_shutdown.go index 1ce2951f0..5557efebd 100644 --- a/pkg/deployment/reconcile/helper_shutdown.go +++ b/pkg/deployment/reconcile/helper_shutdown.go @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,7 @@ import ( "context" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "github.com/rs/zerolog" @@ -60,23 +62,25 @@ func (s shutdownHelperAPI) Start(ctx context.Context) (bool, error) { return true, nil } // Remove finalizers, so Kubernetes will quickly terminate the pod - if err := s.actionCtx.RemovePodFinalizers(m.PodName); err != nil { + if err := s.actionCtx.RemovePodFinalizers(ctx, m.PodName); err != nil { return false, errors.WithStack(err) } if group.IsArangod() { // Invoke shutdown endpoint - c, err := s.actionCtx.GetServerClient(ctx, group, s.action.MemberID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := s.actionCtx.GetServerClient(ctxChild, group, s.action.MemberID) if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return false, errors.WithStack(err) } removeFromCluster := false log.Debug().Bool("removeFromCluster", removeFromCluster).Msg("Shutting down member") - ctx, cancel := context.WithTimeout(ctx, shutdownTimeout) + ctxChild, cancel = context.WithTimeout(ctx, shutdownTimeout) defer cancel() - if err := c.Shutdown(ctx, removeFromCluster); err != nil { + if err := c.Shutdown(ctxChild, removeFromCluster); err != nil { // Shutdown failed. Let's check if we're already done - if ready, _, err := s.CheckProgress(ctx); err == nil && ready { + if ready, _, err := s.CheckProgress(ctxChild); err == nil && ready { // We're done return true, nil } @@ -85,7 +89,7 @@ func (s shutdownHelperAPI) Start(ctx context.Context) (bool, error) { } } else if group.IsArangosync() { // Terminate pod - if err := s.actionCtx.DeletePod(m.PodName); err != nil { + if err := s.actionCtx.DeletePod(ctx, m.PodName); err != nil { return false, errors.WithStack(err) } } @@ -127,7 +131,7 @@ func (s shutdownHelperDelete) Start(ctx context.Context) (bool, error) { } // Terminate pod - if err := s.actionCtx.DeletePod(m.PodName); err != nil { + if err := s.actionCtx.DeletePod(ctx, m.PodName); err != nil { if !k8sutil.IsNotFound(err) { return false, errors.WithStack(err) } @@ -153,7 +157,7 @@ func (s shutdownHelperDelete) CheckProgress(ctx context.Context) (bool, bool, er } if m.PodName != "" { - if _, err := s.actionCtx.GetPod(m.PodName); err == nil { + if _, err := s.actionCtx.GetPod(ctx, m.PodName); err == nil { log.Warn().Msgf("Pod still exists") return false, false, nil } else if !k8sutil.IsNotFound(err) { diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index 785f12f30..833bbf0ff 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -85,7 +85,7 @@ func (d *Reconciler) CreatePlan(ctx context.Context, cachedStatus inspectorInter status.Plan = newPlan - if err := d.context.UpdateStatus(status, lastVersion); err != nil { + if err := d.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err), false } return nil, true diff --git a/pkg/deployment/reconcile/plan_builder_cluster.go b/pkg/deployment/reconcile/plan_builder_cluster.go index 44dfffdb0..5acb825c0 100644 --- a/pkg/deployment/reconcile/plan_builder_cluster.go +++ b/pkg/deployment/reconcile/plan_builder_cluster.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -29,6 +30,7 @@ import ( "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" "github.com/rs/zerolog" @@ -39,24 +41,30 @@ const coordinatorHealthFailedTimeout time.Duration = time.Minute func createClusterOperationPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) api.Plan { if spec.GetMode() != api.DeploymentModeCluster { return nil } - c, err := context.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := planCtx.GetDatabaseClient(ctxChild) if err != nil { return nil } - cluster, err := c.Cluster(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + cluster, err := c.Cluster(ctxChild) if err != nil { log.Warn().Err(err).Msgf("Unable to get Cluster client") return nil } - health, err := cluster.Health(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + health, err := cluster.Health(ctxChild) if err != nil { log.Warn().Err(err).Msgf("Unable to get Cluster health") return nil diff --git a/pkg/deployment/reconcile/plan_builder_common.go b/pkg/deployment/reconcile/plan_builder_common.go index da83465f9..0aede0445 100644 --- a/pkg/deployment/reconcile/plan_builder_common.go +++ b/pkg/deployment/reconcile/plan_builder_common.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,6 +26,8 @@ package reconcile import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/agency" "github.com/arangodb/kube-arangodb/pkg/deployment/features" @@ -36,7 +39,7 @@ import ( func createMaintenanceManagementPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) api.Plan { if spec.Mode.Get() == api.DeploymentModeSingle { return nil } @@ -46,13 +49,17 @@ func createMaintenanceManagementPlan(ctx context.Context, return nil } - client, err := context.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + client, err := planCtx.GetDatabaseClient(ctxChild) if err != nil { log.Error().Err(err).Msgf("Unable to get agency client") return nil } - m, err := agency.GetMaintenanceMode(ctx, client) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + m, err := agency.GetMaintenanceMode(ctxChild, client) if err != nil { log.Error().Err(err).Msgf("Unable to get agency maintenance mode") return nil diff --git a/pkg/deployment/reconcile/plan_builder_context.go b/pkg/deployment/reconcile/plan_builder_context.go index fc13cc108..2cb8550fe 100644 --- a/pkg/deployment/reconcile/plan_builder_context.go +++ b/pkg/deployment/reconcile/plan_builder_context.go @@ -48,7 +48,7 @@ type PlanBuilderContext interface { // On error, the error is logged. CreateEvent(evt *k8sutil.Event) // GetPvc gets a PVC by the given name, in the samespace of the deployment. - GetPvc(pvcName string) (*core.PersistentVolumeClaim, error) + GetPvc(ctx context.Context, pvcName string) (*core.PersistentVolumeClaim, error) // GetShardSyncStatus returns true if all shards are in sync GetShardSyncStatus() bool // InvalidateSyncStatus resets the sync state to false and triggers an inspection @@ -60,7 +60,7 @@ type PlanBuilderContext interface { // GetAgencyData object for key path GetAgencyData(ctx context.Context, i interface{}, keyParts ...string) error // Renders Pod definition for member - RenderPodForMember(cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) + RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) // SelectImage select currently used image by pod SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), @@ -73,7 +73,7 @@ type PlanBuilderContext interface { // SecretsInterface return secret interface SecretsInterface() k8sutil.SecretInterface // GetBackup receives information about a backup resource - GetBackup(backup string) (*backupApi.ArangoBackup, error) + GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) // GetName receives deployment name GetName() string // GetAgency returns a connection to the entire agency. diff --git a/pkg/deployment/reconcile/plan_builder_encryption.go b/pkg/deployment/reconcile/plan_builder_encryption.go index 00bffb0c8..37229190c 100644 --- a/pkg/deployment/reconcile/plan_builder_encryption.go +++ b/pkg/deployment/reconcile/plan_builder_encryption.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,7 @@ import ( "context" "github.com/arangodb/kube-arangodb/pkg/deployment/features" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" core "k8s.io/api/core/v1" @@ -269,7 +271,7 @@ func areEncryptionKeysUpToDate(ctx context.Context, func isEncryptionKeyUpToDate(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext, + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext, group api.ServerGroup, m api.MemberStatus, folder *core.Secret) (updateRequired bool, failed bool) { if m.Phase != api.MemberPhaseCreated { @@ -282,7 +284,9 @@ func isEncryptionKeyUpToDate(ctx context.Context, mlog := log.With().Str("group", group.AsRole()).Str("member", m.ID).Logger() - c, err := context.GetServerClient(ctx, group, m.ID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := planCtx.GetServerClient(ctxChild, group, m.ID) if err != nil { mlog.Warn().Err(err).Msg("Unable to get client") return false, true @@ -290,7 +294,9 @@ func isEncryptionKeyUpToDate(ctx context.Context, client := client.NewClient(c.Connection()) - e, err := client.GetEncryption(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + e, err := client.GetEncryption(ctxChild) if err != nil { mlog.Error().Err(err).Msgf("Unable to fetch encryption keys") return false, true diff --git a/pkg/deployment/reconcile/plan_builder_restore.go b/pkg/deployment/reconcile/plan_builder_restore.go index bce1d0c60..2468bd8af 100644 --- a/pkg/deployment/reconcile/plan_builder_restore.go +++ b/pkg/deployment/reconcile/plan_builder_restore.go @@ -49,7 +49,7 @@ func createRestorePlan(ctx context.Context, } if spec.RestoreFrom != nil && status.Restore == nil { - backup, err := context.GetBackup(spec.GetRestoreFrom()) + backup, err := context.GetBackup(ctx, spec.GetRestoreFrom()) if err != nil { log.Warn().Err(err).Msg("Backup not found") return nil @@ -111,7 +111,7 @@ func createRestorePlanEncryption(ctx context.Context, log zerolog.Logger, spec a secret := *spec.RestoreEncryptionSecret // Additional logic to do restore with encryption key - name, _, exists, err := pod.GetEncryptionKey(builderCtx.SecretsInterface(), secret) + name, _, exists, err := pod.GetEncryptionKey(ctx, builderCtx.SecretsInterface(), secret) if err != nil { log.Err(err).Msgf("Unable to fetch encryption key") return false, nil diff --git a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go index 7c49765e5..af4b79d33 100644 --- a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go +++ b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go @@ -58,7 +58,7 @@ func createRotateOrUpgradePlan(ctx context.Context, cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { var plan api.Plan - newPlan, idle := createRotateOrUpgradePlanInternal(log, apiObject, spec, status, cachedStatus, context) + newPlan, idle := createRotateOrUpgradePlanInternal(ctx, log, apiObject, spec, status, cachedStatus, context) if idle { plan = append(plan, api.NewAction(api.ActionTypeIdle, api.ServerGroupUnknown, "")) @@ -68,7 +68,7 @@ func createRotateOrUpgradePlan(ctx context.Context, return plan } -func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, +func createRotateOrUpgradePlanInternal(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) (api.Plan, bool) { var newPlan api.Plan @@ -116,7 +116,7 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API !decision.AutoUpgradeNeeded) } else { // Use new level of rotate logic - rotNeeded, reason := podNeedsRotation(log, pod, apiObject, spec, group, status, m, cachedStatus, context) + rotNeeded, reason := podNeedsRotation(ctx, log, pod, apiObject, spec, group, status, m, cachedStatus, context) if rotNeeded { newPlan = createRotateMemberPlan(log, m, group, reason) } @@ -283,9 +283,9 @@ func memberImageInfo(spec api.DeploymentSpec, status api.MemberStatus, images ap // given pod differs from what it should be according to the // given deployment spec. // When true is returned, a reason for the rotation is already returned. -func podNeedsRotation(log zerolog.Logger, p *core.Pod, apiObject metav1.Object, spec api.DeploymentSpec, +func podNeedsRotation(ctx context.Context, log zerolog.Logger, p *core.Pod, apiObject metav1.Object, spec api.DeploymentSpec, group api.ServerGroup, status api.DeploymentStatus, m api.MemberStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) (bool, string) { + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) (bool, string) { if m.PodUID != p.UID { return true, "Pod UID does not match, this pod is not managed by Operator. Recreating" } @@ -294,7 +294,7 @@ func podNeedsRotation(log zerolog.Logger, p *core.Pod, apiObject metav1.Object, return true, "Pod Spec Version is nil - recreating pod" } - imageInfo, imageFound := context.SelectImage(spec, status) + imageInfo, imageFound := planCtx.SelectImage(spec, status) if !imageFound { // Image is not found, so rotation is not needed return false, "" @@ -306,7 +306,7 @@ func podNeedsRotation(log zerolog.Logger, p *core.Pod, apiObject metav1.Object, groupSpec := spec.GetServerGroupSpec(group) - renderedPod, err := context.RenderPodForMember(cachedStatus, spec, status, m.ID, imageInfo) + renderedPod, err := planCtx.RenderPodForMember(ctx, cachedStatus, spec, status, m.ID, imageInfo) if err != nil { log.Err(err).Msg("Error while rendering pod") return false, "" diff --git a/pkg/deployment/reconcile/plan_builder_test.go b/pkg/deployment/reconcile/plan_builder_test.go index 1bfe9acdb..cd0dd0489 100644 --- a/pkg/deployment/reconcile/plan_builder_test.go +++ b/pkg/deployment/reconcile/plan_builder_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -70,7 +71,7 @@ type testContext struct { RecordedEvent *k8sutil.Event } -func (c *testContext) GetPod(podName string) (*core.Pod, error) { +func (c *testContext) GetPod(_ context.Context, podName string) (*core.Pod, error) { if c.ErrPods != nil { return nil, c.ErrPods } @@ -90,7 +91,7 @@ func (c *testContext) GetAuthentication() conn.Auth { } } -func (c *testContext) RenderPodForMember(cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { +func (c *testContext) RenderPodForMember(_ context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { panic("implement me") } @@ -98,7 +99,7 @@ func (c *testContext) GetName() string { panic("implement me") } -func (c *testContext) GetBackup(backup string) (*backupApi.ArangoBackup, error) { +func (c *testContext) GetBackup(_ context.Context, backup string) (*backupApi.ArangoBackup, error) { panic("implement me") } @@ -106,7 +107,7 @@ func (c *testContext) SecretsInterface() k8sutil.SecretInterface { panic("implement me") } -func (c *testContext) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error { +func (c *testContext) WithStatusUpdate(_ context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error { panic("implement me") } @@ -114,7 +115,7 @@ func (c *testContext) SelectImage(spec api.DeploymentSpec, status api.Deployment panic("implement me") } -func (c *testContext) UpdatePvc(pvc *core.PersistentVolumeClaim) error { +func (c *testContext) UpdatePvc(_ context.Context, pvc *core.PersistentVolumeClaim) error { panic("implement me") } @@ -137,12 +138,12 @@ func (c *testContext) GetSpec() api.DeploymentSpec { return c.ArangoDeployment.Spec } -func (c *testContext) UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error { +func (c *testContext) UpdateStatus(_ context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error { c.ArangoDeployment.Status = status return nil } -func (c *testContext) UpdateMember(member api.MemberStatus) error { +func (c *testContext) UpdateMember(_ context.Context, member api.MemberStatus) error { panic("implement me") } @@ -166,23 +167,23 @@ func (c *testContext) GetSyncServerClient(ctx context.Context, group api.ServerG panic("implement me") } -func (c *testContext) CreateMember(group api.ServerGroup, id string) (string, error) { +func (c *testContext) CreateMember(_ context.Context, group api.ServerGroup, id string) (string, error) { panic("implement me") } -func (c *testContext) DeletePod(podName string) error { +func (c *testContext) DeletePod(_ context.Context, podName string) error { panic("implement me") } -func (c *testContext) DeletePvc(pvcName string) error { +func (c *testContext) DeletePvc(_ context.Context, pvcName string) error { panic("implement me") } -func (c *testContext) RemovePodFinalizers(podName string) error { +func (c *testContext) RemovePodFinalizers(_ context.Context, podName string) error { panic("implement me") } -func (c *testContext) GetOwnedPods() ([]core.Pod, error) { +func (c *testContext) GetOwnedPods(_ context.Context) ([]core.Pod, error) { if c.ErrPods != nil { return nil, c.ErrPods } @@ -193,7 +194,7 @@ func (c *testContext) GetOwnedPods() ([]core.Pod, error) { return c.Pods, c.ErrPods } -func (c *testContext) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error { +func (c *testContext) DeleteTLSKeyfile(_ context.Context, group api.ServerGroup, member api.MemberStatus) error { panic("implement me") } @@ -205,11 +206,11 @@ func (c *testContext) GetDeploymentHealth() (driver.ClusterHealth, error) { panic("implement me") } -func (c *testContext) DisableScalingCluster() error { +func (c *testContext) DisableScalingCluster(_ context.Context) error { panic("implement me") } -func (c *testContext) EnableScalingCluster() error { +func (c *testContext) EnableScalingCluster(_ context.Context) error { panic("implement me") } @@ -232,7 +233,7 @@ func (c *testContext) CreateEvent(evt *k8sutil.Event) { } // GetPvc gets a PVC by the given name, in the samespace of the deployment. -func (c *testContext) GetPvc(pvcName string) (*core.PersistentVolumeClaim, error) { +func (c *testContext) GetPvc(_ context.Context, pvcName string) (*core.PersistentVolumeClaim, error) { return c.PVC, c.PVCErr } diff --git a/pkg/deployment/reconcile/plan_builder_tls_sni.go b/pkg/deployment/reconcile/plan_builder_tls_sni.go index 4aaff3f59..cd59250c4 100644 --- a/pkg/deployment/reconcile/plan_builder_tls_sni.go +++ b/pkg/deployment/reconcile/plan_builder_tls_sni.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,7 +26,10 @@ package reconcile import ( "context" + "github.com/arangodb/go-driver" + "github.com/arangodb/kube-arangodb/pkg/deployment/features" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" @@ -39,7 +43,7 @@ import ( func createRotateTLSServerSNIPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) api.Plan { if !spec.TLS.IsSecure() { return nil } @@ -80,13 +84,24 @@ func createRotateTLSServerSNIPlan(ctx context.Context, continue } - c, err := context.GetServerClient(ctx, group, m.ID) + var c driver.Client + err := arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + c, err = planCtx.GetServerClient(ctxChild, group, m.ID) + return err + }) if err != nil { log.Warn().Err(err).Msg("Unable to get client") continue } - if ok, err := compareTLSSNIConfig(ctx, c.Connection(), fetchedSecrets, false); err != nil { + var ok bool + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + ok, err = compareTLSSNIConfig(ctxChild, c.Connection(), fetchedSecrets, false) + return err + }) + if err != nil { log.Warn().Err(err).Msg("SNI compare failed") return nil diff --git a/pkg/deployment/reconcile/plan_executor.go b/pkg/deployment/reconcile/plan_executor.go index 5d4d33e58..4b175f989 100644 --- a/pkg/deployment/reconcile/plan_executor.go +++ b/pkg/deployment/reconcile/plan_executor.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -70,7 +71,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspectorInte log := logContext.Logger() - action := d.createAction(ctx, log, planAction, cachedStatus) + action := d.createAction(log, planAction, cachedStatus) if planAction.StartTime.IsZero() { // Not started yet ready, err := action.Start(ctx) @@ -95,7 +96,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspectorInte status.Plan[0].StartTime = &now } // Save plan update - if err := d.context.UpdateStatus(status, lastVersion, true); err != nil { + if err := d.context.UpdateStatus(ctx, status, lastVersion, true); err != nil { log.Debug().Err(err).Msg("Failed to update CR status") return false, errors.WithStack(err) } @@ -120,7 +121,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspectorInte status.Plan[0].MemberID = action.MemberID() } // Save plan update - if err := d.context.UpdateStatus(status, lastVersion); err != nil { + if err := d.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Debug().Err(err).Msg("Failed to update CR status") return false, errors.WithStack(err) } @@ -149,7 +150,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspectorInte // Replace plan with empty one and save it. status, lastVersion := d.context.GetStatus() status.Plan = api.Plan{} - if err := d.context.UpdateStatus(status, lastVersion); err != nil { + if err := d.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Debug().Err(err).Msg("Failed to update CR status") return false, errors.WithStack(err) } @@ -164,7 +165,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspectorInte } // createAction create action object based on action type -func (d *Reconciler) createAction(ctx context.Context, log zerolog.Logger, action api.Action, cachedStatus inspectorInterface.Inspector) Action { +func (d *Reconciler) createAction(log zerolog.Logger, action api.Action, cachedStatus inspectorInterface.Inspector) Action { actionCtx := newActionContext(log.With().Str("id", action.ID).Str("type", action.Type.String()).Logger(), d.context, cachedStatus) f, ok := getActionFactory(action.Type) diff --git a/pkg/deployment/reconcile/reconciler.go b/pkg/deployment/reconcile/reconciler.go index aa9ff6e7b..16757d4bc 100644 --- a/pkg/deployment/reconcile/reconciler.go +++ b/pkg/deployment/reconcile/reconciler.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,13 +18,17 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile import ( - api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "context" + "github.com/rs/zerolog" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" ) // Reconciler is the service that takes care of bring the a deployment @@ -43,7 +47,7 @@ func NewReconciler(log zerolog.Logger, context Context) *Reconciler { } // CheckDeployment checks for obviously broken things and fixes them immediately -func (r *Reconciler) CheckDeployment() error { +func (r *Reconciler) CheckDeployment(ctx context.Context) error { spec := r.context.GetSpec() status, _ := r.context.GetStatus() @@ -52,19 +56,19 @@ func (r *Reconciler) CheckDeployment() error { if len(status.Members.Coordinators) == 0 { // No more coordinators! Take immediate action r.log.Error().Msg("No Coordinator members! Create one member immediately") - _, err := r.context.CreateMember(api.ServerGroupCoordinators, "") + _, err := r.context.CreateMember(ctx, api.ServerGroupCoordinators, "") if err != nil { return err } } else if status.Members.Coordinators.AllFailed() { r.log.Error().Msg("All coordinators failed - reset") for _, m := range status.Members.Coordinators { - if err := r.context.DeletePod(m.PodName); err != nil { + if err := r.context.DeletePod(ctx, m.PodName); err != nil { r.log.Error().Err(err).Msg("Failed to delete pod") } m.Phase = api.MemberPhaseNone - if err := r.context.UpdateMember(m); err != nil { + if err := r.context.UpdateMember(ctx, m); err != nil { r.log.Error().Err(err).Msg("Failed to update member") } } diff --git a/pkg/deployment/resilience/context.go b/pkg/deployment/resilience/context.go index cc4543894..67ca3fa56 100644 --- a/pkg/deployment/resilience/context.go +++ b/pkg/deployment/resilience/context.go @@ -37,7 +37,7 @@ type Context interface { GetStatus() (api.DeploymentStatus, int32) // UpdateStatus replaces the status of the deployment with the given status and // updates the resources in k8s. - UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error + UpdateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error // GetAgencyClients returns a client connection for every agency member. // If the given predicate is not nil, only agents are included where the given predicate returns true. GetAgencyClients(ctx context.Context, predicate func(id string) bool) ([]driver.Connection, error) diff --git a/pkg/deployment/resilience/member_failure.go b/pkg/deployment/resilience/member_failure.go index e1339b52e..9eb415131 100644 --- a/pkg/deployment/resilience/member_failure.go +++ b/pkg/deployment/resilience/member_failure.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resilience @@ -42,7 +43,7 @@ const ( // CheckMemberFailure performs a check for members that should be in failed state because: // - They are frequently restarted // - They cannot be scheduled for a long time (TODO) -func (r *Resilience) CheckMemberFailure() error { +func (r *Resilience) CheckMemberFailure(ctx context.Context) error { status, lastVersion := r.context.GetStatus() updateStatusNeeded := false if err := status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { @@ -75,7 +76,8 @@ func (r *Resilience) CheckMemberFailure() error { if !m.Phase.IsFailed() { if m.IsNotReadySince(time.Now().Add(-notReadySinceGracePeriod)) { // Member has terminated too often in recent history. - failureAcceptable, reason, err := r.isMemberFailureAcceptable(status, group, m) + + failureAcceptable, reason, err := r.isMemberFailureAcceptable(ctx, group, m) if err != nil { log.Warn().Err(err).Msg("Failed to check is member failure is acceptable") } else if failureAcceptable { @@ -94,7 +96,7 @@ func (r *Resilience) CheckMemberFailure() error { count := m.RecentTerminationsSince(time.Now().Add(-recentTerminationsSinceGracePeriod)) if count >= recentTerminationThreshold { // Member has terminated too often in recent history. - failureAcceptable, reason, err := r.isMemberFailureAcceptable(status, group, m) + failureAcceptable, reason, err := r.isMemberFailureAcceptable(ctx, group, m) if err != nil { log.Warn().Err(err).Msg("Failed to check is member failure is acceptable") } else if failureAcceptable { @@ -114,7 +116,7 @@ func (r *Resilience) CheckMemberFailure() error { return errors.WithStack(err) } if updateStatusNeeded { - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } } @@ -125,12 +127,14 @@ func (r *Resilience) CheckMemberFailure() error { // isMemberFailureAcceptable checks if it is currently acceptable to switch the phase of the given member // to failed, which means that it will be replaced. // Return: failureAcceptable, notAcceptableReason, error -func (r *Resilience) isMemberFailureAcceptable(status api.DeploymentStatus, group api.ServerGroup, m api.MemberStatus) (bool, string, error) { - ctx := context.Background() +func (r *Resilience) isMemberFailureAcceptable(ctx context.Context, group api.ServerGroup, m api.MemberStatus) (bool, string, error) { + switch group { case api.ServerGroupAgents: // All good when remaining agents are health - clients, err := r.context.GetAgencyClients(ctx, func(id string) bool { return id != m.ID }) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + clients, err := r.context.GetAgencyClients(ctxChild, func(id string) bool { return id != m.ID }) if err != nil { return false, "", errors.WithStack(err) } @@ -139,7 +143,9 @@ func (r *Resilience) isMemberFailureAcceptable(status api.DeploymentStatus, grou } return true, "", nil case api.ServerGroupDBServers: - client, err := r.context.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + client, err := r.context.GetDatabaseClient(ctxChild) if err != nil { return false, "", errors.WithStack(err) } diff --git a/pkg/deployment/resources/annotations.go b/pkg/deployment/resources/annotations.go index 471e50f8a..ba0994a33 100644 --- a/pkg/deployment/resources/annotations.go +++ b/pkg/deployment/resources/annotations.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package resources @@ -29,7 +30,6 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/collection" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - monitoringTypedClient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1" "k8s.io/apimachinery/pkg/types" "github.com/arangodb/kube-arangodb/pkg/apis/deployment" @@ -39,17 +39,25 @@ import ( core "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" - typedCore "k8s.io/client-go/kubernetes/typed/core/v1" - policyTyped "k8s.io/client-go/kubernetes/typed/policy/v1beta1" ) -func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) error { +type PatchFunc func(name string, d []byte) error + +func (r *Resources) EnsureAnnotations(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { kubecli := r.context.GetKubeCli() monitoringcli := r.context.GetMonitoringV1Cli() log.Info().Msgf("Ensuring annotations") - if err := ensureSecretsAnnotations(kubecli.CoreV1().Secrets(r.context.GetNamespace()), + patchSecret := func(name string, d []byte) error { + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.CoreV1().Secrets(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + }) + } + + if err := ensureSecretsAnnotations(patchSecret, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -58,7 +66,15 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensureServiceAccountsAnnotations(kubecli.CoreV1().ServiceAccounts(r.context.GetNamespace()), + patchServiceAccount := func(name string, d []byte) error { + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.CoreV1().ServiceAccounts(r.context.GetNamespace()).Patch(ctxChild, name, + types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) + } + + if err := ensureServiceAccountsAnnotations(patchServiceAccount, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -67,7 +83,15 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensureServicesAnnotations(kubecli.CoreV1().Services(r.context.GetNamespace()), + patchService := func(name string, d []byte) error { + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.CoreV1().Services(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + }) + } + + if err := ensureServicesAnnotations(patchService, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -76,7 +100,15 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensurePdbsAnnotations(kubecli.PolicyV1beta1().PodDisruptionBudgets(r.context.GetNamespace()), + patchPDB := func(name string, d []byte) error { + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.PolicyV1beta1().PodDisruptionBudgets(r.context.GetNamespace()).Patch(ctxChild, name, + types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) + } + + if err := ensurePdbsAnnotations(patchPDB, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -85,7 +117,15 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensurePvcsAnnotations(kubecli.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()), + patchPVC := func(name string, d []byte) error { + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()).Patch(ctxChild, name, + types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) + } + + if err := ensurePvcsAnnotations(patchPVC, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -94,7 +134,15 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensurePodsAnnotations(kubecli.CoreV1().Pods(r.context.GetNamespace()), + patchPod := func(name string, d []byte) error { + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.CoreV1().Pods(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + }) + } + + if err := ensurePodsAnnotations(patchPod, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -104,7 +152,15 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensureServiceMonitorsAnnotations(monitoringcli.ServiceMonitors(r.context.GetNamespace()), + patchServiceMonitor := func(name string, d []byte) error { + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := monitoringcli.ServiceMonitors(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + }) + } + + if err := ensureServiceMonitorsAnnotations(patchServiceMonitor, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -116,12 +172,9 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return nil } -func ensureSecretsAnnotations(client typedCore.SecretInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensureSecretsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IterateSecrets(func(secret *core.Secret) error { - ensureAnnotationsMap(secret.Kind, secret, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureAnnotationsMap(secret.Kind, secret, spec, patch) return nil }, func(secret *core.Secret) bool { return k8sutil.IsChildResource(kind, name, namespace, secret) @@ -132,12 +185,9 @@ func ensureSecretsAnnotations(client typedCore.SecretInterface, cachedStatus ins return nil } -func ensureServiceAccountsAnnotations(client typedCore.ServiceAccountInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensureServiceAccountsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IterateServiceAccounts(func(serviceAccount *core.ServiceAccount) error { - ensureAnnotationsMap(serviceAccount.Kind, serviceAccount, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureAnnotationsMap(serviceAccount.Kind, serviceAccount, spec, patch) return nil }, func(serviceAccount *core.ServiceAccount) bool { return k8sutil.IsChildResource(kind, name, namespace, serviceAccount) @@ -148,12 +198,9 @@ func ensureServiceAccountsAnnotations(client typedCore.ServiceAccountInterface, return nil } -func ensureServicesAnnotations(client typedCore.ServiceInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensureServicesAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IterateServices(func(service *core.Service) error { - ensureAnnotationsMap(service.Kind, service, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureAnnotationsMap(service.Kind, service, spec, patch) return nil }, func(service *core.Service) bool { return k8sutil.IsChildResource(kind, name, namespace, service) @@ -164,12 +211,9 @@ func ensureServicesAnnotations(client typedCore.ServiceInterface, cachedStatus i return nil } -func ensurePdbsAnnotations(client policyTyped.PodDisruptionBudgetInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensurePdbsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IteratePodDisruptionBudgets(func(podDisruptionBudget *policy.PodDisruptionBudget) error { - ensureAnnotationsMap(podDisruptionBudget.Kind, podDisruptionBudget, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureAnnotationsMap(podDisruptionBudget.Kind, podDisruptionBudget, spec, patch) return nil }, func(podDisruptionBudget *policy.PodDisruptionBudget) bool { return k8sutil.IsChildResource(kind, name, namespace, podDisruptionBudget) @@ -180,12 +224,9 @@ func ensurePdbsAnnotations(client policyTyped.PodDisruptionBudgetInterface, cach return nil } -func ensurePvcsAnnotations(client typedCore.PersistentVolumeClaimInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensurePvcsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IteratePersistentVolumeClaims(func(persistentVolumeClaim *core.PersistentVolumeClaim) error { - ensureGroupAnnotationsMap(persistentVolumeClaim.Kind, persistentVolumeClaim, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureGroupAnnotationsMap(persistentVolumeClaim.Kind, persistentVolumeClaim, spec, patch) return nil }, func(persistentVolumeClaim *core.PersistentVolumeClaim) bool { return k8sutil.IsChildResource(kind, name, namespace, persistentVolumeClaim) @@ -196,12 +237,9 @@ func ensurePvcsAnnotations(client typedCore.PersistentVolumeClaimInterface, cach return nil } -func ensureServiceMonitorsAnnotations(client monitoringTypedClient.ServiceMonitorInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensureServiceMonitorsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IterateServiceMonitors(func(serviceMonitor *monitoring.ServiceMonitor) error { - ensureAnnotationsMap(serviceMonitor.Kind, serviceMonitor, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureAnnotationsMap(serviceMonitor.Kind, serviceMonitor, spec, patch) return nil }, func(serviceMonitor *monitoring.ServiceMonitor) bool { return k8sutil.IsChildResource(kind, name, namespace, serviceMonitor) @@ -226,12 +264,9 @@ func getObjectGroup(obj meta.Object) api.ServerGroup { return api.ServerGroupFromRole(group) } -func ensurePodsAnnotations(client typedCore.PodInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, annotations map[string]string, spec api.DeploymentSpec) error { +func ensurePodsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, annotations map[string]string, spec api.DeploymentSpec) error { if err := cachedStatus.IteratePods(func(pod *core.Pod) error { - ensureGroupAnnotationsMap(pod.Kind, pod, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureGroupAnnotationsMap(pod.Kind, pod, spec, patch) return nil }, func(pod *core.Pod) bool { return k8sutil.IsChildResource(kind, name, namespace, pod) @@ -292,8 +327,7 @@ func ensureGroupAnnotationsMap(kind string, obj meta.Object, spec api.Deployment return ensureObjectMap(kind, obj, mode, expected, obj.GetAnnotations(), collection.AnnotationsPatch, patchCmd, ignoredList...) } -func ensureAnnotationsMap(kind string, obj meta.Object, spec api.DeploymentSpec, - patchCmd func(name string, d []byte) error) bool { +func ensureAnnotationsMap(kind string, obj meta.Object, spec api.DeploymentSpec, patchCmd PatchFunc) bool { expected := spec.Annotations ignored := spec.AnnotationsIgnoreList @@ -305,7 +339,7 @@ func ensureAnnotationsMap(kind string, obj meta.Object, spec api.DeploymentSpec, func ensureObjectMap(kind string, obj meta.Object, mode api.LabelsMode, expected, actual map[string]string, patchGetter func(mode api.LabelsMode, expected map[string]string, actual map[string]string, ignored ...string) patch.Patch, - patchCmd func(name string, d []byte) error, + patchCmd PatchFunc, ignored ...string) bool { p := patchGetter(mode, expected, actual, ignored...) diff --git a/pkg/deployment/resources/certificates_client_auth.go b/pkg/deployment/resources/certificates_client_auth.go index 37d23acae..2b7cfd3e6 100644 --- a/pkg/deployment/resources/certificates_client_auth.go +++ b/pkg/deployment/resources/certificates_client_auth.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources import ( + "context" "fmt" "strings" "time" @@ -44,7 +46,7 @@ const ( // createClientAuthCACertificate creates a client authentication CA certificate and stores it in a secret with name // specified in the given spec. -func createClientAuthCACertificate(log zerolog.Logger, secrets k8sutil.SecretInterface, spec api.SyncAuthenticationSpec, deploymentName string, ownerRef *metav1.OwnerReference) error { +func createClientAuthCACertificate(ctx context.Context, log zerolog.Logger, secrets k8sutil.SecretInterface, spec api.SyncAuthenticationSpec, deploymentName string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", spec.GetClientCASecretName()).Logger() options := certificates.CreateCertificateOptions{ CommonName: fmt.Sprintf("%s Client Authentication Root Certificate", deploymentName), @@ -59,7 +61,7 @@ func createClientAuthCACertificate(log zerolog.Logger, secrets k8sutil.SecretInt log.Debug().Err(err).Msg("Failed to create CA certificate") return errors.WithStack(err) } - if err := k8sutil.CreateCASecret(secrets, spec.GetClientCASecretName(), cert, priv, ownerRef); err != nil { + if err := k8sutil.CreateCASecret(ctx, secrets, spec.GetClientCASecretName(), cert, priv, ownerRef); err != nil { if k8sutil.IsAlreadyExists(err) { log.Debug().Msg("CA Secret already exists") } else { @@ -76,7 +78,7 @@ func createClientAuthCACertificate(log zerolog.Logger, secrets k8sutil.SecretInt func createClientAuthCertificateKeyfile(log zerolog.Logger, secrets v1.SecretInterface, commonName string, ttl time.Duration, spec api.SyncAuthenticationSpec, secretName string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", secretName).Logger() // Load CA certificate - caCert, caKey, _, err := k8sutil.GetCASecret(secrets, spec.GetClientCASecretName(), nil) + caCert, caKey, _, err := k8sutil.GetCASecret(context.TODO(), secrets, spec.GetClientCASecretName(), nil) if err != nil { log.Debug().Err(err).Msg("Failed to load CA certificate") return errors.WithStack(err) @@ -102,7 +104,7 @@ func createClientAuthCertificateKeyfile(log zerolog.Logger, secrets v1.SecretInt } keyfile := strings.TrimSpace(cert) + "\n" + strings.TrimSpace(priv) - if err := k8sutil.CreateTLSKeyfileSecret(secrets, secretName, keyfile, ownerRef); err != nil { + if err := k8sutil.CreateTLSKeyfileSecret(context.TODO(), secrets, secretName, keyfile, ownerRef); err != nil { if k8sutil.IsAlreadyExists(err) { log.Debug().Msg("Server Secret already exists") } else { diff --git a/pkg/deployment/resources/certificates_tls.go b/pkg/deployment/resources/certificates_tls.go index 73913af1a..b4c54ebc1 100644 --- a/pkg/deployment/resources/certificates_tls.go +++ b/pkg/deployment/resources/certificates_tls.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources import ( + "context" "fmt" "strings" "time" @@ -45,7 +47,8 @@ const ( // createTLSCACertificate creates a CA certificate and stores it in a secret with name // specified in the given spec. -func createTLSCACertificate(log zerolog.Logger, secrets k8sutil.SecretInterface, spec api.TLSSpec, deploymentName string, ownerRef *metav1.OwnerReference) error { +func createTLSCACertificate(ctx context.Context, log zerolog.Logger, secrets k8sutil.SecretInterface, spec api.TLSSpec, + deploymentName string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", spec.GetCASecretName()).Logger() options := certificates.CreateCertificateOptions{ @@ -60,7 +63,7 @@ func createTLSCACertificate(log zerolog.Logger, secrets k8sutil.SecretInterface, log.Debug().Err(err).Msg("Failed to create CA certificate") return errors.WithStack(err) } - if err := k8sutil.CreateCASecret(secrets, spec.GetCASecretName(), cert, priv, ownerRef); err != nil { + if err := k8sutil.CreateCASecret(ctx, secrets, spec.GetCASecretName(), cert, priv, ownerRef); err != nil { if k8sutil.IsAlreadyExists(err) { log.Debug().Msg("CA Secret already exists") } else { @@ -74,7 +77,7 @@ func createTLSCACertificate(log zerolog.Logger, secrets k8sutil.SecretInterface, // createTLSServerCertificate creates a TLS certificate for a specific server and stores // it in a secret with the given name. -func createTLSServerCertificate(log zerolog.Logger, secrets v1.SecretInterface, serverNames []string, spec api.TLSSpec, +func createTLSServerCertificate(ctx context.Context, log zerolog.Logger, secrets v1.SecretInterface, serverNames []string, spec api.TLSSpec, secretName string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", secretName).Logger() @@ -86,7 +89,9 @@ func createTLSServerCertificate(log zerolog.Logger, secrets v1.SecretInterface, } // Load CA certificate - caCert, caKey, _, err := k8sutil.GetCASecret(secrets, spec.GetCASecretName(), nil) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + caCert, caKey, _, err := k8sutil.GetCASecret(ctxChild, secrets, spec.GetCASecretName(), nil) if err != nil { log.Debug().Err(err).Msg("Failed to load CA certificate") return errors.WithStack(err) @@ -113,7 +118,11 @@ func createTLSServerCertificate(log zerolog.Logger, secrets v1.SecretInterface, } keyfile := strings.TrimSpace(cert) + "\n" + strings.TrimSpace(priv) - if err := k8sutil.CreateTLSKeyfileSecret(secrets, secretName, keyfile, ownerRef); err != nil { + + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.CreateTLSKeyfileSecret(ctxChild, secrets, secretName, keyfile, ownerRef) + }) + if err != nil { if k8sutil.IsAlreadyExists(err) { log.Debug().Msg("Server Secret already exists") } else { diff --git a/pkg/deployment/resources/context.go b/pkg/deployment/resources/context.go index 6da4c6e8d..2f4ad19aa 100644 --- a/pkg/deployment/resources/context.go +++ b/pkg/deployment/resources/context.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -65,7 +66,7 @@ type Context interface { GetStatus() (api.DeploymentStatus, int32) // UpdateStatus replaces the status of the deployment with the given status and // updates the resources in k8s. - UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error + UpdateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error // GetKubeCli returns the kubernetes client GetKubeCli() kubernetes.Interface // GetMonitoringV1Cli returns monitoring client @@ -89,13 +90,13 @@ type Context interface { GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) // CleanupPod deletes a given pod with force and explicit UID. // If the pod does not exist, the error is ignored. - CleanupPod(p *v1.Pod) error + CleanupPod(ctx context.Context, p *v1.Pod) error // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - DeletePod(podName string) error + DeletePod(ctx context.Context, podName string) error // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. - DeletePvc(pvcName string) error + DeletePvc(ctx context.Context, pvcName string) error // GetAgencyClients returns a client connection for every agency member. GetAgencyClients(ctx context.Context, predicate func(memberID string) bool) ([]driver.Connection, error) // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), @@ -104,9 +105,9 @@ type Context interface { // GetAgency returns a connection to the entire agency. GetAgency(ctx context.Context) (agency.Agency, error) // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken - WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error + WithStatusUpdate(ctx context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error // GetBackup receives information about a backup resource - GetBackup(backup string) (*backupApi.ArangoBackup, error) + GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) GetScope() scope.Scope GetCachedStatus() inspectorInterface.Inspector diff --git a/pkg/deployment/resources/inspector/inspector.go b/pkg/deployment/resources/inspector/inspector.go index f91dffc15..b1b428716 100644 --- a/pkg/deployment/resources/inspector/inspector.go +++ b/pkg/deployment/resources/inspector/inspector.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -46,42 +47,43 @@ type SecretReadInterface interface { } func NewInspector(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, c versioned.Interface, namespace string) (inspectorInterface.Inspector, error) { - pods, err := podsToMap(k, namespace) + ctx := context.TODO() + pods, err := podsToMap(ctx, k, namespace) if err != nil { return nil, err } - secrets, err := secretsToMap(k, namespace) + secrets, err := secretsToMap(ctx, k, namespace) if err != nil { return nil, err } - pvcs, err := pvcsToMap(k, namespace) + pvcs, err := pvcsToMap(ctx, k, namespace) if err != nil { return nil, err } - services, err := servicesToMap(k, namespace) + services, err := servicesToMap(ctx, k, namespace) if err != nil { return nil, err } - serviceAccounts, err := serviceAccountsToMap(k, namespace) + serviceAccounts, err := serviceAccountsToMap(ctx, k, namespace) if err != nil { return nil, err } - podDisruptionBudgets, err := podDisruptionBudgetsToMap(k, namespace) + podDisruptionBudgets, err := podDisruptionBudgetsToMap(ctx, k, namespace) if err != nil { return nil, err } - serviceMonitors, err := serviceMonitorsToMap(m, namespace) + serviceMonitors, err := serviceMonitorsToMap(ctx, m, namespace) if err != nil { return nil, err } - arangoMembers, err := arangoMembersToMap(c, namespace) + arangoMembers, err := arangoMembersToMap(ctx, c, namespace) if err != nil { return nil, err } @@ -130,46 +132,47 @@ type inspector struct { m monitoringClient.MonitoringV1Interface } -func (i *inspector) Refresh(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, c versioned.Interface, namespace string) error { +func (i *inspector) Refresh(ctx context.Context, k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, + c versioned.Interface, namespace string) error { i.lock.Lock() defer i.lock.Unlock() - pods, err := podsToMap(k, namespace) + pods, err := podsToMap(ctx, k, namespace) if err != nil { return err } - secrets, err := secretsToMap(k, namespace) + secrets, err := secretsToMap(ctx, k, namespace) if err != nil { return err } - pvcs, err := pvcsToMap(k, namespace) + pvcs, err := pvcsToMap(ctx, k, namespace) if err != nil { return err } - services, err := servicesToMap(k, namespace) + services, err := servicesToMap(ctx, k, namespace) if err != nil { return err } - serviceAccounts, err := serviceAccountsToMap(k, namespace) + serviceAccounts, err := serviceAccountsToMap(ctx, k, namespace) if err != nil { return err } - podDisruptionBudgets, err := podDisruptionBudgetsToMap(k, namespace) + podDisruptionBudgets, err := podDisruptionBudgetsToMap(ctx, k, namespace) if err != nil { return err } - serviceMonitors, err := serviceMonitorsToMap(m, namespace) + serviceMonitors, err := serviceMonitorsToMap(ctx, m, namespace) if err != nil { return err } - arangoMembers, err := arangoMembersToMap(c, namespace) + arangoMembers, err := arangoMembersToMap(ctx, c, namespace) if err != nil { return err } diff --git a/pkg/deployment/resources/inspector/members.go b/pkg/deployment/resources/inspector/members.go index 620b45679..8d5ef7b9f 100644 --- a/pkg/deployment/resources/inspector/members.go +++ b/pkg/deployment/resources/inspector/members.go @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" "github.com/arangodb/kube-arangodb/pkg/util/errors" @@ -75,8 +78,8 @@ func (i *inspector) ArangoMember(name string) (*api.ArangoMember, bool) { return arangoMember, true } -func arangoMembersToMap(k versioned.Interface, namespace string) (map[string]*api.ArangoMember, error) { - arangoMembers, err := getArangoMembers(k, namespace, "") +func arangoMembersToMap(ctx context.Context, k versioned.Interface, namespace string) (map[string]*api.ArangoMember, error) { + arangoMembers, err := getArangoMembers(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,8 +102,10 @@ func arangoMemberPointer(pod api.ArangoMember) *api.ArangoMember { return &pod } -func getArangoMembers(k versioned.Interface, namespace, cont string) ([]api.ArangoMember, error) { - arangoMembers, err := k.DatabaseV1().ArangoMembers(namespace).List(context.Background(), meta.ListOptions{ +func getArangoMembers(ctx context.Context, k versioned.Interface, namespace, cont string) ([]api.ArangoMember, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + arangoMembers, err := k.DatabaseV1().ArangoMembers(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) @@ -110,7 +115,7 @@ func getArangoMembers(k versioned.Interface, namespace, cont string) ([]api.Aran } if arangoMembers.Continue != "" { - nextArangoMembersLayer, err := getArangoMembers(k, namespace, arangoMembers.Continue) + nextArangoMembersLayer, err := getArangoMembers(ctx, k, namespace, arangoMembers.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/pdbs.go b/pkg/deployment/resources/inspector/pdbs.go index 95bc74b0f..a7f6c0263 100644 --- a/pkg/deployment/resources/inspector/pdbs.go +++ b/pkg/deployment/resources/inspector/pdbs.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/poddisruptionbudget" policy "k8s.io/api/policy/v1beta1" @@ -75,8 +78,8 @@ func (i *inspector) PodDisruptionBudget(name string) (*policy.PodDisruptionBudge return podDisruptionBudget, true } -func podDisruptionBudgetsToMap(k kubernetes.Interface, namespace string) (map[string]*policy.PodDisruptionBudget, error) { - podDisruptionBudgets, err := getPodDisruptionBudgets(k, namespace, "") +func podDisruptionBudgetsToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*policy.PodDisruptionBudget, error) { + podDisruptionBudgets, err := getPodDisruptionBudgets(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,8 +102,10 @@ func podDisruptionBudgetPointer(podDisruptionBudget policy.PodDisruptionBudget) return &podDisruptionBudget } -func getPodDisruptionBudgets(k kubernetes.Interface, namespace, cont string) ([]policy.PodDisruptionBudget, error) { - podDisruptionBudgets, err := k.PolicyV1beta1().PodDisruptionBudgets(namespace).List(context.Background(), meta.ListOptions{ +func getPodDisruptionBudgets(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]policy.PodDisruptionBudget, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + podDisruptionBudgets, err := k.PolicyV1beta1().PodDisruptionBudgets(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) @@ -110,7 +115,7 @@ func getPodDisruptionBudgets(k kubernetes.Interface, namespace, cont string) ([] } if podDisruptionBudgets.Continue != "" { - nextPodDisruptionBudgetsLayer, err := getPodDisruptionBudgets(k, namespace, podDisruptionBudgets.Continue) + nextPodDisruptionBudgetsLayer, err := getPodDisruptionBudgets(ctx, k, namespace, podDisruptionBudgets.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/pods.go b/pkg/deployment/resources/inspector/pods.go index aad1db3a3..1f2eb514b 100644 --- a/pkg/deployment/resources/inspector/pods.go +++ b/pkg/deployment/resources/inspector/pods.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -26,6 +27,7 @@ import ( "context" "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/pod" core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -75,8 +77,8 @@ func (i *inspector) Pod(name string) (*core.Pod, bool) { return pod, true } -func podsToMap(k kubernetes.Interface, namespace string) (map[string]*core.Pod, error) { - pods, err := getPods(k, namespace, "") +func podsToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*core.Pod, error) { + pods, err := getPods(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,8 +101,10 @@ func podPointer(pod core.Pod) *core.Pod { return &pod } -func getPods(k kubernetes.Interface, namespace, cont string) ([]core.Pod, error) { - pods, err := k.CoreV1().Pods(namespace).List(context.Background(), meta.ListOptions{ +func getPods(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.Pod, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + pods, err := k.CoreV1().Pods(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) @@ -110,7 +114,8 @@ func getPods(k kubernetes.Interface, namespace, cont string) ([]core.Pod, error) } if pods.Continue != "" { - nextPodsLayer, err := getPods(k, namespace, pods.Continue) + // pass the original context + nextPodsLayer, err := getPods(ctx, k, namespace, pods.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/pvcs.go b/pkg/deployment/resources/inspector/pvcs.go index 10d71493c..ea83cf8c6 100644 --- a/pkg/deployment/resources/inspector/pvcs.go +++ b/pkg/deployment/resources/inspector/pvcs.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/persistentvolumeclaim" core "k8s.io/api/core/v1" @@ -75,8 +78,8 @@ func (i *inspector) PersistentVolumeClaim(name string) (*core.PersistentVolumeCl return pvc, true } -func pvcsToMap(k kubernetes.Interface, namespace string) (map[string]*core.PersistentVolumeClaim, error) { - pvcs, err := getPersistentVolumeClaims(k, namespace, "") +func pvcsToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*core.PersistentVolumeClaim, error) { + pvcs, err := getPersistentVolumeClaims(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,8 +102,10 @@ func pvcPointer(pvc core.PersistentVolumeClaim) *core.PersistentVolumeClaim { return &pvc } -func getPersistentVolumeClaims(k kubernetes.Interface, namespace, cont string) ([]core.PersistentVolumeClaim, error) { - pvcs, err := k.CoreV1().PersistentVolumeClaims(namespace).List(context.Background(), meta.ListOptions{ +func getPersistentVolumeClaims(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.PersistentVolumeClaim, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + pvcs, err := k.CoreV1().PersistentVolumeClaims(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) @@ -110,7 +115,7 @@ func getPersistentVolumeClaims(k kubernetes.Interface, namespace, cont string) ( } if pvcs.Continue != "" { - nextPersistentVolumeClaimsLayer, err := getPersistentVolumeClaims(k, namespace, pvcs.Continue) + nextPersistentVolumeClaimsLayer, err := getPersistentVolumeClaims(ctx, k, namespace, pvcs.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/sa.go b/pkg/deployment/resources/inspector/sa.go index 514727d01..8ae14e07d 100644 --- a/pkg/deployment/resources/inspector/sa.go +++ b/pkg/deployment/resources/inspector/sa.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/serviceaccount" core "k8s.io/api/core/v1" @@ -75,8 +78,8 @@ func (i *inspector) ServiceAccount(name string) (*core.ServiceAccount, bool) { return serviceAccount, true } -func serviceAccountsToMap(k kubernetes.Interface, namespace string) (map[string]*core.ServiceAccount, error) { - serviceAccounts, err := getServiceAccounts(k, namespace, "") +func serviceAccountsToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*core.ServiceAccount, error) { + serviceAccounts, err := getServiceAccounts(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,8 +102,10 @@ func serviceAccountPointer(serviceAccount core.ServiceAccount) *core.ServiceAcco return &serviceAccount } -func getServiceAccounts(k kubernetes.Interface, namespace, cont string) ([]core.ServiceAccount, error) { - serviceAccounts, err := k.CoreV1().ServiceAccounts(namespace).List(context.Background(), meta.ListOptions{ +func getServiceAccounts(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.ServiceAccount, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + serviceAccounts, err := k.CoreV1().ServiceAccounts(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) @@ -110,7 +115,7 @@ func getServiceAccounts(k kubernetes.Interface, namespace, cont string) ([]core. } if serviceAccounts.Continue != "" { - nextServiceAccountsLayer, err := getServiceAccounts(k, namespace, serviceAccounts.Continue) + nextServiceAccountsLayer, err := getServiceAccounts(ctx, k, namespace, serviceAccounts.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/secrets.go b/pkg/deployment/resources/inspector/secrets.go index d05502dcb..7c3ca8cfa 100644 --- a/pkg/deployment/resources/inspector/secrets.go +++ b/pkg/deployment/resources/inspector/secrets.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/secret" core "k8s.io/api/core/v1" @@ -96,8 +99,8 @@ func (s secretReadInterface) Get(ctx context.Context, name string, opts meta.Get } } -func secretsToMap(k kubernetes.Interface, namespace string) (map[string]*core.Secret, error) { - secrets, err := getSecrets(k, namespace, "") +func secretsToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*core.Secret, error) { + secrets, err := getSecrets(ctx, k, namespace, "") if err != nil { return nil, err } @@ -120,8 +123,10 @@ func secretPointer(pod core.Secret) *core.Secret { return &pod } -func getSecrets(k kubernetes.Interface, namespace, cont string) ([]core.Secret, error) { - secrets, err := k.CoreV1().Secrets(namespace).List(context.Background(), meta.ListOptions{ +func getSecrets(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.Secret, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + secrets, err := k.CoreV1().Secrets(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) @@ -131,7 +136,7 @@ func getSecrets(k kubernetes.Interface, namespace, cont string) ([]core.Secret, } if secrets.Continue != "" { - nextSecretsLayer, err := getSecrets(k, namespace, secrets.Continue) + nextSecretsLayer, err := getSecrets(ctx, k, namespace, secrets.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/services.go b/pkg/deployment/resources/inspector/services.go index 410a65a62..f60977b84 100644 --- a/pkg/deployment/resources/inspector/services.go +++ b/pkg/deployment/resources/inspector/services.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/service" core "k8s.io/api/core/v1" @@ -75,8 +78,8 @@ func (i *inspector) Service(name string) (*core.Service, bool) { return service, true } -func servicesToMap(k kubernetes.Interface, namespace string) (map[string]*core.Service, error) { - services, err := getServices(k, namespace, "") +func servicesToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*core.Service, error) { + services, err := getServices(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,8 +102,10 @@ func servicePointer(pod core.Service) *core.Service { return &pod } -func getServices(k kubernetes.Interface, namespace, cont string) ([]core.Service, error) { - services, err := k.CoreV1().Services(namespace).List(context.Background(), meta.ListOptions{ +func getServices(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.Service, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + services, err := k.CoreV1().Services(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) @@ -110,7 +115,7 @@ func getServices(k kubernetes.Interface, namespace, cont string) ([]core.Service } if services.Continue != "" { - nextServicesLayer, err := getServices(k, namespace, services.Continue) + nextServicesLayer, err := getServices(ctx, k, namespace, services.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/sms.go b/pkg/deployment/resources/inspector/sms.go index 3f1d4bbc8..3a6df419e 100644 --- a/pkg/deployment/resources/inspector/sms.go +++ b/pkg/deployment/resources/inspector/sms.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/servicemonitor" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -75,8 +78,8 @@ func (i *inspector) ServiceMonitor(name string) (*monitoring.ServiceMonitor, boo return serviceMonitor, true } -func serviceMonitorsToMap(m monitoringClient.MonitoringV1Interface, namespace string) (map[string]*monitoring.ServiceMonitor, error) { - serviceMonitors, err := getServiceMonitors(m, namespace, "") +func serviceMonitorsToMap(ctx context.Context, m monitoringClient.MonitoringV1Interface, namespace string) (map[string]*monitoring.ServiceMonitor, error) { + serviceMonitors, err := getServiceMonitors(ctx, m, namespace, "") if err != nil { return nil, err } @@ -95,8 +98,10 @@ func serviceMonitorsToMap(m monitoringClient.MonitoringV1Interface, namespace st return serviceMonitorMap, nil } -func getServiceMonitors(m monitoringClient.MonitoringV1Interface, namespace, cont string) ([]*monitoring.ServiceMonitor, error) { - serviceMonitors, err := m.ServiceMonitors(namespace).List(context.Background(), meta.ListOptions{ +func getServiceMonitors(ctx context.Context, m monitoringClient.MonitoringV1Interface, namespace, cont string) ([]*monitoring.ServiceMonitor, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + serviceMonitors, err := m.ServiceMonitors(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) diff --git a/pkg/deployment/resources/labels.go b/pkg/deployment/resources/labels.go index 2c8bfe58d..4fffcdfa7 100644 --- a/pkg/deployment/resources/labels.go +++ b/pkg/deployment/resources/labels.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package resources @@ -26,55 +27,58 @@ import ( "context" "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - core "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) -func (r *Resources) EnsureLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { r.log.Info().Msgf("Ensuring labels") - if err := r.EnsureSecretLabels(cachedStatus); err != nil { + if err := r.EnsureSecretLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsureServiceAccountsLabels(cachedStatus); err != nil { + if err := r.EnsureServiceAccountsLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsureServicesLabels(cachedStatus); err != nil { + if err := r.EnsureServicesLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsureServiceMonitorsLabels(cachedStatus); err != nil { + if err := r.EnsureServiceMonitorsLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsurePodsLabels(cachedStatus); err != nil { + if err := r.EnsurePodsLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsurePersistentVolumeClaimsLabels(cachedStatus); err != nil { + if err := r.EnsurePersistentVolumeClaimsLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsurePodDisruptionBudgetsLabels(cachedStatus); err != nil { + if err := r.EnsurePodDisruptionBudgetsLabels(ctx, cachedStatus); err != nil { return err } return nil } -func (r *Resources) EnsureSecretLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureSecretLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IterateSecrets(func(secret *core.Secret) error { if ensureLabelsMap(secret.Kind, secret, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().CoreV1().Secrets(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().CoreV1().Secrets(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, + name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -93,12 +97,15 @@ func (r *Resources) EnsureSecretLabels(cachedStatus inspectorInterface.Inspector return nil } -func (r *Resources) EnsureServiceAccountsLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureServiceAccountsLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IterateServiceAccounts(func(serviceAccount *core.ServiceAccount) error { if ensureLabelsMap(serviceAccount.Kind, serviceAccount, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().CoreV1().ServiceAccounts(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().CoreV1().ServiceAccounts(r.context.GetAPIObject().GetNamespace()). + Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -117,12 +124,15 @@ func (r *Resources) EnsureServiceAccountsLabels(cachedStatus inspectorInterface. return nil } -func (r *Resources) EnsureServicesLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureServicesLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IterateServices(func(service *core.Service) error { if ensureLabelsMap(service.Kind, service, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().CoreV1().Services(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().CoreV1().Services(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, + name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -141,12 +151,15 @@ func (r *Resources) EnsureServicesLabels(cachedStatus inspectorInterface.Inspect return nil } -func (r *Resources) EnsureServiceMonitorsLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureServiceMonitorsLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IterateServiceMonitors(func(serviceMonitor *monitoring.ServiceMonitor) error { if ensureLabelsMap(serviceMonitor.Kind, serviceMonitor, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetMonitoringV1Cli().ServiceMonitors(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetMonitoringV1Cli().ServiceMonitors(r.context.GetAPIObject().GetNamespace()). + Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -165,12 +178,15 @@ func (r *Resources) EnsureServiceMonitorsLabels(cachedStatus inspectorInterface. return nil } -func (r *Resources) EnsurePodsLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsurePodsLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IteratePods(func(pod *core.Pod) error { if ensureGroupLabelsMap(pod.Kind, pod, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().CoreV1().Pods(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().CoreV1().Pods(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, + name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -189,12 +205,15 @@ func (r *Resources) EnsurePodsLabels(cachedStatus inspectorInterface.Inspector) return nil } -func (r *Resources) EnsurePersistentVolumeClaimsLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsurePersistentVolumeClaimsLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IteratePersistentVolumeClaims(func(persistentVolumeClaim *core.PersistentVolumeClaim) error { if ensureGroupLabelsMap(persistentVolumeClaim.Kind, persistentVolumeClaim, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetAPIObject().GetNamespace()). + Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -213,12 +232,15 @@ func (r *Resources) EnsurePersistentVolumeClaimsLabels(cachedStatus inspectorInt return nil } -func (r *Resources) EnsurePodDisruptionBudgetsLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsurePodDisruptionBudgetsLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IteratePodDisruptionBudgets(func(budget *policy.PodDisruptionBudget) error { if ensureLabelsMap(budget.Kind, budget, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().PolicyV1beta1().PodDisruptionBudgets(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().PolicyV1beta1().PodDisruptionBudgets(r.context.GetAPIObject(). + GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } diff --git a/pkg/deployment/resources/member_cleanup.go b/pkg/deployment/resources/member_cleanup.go index aba0af2d7..15cc09fb6 100644 --- a/pkg/deployment/resources/member_cleanup.go +++ b/pkg/deployment/resources/member_cleanup.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -49,12 +50,12 @@ var ( ) // CleanupRemovedMembers removes all arangod members that are no longer part of ArangoDB deployment. -func (r *Resources) CleanupRemovedMembers() error { +func (r *Resources) CleanupRemovedMembers(ctx context.Context) error { // Decide what to do depending on cluster mode switch r.context.GetSpec().GetMode() { case api.DeploymentModeCluster: deploymentName := r.context.GetAPIObject().GetName() - if err := r.cleanupRemovedClusterMembers(); err != nil { + if err := r.cleanupRemovedClusterMembers(ctx); err != nil { cleanupRemovedMembersCounters.WithLabelValues(deploymentName, metrics.Failed).Inc() return errors.WithStack(err) } @@ -67,7 +68,7 @@ func (r *Resources) CleanupRemovedMembers() error { } // cleanupRemovedClusterMembers removes all arangod members that are no longer part of the cluster. -func (r *Resources) cleanupRemovedClusterMembers() error { +func (r *Resources) cleanupRemovedClusterMembers(ctx context.Context) error { log := r.log // Fetch recent cluster health @@ -141,7 +142,7 @@ func (r *Resources) cleanupRemovedClusterMembers() error { if updateStatusNeeded { log.Debug().Msg("UpdateStatus needed") - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Warn().Err(err).Msg("Failed to update deployment status") return errors.WithStack(err) } @@ -149,14 +150,14 @@ func (r *Resources) cleanupRemovedClusterMembers() error { for _, podName := range podNamesToRemove { log.Info().Str("pod", podName).Msg("Removing obsolete member pod") - if err := r.context.DeletePod(podName); err != nil && !k8sutil.IsNotFound(err) { + if err := r.context.DeletePod(ctx, podName); err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Str("pod", podName).Msg("Failed to remove obsolete pod") } } for _, pvcName := range pvcNamesToRemove { log.Info().Str("pvc", pvcName).Msg("Removing obsolete member PVC") - if err := r.context.DeletePvc(pvcName); err != nil && !k8sutil.IsNotFound(err) { + if err := r.context.DeletePvc(ctx, pvcName); err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Str("pvc", pvcName).Msg("Failed to remove obsolete PVC") } } @@ -164,7 +165,7 @@ func (r *Resources) cleanupRemovedClusterMembers() error { return nil } -func (r *Resources) EnsureArangoMembers(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureArangoMembers(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { // Create all missing arangomembers s, _ := r.context.GetStatus() @@ -187,7 +188,11 @@ func (r *Resources) EnsureArangoMembers(cachedStatus inspectorInterface.Inspecto }, } - if _, err := r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Create(context.Background(), &a, metav1.CreateOptions{}); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Create(ctxChild, &a, metav1.CreateOptions{}) + return err + }) + if err != nil { return err } @@ -205,7 +210,11 @@ func (r *Resources) EnsureArangoMembers(cachedStatus inspectorInterface.Inspecto if !ok || g != member.Spec.Group { // Remove member - if err := r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Delete(context.Background(), member.GetName(), metav1.DeleteOptions{}); err != nil { + + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Delete(ctxChild, member.GetName(), metav1.DeleteOptions{}) + }) + if err != nil { if !k8sutil.IsNotFound(err) { return err } diff --git a/pkg/deployment/resources/pdbs.go b/pkg/deployment/resources/pdbs.go index a43a14679..b68cdf401 100644 --- a/pkg/deployment/resources/pdbs.go +++ b/pkg/deployment/resources/pdbs.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package resources @@ -44,7 +45,7 @@ func min(a int, b int) int { } // EnsurePDBs ensures Pod Disruption Budgets for different server groups in Cluster mode -func (r *Resources) EnsurePDBs() error { +func (r *Resources) EnsurePDBs(ctx context.Context) error { // Only in Cluster and Production Mode spec := r.context.GetSpec() @@ -65,19 +66,19 @@ func (r *Resources) EnsurePDBs() error { } // Ensure all PDBs as calculated - if err := r.ensurePDBForGroup(api.ServerGroupAgents, minAgents); err != nil { + if err := r.ensurePDBForGroup(ctx, api.ServerGroupAgents, minAgents); err != nil { return err } - if err := r.ensurePDBForGroup(api.ServerGroupDBServers, minDBServers); err != nil { + if err := r.ensurePDBForGroup(ctx, api.ServerGroupDBServers, minDBServers); err != nil { return err } - if err := r.ensurePDBForGroup(api.ServerGroupCoordinators, minCoordinators); err != nil { + if err := r.ensurePDBForGroup(ctx, api.ServerGroupCoordinators, minCoordinators); err != nil { return err } - if err := r.ensurePDBForGroup(api.ServerGroupSyncMasters, minSyncMaster); err != nil { + if err := r.ensurePDBForGroup(ctx, api.ServerGroupSyncMasters, minSyncMaster); err != nil { return err } - if err := r.ensurePDBForGroup(api.ServerGroupSyncWorkers, minSyncWorker); err != nil { + if err := r.ensurePDBForGroup(ctx, api.ServerGroupSyncWorkers, minSyncWorker); err != nil { return err } } @@ -105,23 +106,29 @@ func newPDB(minAvail int, deplname string, group api.ServerGroup, owner metav1.O } // ensurePDBForGroup ensure pdb for a specific server group, if wantMinAvail is zero, the PDB is removed and not recreated -func (r *Resources) ensurePDBForGroup(group api.ServerGroup, wantedMinAvail int) error { +func (r *Resources) ensurePDBForGroup(ctx context.Context, group api.ServerGroup, wantedMinAvail int) error { deplname := r.context.GetAPIObject().GetName() pdbname := PDBNameForGroup(deplname, group) pdbcli := r.context.GetKubeCli().PolicyV1beta1().PodDisruptionBudgets(r.context.GetNamespace()) log := r.log.With().Str("group", group.AsRole()).Logger() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - for { - pdb, err := pdbcli.Get(context.Background(), pdbname, metav1.GetOptions{}) + var pdb *policyv1beta1.PodDisruptionBudget + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + pdb, err = pdbcli.Get(ctxChild, pdbname, metav1.GetOptions{}) + return err + }) if k8sutil.IsNotFound(err) { if wantedMinAvail != 0 { // No PDB found - create new pdb := newPDB(wantedMinAvail, deplname, group, r.context.GetAPIObject().AsOwner()) log.Debug().Msg("Creating new PDB") - if _, err := pdbcli.Create(context.Background(), pdb, metav1.CreateOptions{}); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := pdbcli.Create(ctxChild, pdb, metav1.CreateOptions{}) + return err + }) + if err != nil { log.Error().Err(err).Msg("failed to create PDB") return errors.WithStack(err) } @@ -142,7 +149,10 @@ func (r *Resources) ensurePDBForGroup(group api.ServerGroup, wantedMinAvail int) // Trigger deletion only if not already deleted if pdb.GetDeletionTimestamp() == nil { // Update the PDB - if err := pdbcli.Delete(context.Background(), pdbname, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return pdbcli.Delete(ctxChild, pdbname, metav1.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Error().Err(err).Msg("PDB deletion failed") return errors.WithStack(err) } diff --git a/pkg/deployment/resources/pod_cleanup.go b/pkg/deployment/resources/pod_cleanup.go index 501ab5503..d47c31e9b 100644 --- a/pkg/deployment/resources/pod_cleanup.go +++ b/pkg/deployment/resources/pod_cleanup.go @@ -23,6 +23,7 @@ package resources import ( + "context" "time" "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" @@ -42,7 +43,7 @@ const ( // CleanupTerminatedPods removes all pods in Terminated state that belong to a member in Created state. // Returns: Interval_till_next_inspection, error -func (r *Resources) CleanupTerminatedPods(cachedStatus inspectorInterface.Inspector) (util.Interval, error) { +func (r *Resources) CleanupTerminatedPods(ctx context.Context, cachedStatus inspectorInterface.Inspector) (util.Interval, error) { log := r.log nextInterval := maxPodInspectorInterval // Large by default, will be made smaller if needed in the rest of the function @@ -84,7 +85,7 @@ func (r *Resources) CleanupTerminatedPods(cachedStatus inspectorInterface.Inspec // Ok, we can delete the pod log.Debug().Str("pod-name", pod.GetName()).Msg("Cleanup terminated pod") - if err := r.context.CleanupPod(pod); err != nil { + if err := r.context.CleanupPod(ctx, pod); err != nil { log.Warn().Err(err).Str("pod-name", pod.GetName()).Msg("Failed to cleanup pod") } diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index aa46125f6..be974f0e2 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources import ( + "context" "crypto/sha1" "crypto/sha256" "encoding/json" @@ -295,7 +297,7 @@ func (r *Resources) CreatePodTolerations(group api.ServerGroup, groupSpec api.Se return tolerations } -func (r *Resources) RenderPodForMember(cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { +func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { log := r.log apiObject := r.context.GetAPIObject() m, group, found := status.Members.ElementByID(memberID) @@ -365,14 +367,20 @@ func (r *Resources) RenderPodForMember(cachedStatus inspectorInterface.Inspector var tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName string // Check master JWT secret - masterJWTSecretName = spec.Sync.Authentication.GetJWTSecretName() - if err := k8sutil.ValidateTokenSecret(secrets, masterJWTSecretName); err != nil { + masterJWTSecretName = spec.Sync.Authentication.GetJWTSecretName() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.ValidateTokenSecret(ctxChild, secrets, masterJWTSecretName) + }) + if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Master JWT secret validation failed")) } monitoringTokenSecretName := spec.Sync.Monitoring.GetTokenSecretName() - if err := k8sutil.ValidateTokenSecret(secrets, monitoringTokenSecretName); err != nil { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.ValidateTokenSecret(ctxChild, secrets, monitoringTokenSecretName) + }) + if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Monitoring token secret validation failed")) } @@ -382,13 +390,19 @@ func (r *Resources) RenderPodForMember(cachedStatus inspectorInterface.Inspector // Check cluster JWT secret if spec.IsAuthenticated() { clusterJWTSecretName = spec.Authentication.GetJWTSecretName() - if err := k8sutil.ValidateTokenSecret(secrets, clusterJWTSecretName); err != nil { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.ValidateTokenSecret(ctxChild, secrets, clusterJWTSecretName) + }) + if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Cluster JWT secret validation failed")) } } // Check client-auth CA certificate secret clientAuthCASecretName = spec.Sync.Authentication.GetClientCASecretName() - if err := k8sutil.ValidateCACertificateSecret(secrets, clientAuthCASecretName); err != nil { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.ValidateCACertificateSecret(ctxChild, secrets, clientAuthCASecretName) + }) + if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Client authentication CA certificate secret validation failed")) } } @@ -434,7 +448,7 @@ func (r *Resources) SelectImage(spec api.DeploymentSpec, status api.DeploymentSt } // createPodForMember creates all Pods listed in member status -func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, imageNotFoundOnce *sync.Once, cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentSpec, memberID string, imageNotFoundOnce *sync.Once, cachedStatus inspectorInterface.Inspector) error { log := r.log status, lastVersion := r.context.GetStatus() @@ -462,7 +476,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, imageInfo = *m.Image - pod, err := r.RenderPodForMember(cachedStatus, spec, status, memberID, imageInfo) + pod, err := r.RenderPodForMember(ctx, cachedStatus, spec, status, memberID, imageInfo) if err != nil { return errors.WithStack(err) } @@ -495,7 +509,9 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, return errors.WithStack(err) } - uid, err := CreateArangoPod(kubecli, apiObject, spec, group, pod) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + uid, err := CreateArangoPod(ctxChild, kubecli, apiObject, spec, group, pod) if err != nil { return errors.WithStack(err) } @@ -535,7 +551,8 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, } } owner := apiObject.AsOwner() - if err := createTLSServerCertificate(log, secrets, serverNames, spec.Sync.TLS, tlsKeyfileSecretName, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { + err := createTLSServerCertificate(ctx, log, secrets, serverNames, spec.Sync.TLS, tlsKeyfileSecretName, &owner) + if err != nil && !k8sutil.IsAlreadyExists(err) { return errors.WithStack(errors.Wrapf(err, "Failed to create TLS keyfile secret")) } } @@ -545,7 +562,9 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, return errors.WithStack(err) } - uid, err := CreateArangoPod(kubecli, apiObject, spec, group, pod) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + uid, err := CreateArangoPod(ctxChild, kubecli, apiObject, spec, group, pod) if err != nil { return errors.WithStack(err) } @@ -567,7 +586,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, if err := status.Members.Update(m, group); err != nil { return errors.WithStack(err) } - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } // Create event @@ -633,8 +652,8 @@ func RenderArangoPod(deployment k8sutil.APIObject, role, id, podName string, // CreateArangoPod creates a new Pod with container provided by parameter 'containerCreator' // If the pod already exists, nil is returned. // If another error occurs, that error is returned. -func CreateArangoPod(kubecli kubernetes.Interface, deployment k8sutil.APIObject, deploymentSpec api.DeploymentSpec, group api.ServerGroup, pod *core.Pod) (types.UID, error) { - uid, err := k8sutil.CreatePod(kubecli, pod, deployment.GetNamespace(), deployment.AsOwner()) +func CreateArangoPod(ctx context.Context, kubecli kubernetes.Interface, deployment k8sutil.APIObject, deploymentSpec api.DeploymentSpec, group api.ServerGroup, pod *core.Pod) (types.UID, error) { + uid, err := k8sutil.CreatePod(ctx, kubecli, pod, deployment.GetNamespace(), deployment.AsOwner()) if err != nil { return "", errors.WithStack(err) } @@ -660,7 +679,7 @@ func ChecksumArangoPod(groupSpec api.ServerGroupSpec, pod *core.Pod) (string, er } // EnsurePods creates all Pods listed in member status -func (r *Resources) EnsurePods(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsurePods(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { iterator := r.context.GetServerGroupIterator() deploymentStatus, _ := r.context.GetStatus() imageNotFoundOnce := &sync.Once{} @@ -674,7 +693,7 @@ func (r *Resources) EnsurePods(cachedStatus inspectorInterface.Inspector) error continue } spec := r.context.GetSpec() - if err := r.createPodForMember(spec, m.ID, imageNotFoundOnce, cachedStatus); err != nil { + if err := r.createPodForMember(ctx, spec, m.ID, imageNotFoundOnce, cachedStatus); err != nil { return errors.WithStack(err) } } diff --git a/pkg/deployment/resources/pod_finalizers.go b/pkg/deployment/resources/pod_finalizers.go index 6e05e6061..2a6ccdef6 100644 --- a/pkg/deployment/resources/pod_finalizers.go +++ b/pkg/deployment/resources/pod_finalizers.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -88,8 +89,7 @@ func (r *Resources) runPodFinalizers(ctx context.Context, p *v1.Pod, memberStatu // Remove finalizers (if needed) if len(removalList) > 0 { kubecli := r.context.GetKubeCli() - ignoreNotFound := false - if err := k8sutil.RemovePodFinalizers(log, kubecli, p, removalList, ignoreNotFound); err != nil { + if err := k8sutil.RemovePodFinalizers(ctx, log, kubecli, p, removalList, false); err != nil { log.Debug().Err(err).Msg("Failed to update pod (to remove finalizers)") return 0, errors.WithStack(err) } @@ -119,7 +119,10 @@ func (r *Resources) inspectFinalizerPodAgencyServing(ctx context.Context, log ze // of the agent, also remove the PVC if memberStatus.Conditions.IsTrue(api.ConditionTypeAgentRecoveryNeeded) { pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetNamespace()) - if err := pvcs.Delete(context.Background(), memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return pvcs.Delete(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete PVC for member") return errors.WithStack(err) } @@ -146,7 +149,10 @@ func (r *Resources) inspectFinalizerPodDrainDBServer(ctx context.Context, log ze // If this DBServer is cleaned out, we need to remove the PVC. if memberStatus.Conditions.IsTrue(api.ConditionTypeCleanedOut) || memberStatus.Phase == api.MemberPhaseDrain { pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetNamespace()) - if err := pvcs.Delete(context.Background(), memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return pvcs.Delete(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete PVC for member") return errors.WithStack(err) } diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index f0abcbe7d..2d2351d08 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -85,8 +86,8 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter // Remove all finalizers, so it can be removed. log.Warn().Msg("Pod belongs to this deployment, but we don't know the member. Removing all finalizers") kubecli := r.context.GetKubeCli() - ignoreNotFound := false - if err := k8sutil.RemovePodFinalizers(log, kubecli, pod, pod.GetFinalizers(), ignoreNotFound); err != nil { + err := k8sutil.RemovePodFinalizers(ctx, log, kubecli, pod, pod.GetFinalizers(), false) + if err != nil { log.Debug().Err(err).Msg("Failed to update pod (to remove all finalizers)") return errors.WithStack(err) } @@ -322,7 +323,7 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter } // Save status - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return 0, errors.WithStack(err) } diff --git a/pkg/deployment/resources/pod_termination.go b/pkg/deployment/resources/pod_termination.go index 9c624d19b..346d13ca6 100644 --- a/pkg/deployment/resources/pod_termination.go +++ b/pkg/deployment/resources/pod_termination.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -58,7 +59,9 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog // Check node the pod is scheduled on. Only if not in namespaced scope agentDataWillBeGone := false if !r.context.GetScope().IsNamespaced() && p.Spec.NodeName != "" { - node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(context.Background(), p.Spec.NodeName, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(ctxChild, p.Spec.NodeName, metav1.GetOptions{}) if k8sutil.IsNotFound(err) { log.Warn().Msg("Node not found") } else if err != nil { @@ -71,7 +74,9 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog // Check PVC pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(apiObject.GetNamespace()) - pvc, err := pvcs.Get(context.Background(), memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + pvc, err := pvcs.Get(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) if err != nil { log.Warn().Err(err).Msg("Failed to get PVC for member") return errors.WithStack(err) @@ -88,10 +93,11 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog // Inspect agency state log.Debug().Msg("Agent data will be gone, so we will check agency serving status first") - ctx = agency.WithAllowNoLeader(ctx) // The ID we're checking may be the leader, so ignore situations where all other agents are followers - ctx, cancel := context.WithTimeout(ctx, time.Second*15) // Force a quick check + ctxChild, cancel = context.WithTimeout(ctx, time.Second*15) defer cancel() - agencyConns, err := r.context.GetAgencyClients(ctx, func(id string) bool { return id != memberStatus.ID }) + ctxLeader := agency.WithAllowNoLeader(ctxChild) // The ID we're checking may be the leader, so ignore situations where all other agents are followers + + agencyConns, err := r.context.GetAgencyClients(ctxLeader, func(id string) bool { return id != memberStatus.ID }) if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return errors.WithStack(err) @@ -100,7 +106,7 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog log.Debug().Err(err).Msg("No more remaining agents, we cannot delete this one") return errors.WithStack(errors.Newf("No more remaining agents")) } - if err := agency.AreAgentsHealthy(ctx, agencyConns); err != nil { + if err := agency.AreAgentsHealthy(ctxLeader, agencyConns); err != nil { log.Debug().Err(err).Msg("Remaining agents are not healthy") return errors.WithStack(err) } @@ -152,7 +158,9 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol // Check node the pod is scheduled on dbserverDataWillBeGone := false if !r.context.GetScope().IsNamespaced() && p.Spec.NodeName != "" { - node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(context.Background(), p.Spec.NodeName, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(ctxChild, p.Spec.NodeName, metav1.GetOptions{}) if k8sutil.IsNotFound(err) { log.Warn().Msg("Node not found") } else if err != nil { @@ -167,7 +175,9 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol // Check PVC pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(apiObject.GetNamespace()) - pvc, err := pvcs.Get(context.Background(), memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + pvc, err := pvcs.Get(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) if err != nil { log.Warn().Err(err).Msg("Failed to get PVC for member") return errors.WithStack(err) @@ -188,12 +198,16 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol } // Inspect cleaned out state - c, err := r.context.GetDatabaseClient(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + c, err := r.context.GetDatabaseClient(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return errors.WithStack(err) } - cluster, err := c.Cluster(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + cluster, err := c.Cluster(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") @@ -206,7 +220,9 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol } return errors.WithStack(err) } - cleanedOut, err := cluster.IsCleanedOut(ctx, memberStatus.ID) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + cleanedOut, err := cluster.IsCleanedOut(ctxChild, memberStatus.ID) if err != nil { return errors.WithStack(err) } @@ -244,18 +260,21 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol if memberStatus.Phase == api.MemberPhaseCreated { // No cleanout job triggered var jobID string - ctx = driver.WithJobIDResponse(ctx, &jobID) + ctxChild, cancelChild := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancelChild() + + ctxJobID := driver.WithJobIDResponse(ctxChild, &jobID) // Ensure the cleanout is triggered if dbserverDataWillBeGone { log.Debug().Msg("Server is not yet cleaned out. Triggering a clean out now") - if err := cluster.CleanOutServer(ctx, memberStatus.ID); err != nil { + if err := cluster.CleanOutServer(ctxJobID, memberStatus.ID); err != nil { log.Debug().Err(err).Msg("Failed to clean out server") return errors.WithStack(err) } memberStatus.Phase = api.MemberPhaseDrain } else { log.Debug().Msg("Temporary shutdown, resign leadership") - if err := cluster.ResignServer(ctx, memberStatus.ID); err != nil { + if err := cluster.ResignServer(ctxJobID, memberStatus.ID); err != nil { log.Debug().Err(err).Msg("Failed to resign server") return errors.WithStack(err) } @@ -269,12 +288,16 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol } } else if memberStatus.Phase == api.MemberPhaseDrain { // Check the job progress - agency, err := r.context.GetAgency(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + agency, err := r.context.GetAgency(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return errors.WithStack(err) } - jobStatus, err := arangod.CleanoutServerJobStatus(ctx, memberStatus.CleanoutJobID, c, agency) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, memberStatus.CleanoutJobID, c, agency) if err != nil { log.Debug().Err(err).Msg("Failed to fetch job status") return errors.WithStack(err) @@ -296,12 +319,17 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol } } else if memberStatus.Phase == api.MemberPhaseResign { // Check the job progress - agency, err := r.context.GetAgency(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + agency, err := r.context.GetAgency(ctxChild) if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return errors.WithStack(err) } - jobStatus, err := arangod.CleanoutServerJobStatus(ctx, memberStatus.CleanoutJobID, c, agency) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, memberStatus.CleanoutJobID, c, agency) if err != nil { log.Debug().Err(err).Msg("Failed to fetch job status") return errors.WithStack(err) diff --git a/pkg/deployment/resources/pvc_finalizers.go b/pkg/deployment/resources/pvc_finalizers.go index e3cdd3d66..a230aadad 100644 --- a/pkg/deployment/resources/pvc_finalizers.go +++ b/pkg/deployment/resources/pvc_finalizers.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -43,14 +44,15 @@ const ( ) // runPVCFinalizers goes through the list of PVC finalizers to see if they can be removed. -func (r *Resources) runPVCFinalizers(ctx context.Context, p *v1.PersistentVolumeClaim, group api.ServerGroup, memberStatus api.MemberStatus) (util.Interval, error) { +func (r *Resources) runPVCFinalizers(ctx context.Context, p *v1.PersistentVolumeClaim, group api.ServerGroup, + memberStatus api.MemberStatus) (util.Interval, error) { log := r.log.With().Str("pvc-name", p.GetName()).Logger() var removalList []string for _, f := range p.ObjectMeta.GetFinalizers() { switch f { case constants.FinalizerPVCMemberExists: log.Debug().Msg("Inspecting member exists finalizer") - if err := r.inspectFinalizerPVCMemberExists(ctx, log, p, group, memberStatus); err == nil { + if err := r.inspectFinalizerPVCMemberExists(ctx, log, group, memberStatus); err == nil { removalList = append(removalList, f) } else { log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet") @@ -60,8 +62,8 @@ func (r *Resources) runPVCFinalizers(ctx context.Context, p *v1.PersistentVolume // Remove finalizers (if needed) if len(removalList) > 0 { kubecli := r.context.GetKubeCli() - ignoreNotFound := false - if err := k8sutil.RemovePVCFinalizers(log, kubecli, p, removalList, ignoreNotFound); err != nil { + err := k8sutil.RemovePVCFinalizers(ctx, log, kubecli, p, removalList, false) + if err != nil { log.Debug().Err(err).Msg("Failed to update PVC (to remove finalizers)") return 0, errors.WithStack(err) } @@ -74,7 +76,8 @@ func (r *Resources) runPVCFinalizers(ctx context.Context, p *v1.PersistentVolume // inspectFinalizerPVCMemberExists checks the finalizer condition for member-exists. // It returns nil if the finalizer can be removed. -func (r *Resources) inspectFinalizerPVCMemberExists(ctx context.Context, log zerolog.Logger, p *v1.PersistentVolumeClaim, group api.ServerGroup, memberStatus api.MemberStatus) error { +func (r *Resources) inspectFinalizerPVCMemberExists(ctx context.Context, log zerolog.Logger, group api.ServerGroup, + memberStatus api.MemberStatus) error { // Inspect member phase if memberStatus.Phase.IsFailed() { log.Debug().Msg("Member is already failed, safe to remove member-exists finalizer") @@ -105,7 +108,10 @@ func (r *Resources) inspectFinalizerPVCMemberExists(ctx context.Context, log zer if memberStatus.PodName != "" { log.Info().Msg("Removing Pod of member, because PVC is being removed") pods := r.context.GetKubeCli().CoreV1().Pods(apiObject.GetNamespace()) - if err := pods.Delete(context.Background(), memberStatus.PodName, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return pods.Delete(ctxChild, memberStatus.PodName, metav1.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Msg("Failed to delete pod") return errors.WithStack(err) } diff --git a/pkg/deployment/resources/pvc_inspector.go b/pkg/deployment/resources/pvc_inspector.go index 2319f23fc..338f7c6e1 100644 --- a/pkg/deployment/resources/pvc_inspector.go +++ b/pkg/deployment/resources/pvc_inspector.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -70,8 +71,8 @@ func (r *Resources) InspectPVCs(ctx context.Context, cachedStatus inspectorInter // Remove all finalizers, so it can be removed. log.Warn().Msg("PVC belongs to this deployment, but we don't know the member. Removing all finalizers") kubecli := r.context.GetKubeCli() - ignoreNotFound := false - if err := k8sutil.RemovePVCFinalizers(log, kubecli, pvc, pvc.GetFinalizers(), ignoreNotFound); err != nil { + err := k8sutil.RemovePVCFinalizers(ctx, log, kubecli, pvc, pvc.GetFinalizers(), false) + if err != nil { log.Debug().Err(err).Msg("Failed to update PVC (to remove all finalizers)") return errors.WithStack(err) } diff --git a/pkg/deployment/resources/pvcs.go b/pkg/deployment/resources/pvcs.go index f8679f5a5..2ef141630 100644 --- a/pkg/deployment/resources/pvcs.go +++ b/pkg/deployment/resources/pvcs.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,14 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources import ( + "context" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/errors" @@ -36,7 +39,7 @@ func (r *Resources) createPVCFinalizers(group api.ServerGroup) []string { } // EnsurePVCs creates all PVC's listed in member status -func (r *Resources) EnsurePVCs(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsurePVCs(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { kubecli := r.context.GetKubeCli() apiObject := r.context.GetAPIObject() deploymentName := apiObject.GetName() @@ -62,7 +65,10 @@ func (r *Resources) EnsurePVCs(cachedStatus inspectorInterface.Inspector) error resources := spec.Resources vct := spec.VolumeClaimTemplate finalizers := r.createPVCFinalizers(group) - if err := k8sutil.CreatePersistentVolumeClaim(pvcs, m.PersistentVolumeClaimName, deploymentName, ns, storageClassName, role, enforceAntiAffinity, resources, vct, finalizers, owner); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.CreatePersistentVolumeClaim(ctxChild, pvcs, m.PersistentVolumeClaimName, deploymentName, ns, storageClassName, role, enforceAntiAffinity, resources, vct, finalizers, owner) + }) + if err != nil { return errors.WithStack(err) } } diff --git a/pkg/deployment/resources/secret_hashes.go b/pkg/deployment/resources/secret_hashes.go index 36d0e99cd..8d4f602b0 100644 --- a/pkg/deployment/resources/secret_hashes.go +++ b/pkg/deployment/resources/secret_hashes.go @@ -50,7 +50,7 @@ import ( // If a hash is different, the deployment is marked // with a SecretChangedCondition and the operator will not // touch it until this is resolved. -func (r *Resources) ValidateSecretHashes(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) ValidateSecretHashes(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { // validate performs a secret hash comparison for a single secret. // Return true if all is good, false when the SecretChanged condition // must be set. @@ -130,7 +130,7 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspectorInterface.Inspect status.SecretHashes.Users = make(map[string]string) } updater(status.SecretHashes) - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } // Reload status @@ -210,7 +210,7 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspectorInterface.Inspect if status.Conditions.Update(api.ConditionTypeSecretsChanged, true, "Secrets have changed", fmt.Sprintf("Found %d changed secrets", len(badSecretNames))) { log.Warn().Msgf("Found %d changed secrets. Settings SecretsChanged condition", len(badSecretNames)) - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Error().Err(err).Msg("Failed to save SecretsChanged condition") return errors.WithStack(err) } @@ -221,7 +221,7 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspectorInterface.Inspect // All good, we van remove the SecretsChanged condition if status.Conditions.Remove(api.ConditionTypeSecretsChanged) { log.Info().Msg("Resetting SecretsChanged condition") - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Error().Err(err).Msg("Failed to save SecretsChanged condition") return errors.WithStack(err) } diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index 6ff77d19c..93d56cc11 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -72,7 +73,7 @@ func GetCASecretName(apiObject k8sutil.APIObject) string { } // EnsureSecrets creates all secrets needed to run the given deployment -func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureSecrets(ctx context.Context, log zerolog.Logger, cachedStatus inspectorInterface.Inspector) error { start := time.Now() spec := r.context.GetSpec() kubecli := r.context.GetKubeCli() @@ -88,13 +89,13 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInte if spec.IsAuthenticated() { counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureTokenSecret(cachedStatus, secrets, spec.Authentication.GetJWTSecretName())); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureTokenSecret(ctx, cachedStatus, secrets, spec.Authentication.GetJWTSecretName())); err != nil { return errors.WithStack(err) } if imageFound { if pod.VersionHasJWTSecretKeyfolder(image.ArangoDBVersion, image.Enterprise) { - if err := r.ensureTokenSecretFolder(cachedStatus, secrets, spec.Authentication.GetJWTSecretName(), pod.JWTSecretFolder(deploymentName)); err != nil { + if err := r.ensureTokenSecretFolder(ctx, cachedStatus, secrets, spec.Authentication.GetJWTSecretName(), pod.JWTSecretFolder(deploymentName)); err != nil { return errors.WithStack(err) } } @@ -102,11 +103,11 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInte if spec.Metrics.IsEnabled() { if imageFound && pod.VersionHasJWTSecretKeyfolder(image.ArangoDBVersion, image.Enterprise) { - if err := r.refreshCache(cachedStatus, r.ensureExporterTokenSecret(cachedStatus, secrets, spec.Metrics.GetJWTTokenSecretName(), pod.JWTSecretFolder(deploymentName))); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureExporterTokenSecret(ctx, cachedStatus, secrets, spec.Metrics.GetJWTTokenSecretName(), pod.JWTSecretFolder(deploymentName))); err != nil { return errors.WithStack(err) } } else { - if err := r.refreshCache(cachedStatus, r.ensureExporterTokenSecret(cachedStatus, secrets, spec.Metrics.GetJWTTokenSecretName(), spec.Authentication.GetJWTSecretName())); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureExporterTokenSecret(ctx, cachedStatus, secrets, spec.Metrics.GetJWTTokenSecretName(), spec.Authentication.GetJWTSecretName())); err != nil { return errors.WithStack(err) } } @@ -114,11 +115,11 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInte } if spec.IsSecure() { counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureTLSCACertificateSecret(cachedStatus, secrets, spec.TLS)); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureTLSCACertificateSecret(ctx, cachedStatus, secrets, spec.TLS)); err != nil { return errors.WithStack(err) } - if err := r.refreshCache(cachedStatus, r.ensureSecretWithEmptyKey(cachedStatus, secrets, GetCASecretName(r.context.GetAPIObject()), "empty")); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureSecretWithEmptyKey(ctx, cachedStatus, secrets, GetCASecretName(r.context.GetAPIObject()), "empty")); err != nil { return errors.WithStack(err) } @@ -162,11 +163,12 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInte serverNames = append(serverNames, ip) } owner := member.AsOwner() - if err := r.refreshCache(cachedStatus, createTLSServerCertificate(log, secrets, serverNames, spec.TLS, tlsKeyfileSecretName, &owner)); err != nil && !k8sutil.IsAlreadyExists(err) { + errCert := createTLSServerCertificate(ctx, log, secrets, serverNames, spec.TLS, tlsKeyfileSecretName, &owner) + if err := r.refreshCache(ctx, cachedStatus, errCert); err != nil && !k8sutil.IsAlreadyExists(err) { return errors.WithStack(errors.Wrapf(err, "Failed to create TLS keyfile secret")) } - if err := r.refreshCache(cachedStatus, operatorErrors.Reconcile()); err != nil { + if err := r.refreshCache(ctx, cachedStatus, operatorErrors.Reconcile()); err != nil { return errors.WithStack(err) } } @@ -178,39 +180,40 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInte } if spec.RocksDB.IsEncrypted() { if i := status.CurrentImage; i != nil && features.EncryptionRotation().Supported(i.ArangoDBVersion, i.Enterprise) { - if err := r.refreshCache(cachedStatus, r.ensureEncryptionKeyfolderSecret(cachedStatus, secrets, spec.RocksDB.Encryption.GetKeySecretName(), pod.GetEncryptionFolderSecretName(deploymentName))); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureEncryptionKeyfolderSecret(ctx, cachedStatus, secrets, spec.RocksDB.Encryption.GetKeySecretName(), pod.GetEncryptionFolderSecretName(deploymentName))); err != nil { return errors.WithStack(err) } } } if spec.Sync.IsEnabled() { counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureTokenSecret(cachedStatus, secrets, spec.Sync.Authentication.GetJWTSecretName())); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureTokenSecret(ctx, cachedStatus, secrets, spec.Sync.Authentication.GetJWTSecretName())); err != nil { return errors.WithStack(err) } counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureTokenSecret(cachedStatus, secrets, spec.Sync.Monitoring.GetTokenSecretName())); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureTokenSecret(ctx, cachedStatus, secrets, spec.Sync.Monitoring.GetTokenSecretName())); err != nil { return errors.WithStack(err) } counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureTLSCACertificateSecret(cachedStatus, secrets, spec.Sync.TLS)); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureTLSCACertificateSecret(ctx, cachedStatus, secrets, spec.Sync.TLS)); err != nil { return errors.WithStack(err) } counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureClientAuthCACertificateSecret(cachedStatus, secrets, spec.Sync.Authentication)); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureClientAuthCACertificateSecret(ctx, cachedStatus, secrets, spec.Sync.Authentication)); err != nil { return errors.WithStack(err) } } return nil } -func (r *Resources) refreshCache(cachedStatus inspectorInterface.Inspector, err error) error { +func (r *Resources) refreshCache(ctx context.Context, cachedStatus inspectorInterface.Inspector, err error) error { if err == nil { return nil } if operatorErrors.IsReconcile(err) { - if err := cachedStatus.Refresh(r.context.GetKubeCli(), r.context.GetMonitoringV1Cli(), r.context.GetArangoCli(), r.context.GetNamespace()); err != nil { + err := cachedStatus.Refresh(ctx, r.context.GetKubeCli(), r.context.GetMonitoringV1Cli(), r.context.GetArangoCli(), r.context.GetNamespace()) + if err != nil { return errors.WithStack(err) } } else { @@ -220,7 +223,7 @@ func (r *Resources) refreshCache(cachedStatus inspectorInterface.Inspector, err return nil } -func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, folderSecretName string) error { +func (r *Resources) ensureTokenSecretFolder(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, folderSecretName string) error { if f, exists := cachedStatus.Secret(folderSecretName); exists { if len(f.Data) == 0 { s, exists := cachedStatus.Secret(secretName) @@ -237,7 +240,11 @@ func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Insp f.Data[pod.ActiveJWTKey] = token f.Data[constants.SecretKeyToken] = token - if _, err := secrets.Update(context.Background(), f, meta.UpdateOptions{}); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Update(ctxChild, f, meta.UpdateOptions{}) + return err + }) + if err != nil { return err } @@ -258,7 +265,11 @@ func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Insp return err } - if _, err := secrets.Patch(context.Background(), folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}); err != nil { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Patch(ctxChild, folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}) + return err + }) + if err != nil { return err } } @@ -277,7 +288,11 @@ func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Insp return err } - if _, err := secrets.Patch(context.Background(), folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}); err != nil { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Patch(ctxChild, folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}) + return err + }) + if err != nil { return err } } @@ -295,7 +310,7 @@ func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Insp return errors.Newf("Token secret is invalid") } - if err := r.createSecretWithMod(secrets, folderSecretName, func(s *core.Secret) { + if err := r.createSecretWithMod(ctx, secrets, folderSecretName, func(s *core.Secret) { s.Data[util.SHA256(token)] = token s.Data[pod.ActiveJWTKey] = token s.Data[constants.SecretKeyToken] = token @@ -306,9 +321,9 @@ func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Insp return nil } -func (r *Resources) ensureTokenSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName string) error { +func (r *Resources) ensureTokenSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName string) error { if _, exists := cachedStatus.Secret(secretName); !exists { - return r.createTokenSecret(secrets, secretName) + return r.createTokenSecret(ctx, secrets, secretName) } return nil @@ -340,23 +355,23 @@ func (r *Resources) createSecret(secrets k8sutil.SecretInterface, secretName str return operatorErrors.Reconcile() } -func (r *Resources) ensureSecretWithEmptyKey(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string) error { +func (r *Resources) ensureSecretWithEmptyKey(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string) error { if _, exists := cachedStatus.Secret(secretName); !exists { - return r.createSecretWithKey(secrets, secretName, keyName, nil) + return r.createSecretWithKey(ctx, secrets, secretName, keyName, nil) } return nil } -func (r *Resources) ensureSecretWithKey(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error { +func (r *Resources) ensureSecretWithKey(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error { if _, exists := cachedStatus.Secret(secretName); !exists { - return r.createSecretWithKey(secrets, secretName, keyName, value) + return r.createSecretWithKey(ctx, secrets, secretName, keyName, value) } return nil } -func (r *Resources) createSecretWithMod(secrets k8sutil.SecretInterface, secretName string, f func(s *core.Secret)) error { +func (r *Resources) createSecretWithMod(ctx context.Context, secrets k8sutil.SecretInterface, secretName string, f func(s *core.Secret)) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ @@ -370,7 +385,11 @@ func (r *Resources) createSecretWithMod(secrets k8sutil.SecretInterface, secretN f(secret) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Create(ctxChild, secret, meta.CreateOptions{}) + return err + }) + if err != nil { // Failed to create secret return errors.WithStack(err) } @@ -378,20 +397,23 @@ func (r *Resources) createSecretWithMod(secrets k8sutil.SecretInterface, secretN return operatorErrors.Reconcile() } -func (r *Resources) createSecretWithKey(secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error { - return r.createSecretWithMod(secrets, secretName, func(s *core.Secret) { +func (r *Resources) createSecretWithKey(ctx context.Context, secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error { + return r.createSecretWithMod(ctx, secrets, secretName, func(s *core.Secret) { s.Data[keyName] = value }) } -func (r *Resources) createTokenSecret(secrets k8sutil.SecretInterface, secretName string) error { +func (r *Resources) createTokenSecret(ctx context.Context, secrets k8sutil.SecretInterface, secretName string) error { tokenData := make([]byte, 32) rand.Read(tokenData) token := hex.EncodeToString(tokenData) // Create secret owner := r.context.GetAPIObject().AsOwner() - if err := k8sutil.CreateTokenSecret(secrets, secretName, token, &owner); k8sutil.IsAlreadyExists(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.CreateTokenSecret(ctxChild, secrets, secretName, token, &owner) + }) + if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -402,7 +424,7 @@ func (r *Resources) createTokenSecret(secrets k8sutil.SecretInterface, secretNam return operatorErrors.Reconcile() } -func (r *Resources) ensureEncryptionKeyfolderSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, keyfileSecretName, secretName string) error { +func (r *Resources) ensureEncryptionKeyfolderSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, keyfileSecretName, secretName string) error { _, folderExists := cachedStatus.Secret(secretName) keyfile, exists := cachedStatus.Secret(keyfileSecretName) @@ -429,13 +451,17 @@ func (r *Resources) ensureEncryptionKeyfolderSecret(cachedStatus inspectorInterf } owner := r.context.GetAPIObject().AsOwner() - if err := AppendKeyfileToKeyfolder(cachedStatus, secrets, &owner, secretName, d); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return AppendKeyfileToKeyfolder(ctxChild, cachedStatus, secrets, &owner, secretName, d) + }) + if err != nil { return errors.Wrapf(err, "Unable to create keyfolder secret") } return nil } -func AppendKeyfileToKeyfolder(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, ownerRef *meta.OwnerReference, secretName string, encryptionKey []byte) error { +func AppendKeyfileToKeyfolder(ctx context.Context, cachedStatus inspectorInterface.Inspector, + secrets k8sutil.SecretInterface, ownerRef *meta.OwnerReference, secretName string, encryptionKey []byte) error { encSha := fmt.Sprintf("%0x", sha256.Sum256(encryptionKey)) if _, exists := cachedStatus.Secret(secretName); !exists { @@ -450,7 +476,7 @@ func AppendKeyfileToKeyfolder(cachedStatus inspectorInterface.Inspector, secrets } // Attach secret to owner k8sutil.AddOwnerRefToObject(secret, ownerRef) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + if _, err := secrets.Create(ctx, secret, meta.CreateOptions{}); err != nil { // Failed to create secret return errors.WithStack(err) } @@ -471,14 +497,16 @@ var ( // ensureExporterTokenSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with correct access. -func (r *Resources) ensureExporterTokenSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, tokenSecretName, secretSecretName string) error { +func (r *Resources) ensureExporterTokenSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, + secrets k8sutil.SecretInterface, tokenSecretName, secretSecretName string) error { if update, exists, err := r.ensureExporterTokenSecretCreateRequired(cachedStatus, tokenSecretName, secretSecretName); err != nil { return err } else if update { // Create secret if !exists { owner := r.context.GetAPIObject().AsOwner() - if err := k8sutil.CreateJWTFromSecret(secrets, tokenSecretName, secretSecretName, exporterTokenClaims, &owner); k8sutil.IsAlreadyExists(err) { + err = k8sutil.CreateJWTFromSecret(ctx, secrets, tokenSecretName, secretSecretName, exporterTokenClaims, &owner) + if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -531,13 +559,16 @@ func (r *Resources) ensureExporterTokenSecretCreateRequired(cachedStatus inspect // ensureTLSCACertificateSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a generated CA certificate. -func (r *Resources) ensureTLSCACertificateSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.TLSSpec) error { +func (r *Resources) ensureTLSCACertificateSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.TLSSpec) error { if _, exists := cachedStatus.Secret(spec.GetCASecretName()); !exists { // Secret not found, create it apiObject := r.context.GetAPIObject() owner := apiObject.AsOwner() deploymentName := apiObject.GetName() - if err := createTLSCACertificate(r.log, secrets, spec, deploymentName, &owner); k8sutil.IsAlreadyExists(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return createTLSCACertificate(ctxChild, r.log, secrets, spec, deploymentName, &owner) + }) + if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -552,7 +583,7 @@ func (r *Resources) ensureTLSCACertificateSecret(cachedStatus inspectorInterface // ensureTLSCACertificateSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a generated CA certificate. -func (r *Resources) ensureTLSCAFolderSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.TLSSpec, folderSecretName string) error { +func (r *Resources) ensureTLSCAFolderSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.TLSSpec, folderSecretName string) error { if spec.CASecretName == nil { return errors.Newf("CA Secret Name is nil") } @@ -580,7 +611,7 @@ func (r *Resources) ensureTLSCAFolderSecret(cachedStatus inspectorInterface.Insp certSha := util.SHA256(caData) // Secret not found, create it - return r.createSecretWithMod(secrets, folderSecretName, func(s *core.Secret) { + return r.createSecretWithMod(ctx, secrets, folderSecretName, func(s *core.Secret) { s.Data[certSha] = caData }) } @@ -589,13 +620,16 @@ func (r *Resources) ensureTLSCAFolderSecret(cachedStatus inspectorInterface.Insp // ensureClientAuthCACertificateSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a generated CA certificate. -func (r *Resources) ensureClientAuthCACertificateSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.SyncAuthenticationSpec) error { +func (r *Resources) ensureClientAuthCACertificateSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.SyncAuthenticationSpec) error { if _, exists := cachedStatus.Secret(spec.GetClientCASecretName()); !exists { // Secret not found, create it apiObject := r.context.GetAPIObject() owner := apiObject.AsOwner() deploymentName := apiObject.GetName() - if err := createClientAuthCACertificate(r.log, secrets, spec, deploymentName, &owner); k8sutil.IsAlreadyExists(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return createClientAuthCACertificate(ctxChild, r.log, secrets, spec, deploymentName, &owner) + }) + if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -617,7 +651,7 @@ func (r *Resources) getJWTSecret(spec api.DeploymentSpec) (string, error) { ns := r.context.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := spec.Authentication.GetJWTSecretName() - s, err := k8sutil.GetTokenSecret(secrets, secretName) + s, err := k8sutil.GetTokenSecret(context.TODO(), secrets, secretName) if err != nil { r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get JWT secret") return "", errors.WithStack(err) @@ -631,7 +665,7 @@ func (r *Resources) getSyncJWTSecret(spec api.DeploymentSpec) (string, error) { ns := r.context.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := spec.Sync.Authentication.GetJWTSecretName() - s, err := k8sutil.GetTokenSecret(secrets, secretName) + s, err := k8sutil.GetTokenSecret(context.TODO(), secrets, secretName) if err != nil { r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync JWT secret") return "", errors.WithStack(err) @@ -645,7 +679,7 @@ func (r *Resources) getSyncMonitoringToken(spec api.DeploymentSpec) (string, err ns := r.context.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := spec.Sync.Monitoring.GetTokenSecretName() - s, err := k8sutil.GetTokenSecret(secrets, secretName) + s, err := k8sutil.GetTokenSecret(context.TODO(), secrets, secretName) if err != nil { r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync monitoring secret") return "", errors.WithStack(err) diff --git a/pkg/deployment/resources/servicemonitor.go b/pkg/deployment/resources/servicemonitor.go index 0ec771671..7ab22965a 100644 --- a/pkg/deployment/resources/servicemonitor.go +++ b/pkg/deployment/resources/servicemonitor.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2019 ArangoDB Inc, Cologne, Germany +// Copyright 2019-2021 ArangoDB Inc, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Max Neunhoeffer +// Author Tomasz Mielech // package resources @@ -138,7 +139,7 @@ func (r *Resources) serviceMonitorSpec() (coreosv1.ServiceMonitorSpec, error) { } // EnsureServiceMonitor creates or updates a ServiceMonitor. -func (r *Resources) EnsureServiceMonitor() error { +func (r *Resources) EnsureServiceMonitor(ctx context.Context) error { // Some preparations: log := r.log apiObject := r.context.GetAPIObject() @@ -157,7 +158,9 @@ func (r *Resources) EnsureServiceMonitor() error { // Check if ServiceMonitor already exists serviceMonitors := mClient.ServiceMonitors(ns) - servMon, err := serviceMonitors.Get(context.Background(), serviceMonitorName, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + servMon, err := serviceMonitors.Get(ctxChild, serviceMonitorName, metav1.GetOptions{}) if err != nil { if k8sutil.IsNotFound(err) { if !wantMetrics { @@ -178,7 +181,11 @@ func (r *Resources) EnsureServiceMonitor() error { }, Spec: spec, } - smon, err = serviceMonitors.Create(context.Background(), smon, metav1.CreateOptions{}) + + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := serviceMonitors.Create(ctxChild, smon, metav1.CreateOptions{}) + return err + }) if err != nil { log.Error().Err(err).Msgf("Failed to create ServiceMonitor %s", serviceMonitorName) return errors.WithStack(err) @@ -220,7 +227,10 @@ func (r *Resources) EnsureServiceMonitor() error { servMon.Spec = spec - _, err = serviceMonitors.Update(context.Background(), servMon, metav1.UpdateOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := serviceMonitors.Update(ctxChild, servMon, metav1.UpdateOptions{}) + return err + }) if err != nil { return err } @@ -228,7 +238,9 @@ func (r *Resources) EnsureServiceMonitor() error { return nil } // Need to get rid of the ServiceMonitor: - err = serviceMonitors.Delete(context.Background(), serviceMonitorName, metav1.DeleteOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return serviceMonitors.Delete(ctxChild, serviceMonitorName, metav1.DeleteOptions{}) + }) if err == nil { log.Debug().Msgf("Deleted ServiceMonitor %s", serviceMonitorName) return nil diff --git a/pkg/deployment/resources/services.go b/pkg/deployment/resources/services.go index c5e33d83f..c11e77149 100644 --- a/pkg/deployment/resources/services.go +++ b/pkg/deployment/resources/services.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -49,7 +50,7 @@ var ( ) // EnsureServices creates all services needed to service the deployment -func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureServices(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { log := r.log start := time.Now() kubecli := r.context.GetKubeCli() @@ -99,7 +100,11 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er }, } - if _, err := svcs.Create(context.Background(), s, metav1.CreateOptions{}); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := svcs.Create(ctxChild, s, metav1.CreateOptions{}) + return err + }) + if err != nil { if !k8sutil.IsConflict(err) { return err } @@ -124,7 +129,11 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er if !equality.Semantic.DeepDerivative(*spec, s.Spec) { s.Spec = *spec - if _, err := svcs.Update(context.Background(), s, metav1.UpdateOptions{}); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := svcs.Update(ctxChild, s, metav1.UpdateOptions{}) + return err + }) + if err != nil { return err } @@ -141,7 +150,9 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er // Headless service counterMetric.Inc() if _, exists := cachedStatus.Service(k8sutil.CreateHeadlessServiceName(deploymentName)); !exists { - svcName, newlyCreated, err := k8sutil.CreateHeadlessService(svcs, apiObject, owner) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + svcName, newlyCreated, err := k8sutil.CreateHeadlessService(ctxChild, svcs, apiObject, owner) if err != nil { log.Debug().Err(err).Msg("Failed to create headless service") return errors.WithStack(err) @@ -155,7 +166,9 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er single := spec.GetMode().HasSingleServers() counterMetric.Inc() if _, exists := cachedStatus.Service(k8sutil.CreateDatabaseClientServiceName(deploymentName)); !exists { - svcName, newlyCreated, err := k8sutil.CreateDatabaseClientService(svcs, apiObject, single, owner) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + svcName, newlyCreated, err := k8sutil.CreateDatabaseClientService(ctxChild, svcs, apiObject, single, owner) if err != nil { log.Debug().Err(err).Msg("Failed to create database client service") return errors.WithStack(err) @@ -167,7 +180,7 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er status, lastVersion := r.context.GetStatus() if status.ServiceName != svcName { status.ServiceName = svcName - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } } @@ -180,7 +193,7 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er if single { role = "single" } - if err := r.ensureExternalAccessServices(cachedStatus, svcs, eaServiceName, ns, role, "database", k8sutil.ArangoPort, false, spec.ExternalAccess, apiObject, log, counterMetric); err != nil { + if err := r.ensureExternalAccessServices(ctx, cachedStatus, svcs, eaServiceName, ns, role, "database", k8sutil.ArangoPort, false, spec.ExternalAccess, apiObject, log, counterMetric); err != nil { return errors.WithStack(err) } @@ -189,20 +202,22 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er counterMetric.Inc() eaServiceName := k8sutil.CreateSyncMasterClientServiceName(deploymentName) role := "syncmaster" - if err := r.ensureExternalAccessServices(cachedStatus, svcs, eaServiceName, ns, role, "sync", k8sutil.ArangoSyncMasterPort, true, spec.Sync.ExternalAccess.ExternalAccessSpec, apiObject, log, counterMetric); err != nil { + if err := r.ensureExternalAccessServices(ctx, cachedStatus, svcs, eaServiceName, ns, role, "sync", k8sutil.ArangoSyncMasterPort, true, spec.Sync.ExternalAccess.ExternalAccessSpec, apiObject, log, counterMetric); err != nil { return errors.WithStack(err) } status, lastVersion := r.context.GetStatus() if status.SyncServiceName != eaServiceName { status.SyncServiceName = eaServiceName - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } } } if spec.Metrics.IsEnabled() { - name, _, err := k8sutil.CreateExporterService(cachedStatus, svcs, apiObject, apiObject.AsOwner()) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + name, _, err := k8sutil.CreateExporterService(ctxChild, cachedStatus, svcs, apiObject, apiObject.AsOwner()) if err != nil { log.Debug().Err(err).Msgf("Failed to create %s exporter service", name) return errors.WithStack(err) @@ -210,7 +225,7 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er status, lastVersion := r.context.GetStatus() if status.ExporterServiceName != name { status.ExporterServiceName = name - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } } @@ -219,7 +234,7 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er } // EnsureServices creates all services needed to service the deployment -func (r *Resources) ensureExternalAccessServices(cachedStatus inspectorInterface.Inspector, svcs k8sutil.ServiceInterface, eaServiceName, ns, svcRole, title string, port int, noneIsClusterIP bool, spec api.ExternalAccessSpec, apiObject k8sutil.APIObject, log zerolog.Logger, counterMetric prometheus.Counter) error { +func (r *Resources) ensureExternalAccessServices(ctx context.Context, cachedStatus inspectorInterface.Inspector, svcs k8sutil.ServiceInterface, eaServiceName, ns, svcRole, title string, port int, noneIsClusterIP bool, spec api.ExternalAccessSpec, apiObject k8sutil.APIObject, log zerolog.Logger, counterMetric prometheus.Counter) error { // Database external access service createExternalAccessService := false deleteExternalAccessService := false @@ -275,7 +290,11 @@ func (r *Resources) ensureExternalAccessServices(cachedStatus inspectorInterface } } if updateExternalAccessService && !createExternalAccessService && !deleteExternalAccessService { - if _, err := svcs.Update(context.Background(), existing, metav1.UpdateOptions{}); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := svcs.Update(ctxChild, existing, metav1.UpdateOptions{}) + return err + }) + if err != nil { log.Debug().Err(err).Msgf("Failed to update %s external access service", title) return errors.WithStack(err) } @@ -289,7 +308,10 @@ func (r *Resources) ensureExternalAccessServices(cachedStatus inspectorInterface if deleteExternalAccessService { log.Info().Str("service", eaServiceName).Msgf("Removing obsolete %s external access service", title) - if err := svcs.Delete(context.Background(), eaServiceName, metav1.DeleteOptions{}); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return svcs.Delete(ctxChild, eaServiceName, metav1.DeleteOptions{}) + }) + if err != nil { log.Debug().Err(err).Msgf("Failed to remove %s external access service", title) return errors.WithStack(err) } @@ -299,7 +321,9 @@ func (r *Resources) ensureExternalAccessServices(cachedStatus inspectorInterface nodePort := spec.GetNodePort() loadBalancerIP := spec.GetLoadBalancerIP() loadBalancerSourceRanges := spec.LoadBalancerSourceRanges - _, newlyCreated, err := k8sutil.CreateExternalAccessService(svcs, eaServiceName, svcRole, apiObject, eaServiceType, port, nodePort, loadBalancerIP, loadBalancerSourceRanges, apiObject.AsOwner()) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + _, newlyCreated, err := k8sutil.CreateExternalAccessService(ctxChild, svcs, eaServiceName, svcRole, apiObject, eaServiceType, port, nodePort, loadBalancerIP, loadBalancerSourceRanges, apiObject.AsOwner()) if err != nil { log.Debug().Err(err).Msgf("Failed to create %s external access service", title) return errors.WithStack(err) diff --git a/pkg/replication/server_endpoint_api.go b/pkg/replication/server_endpoint_api.go index 24fb937d1..6242bc7ac 100644 --- a/pkg/replication/server_endpoint_api.go +++ b/pkg/replication/server_endpoint_api.go @@ -23,6 +23,8 @@ package replication import ( + "context" + api "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) @@ -58,7 +60,7 @@ func (ep serverEndpoint) AuthUserSecretName() string { func (ep serverEndpoint) TLSCACert() string { tlsCASecretName := ep.getSpec().TLS.GetCASecretName() secrets := ep.dr.deps.KubeCli.CoreV1().Secrets(ep.dr.apiObject.GetNamespace()) - caCert, err := k8sutil.GetCACertficateSecret(secrets, tlsCASecretName) + caCert, err := k8sutil.GetCACertficateSecret(context.TODO(), secrets, tlsCASecretName) if err != nil { return "" } diff --git a/pkg/replication/sync_client.go b/pkg/replication/sync_client.go index d5dfc1bf5..fbfa07f1c 100644 --- a/pkg/replication/sync_client.go +++ b/pkg/replication/sync_client.go @@ -67,7 +67,7 @@ func (dr *DeploymentReplication) createSyncMasterClient(epSpec api.EndpointSpec) } } else if authJWTSecretName != "" { var err error - jwtSecret, err = k8sutil.GetTokenSecret(secrets, authJWTSecretName) + jwtSecret, err = k8sutil.GetTokenSecret(context.TODO(), secrets, authJWTSecretName) if err != nil { return nil, errors.WithStack(err) } @@ -86,7 +86,7 @@ func (dr *DeploymentReplication) createSyncMasterClient(epSpec api.EndpointSpec) } } if tlsCASecretName != "" { - caCert, err := k8sutil.GetCACertficateSecret(secrets, tlsCASecretName) + caCert, err := k8sutil.GetCACertficateSecret(context.TODO(), secrets, tlsCASecretName) if err != nil { return nil, errors.WithStack(err) } @@ -141,7 +141,7 @@ func (dr *DeploymentReplication) createArangoSyncTLSAuthentication(spec api.Depl } // Fetch TLS CA certificate for source - caCert, err := k8sutil.GetCACertficateSecret(secrets, tlsCASecretName) + caCert, err := k8sutil.GetCACertficateSecret(context.TODO(), secrets, tlsCASecretName) if err != nil { return client.TLSAuthentication{}, errors.WithStack(err) } diff --git a/pkg/util/arangod/cleanout_server.go b/pkg/util/arangod/cleanout_server.go index 0111843f0..cf98e7bf0 100644 --- a/pkg/util/arangod/cleanout_server.go +++ b/pkg/util/arangod/cleanout_server.go @@ -28,7 +28,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/errors" - driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/agency" ) diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index bc6790843..be179a9ab 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package arangod @@ -49,6 +50,36 @@ type ( requireAuthenticationKey struct{} ) +type TimeoutRunFunc k8sutil.TimeoutRunFunc + +const ( + minArangoDDefaultTimeout = time.Second * 10 +) + +var requestTimeout = minArangoDDefaultTimeout + +// GetRequestTimeout gets request timeout for one call to kubernetes. +func GetRequestTimeout() time.Duration { + return requestTimeout +} + +// RunWithTimeout runs the function with the provided timeout or with default timeout. +func RunWithTimeout(ctx context.Context, run TimeoutRunFunc, timeout ...time.Duration) error { + t := GetRequestTimeout() + if len(timeout) > 0 { + t = timeout[0] + } + + return k8sutil.RunWithTimeout(ctx, k8sutil.TimeoutRunFunc(run), t) +} + +// SetRequestTimeout sets request timeout for one call to kubernetes. +func SetRequestTimeout(timeout time.Duration) { + if timeout > minArangoDDefaultTimeout { + requestTimeout = timeout + } +} + // WithSkipAuthentication prepares a context that when given to functions in // this file will avoid creating any authentication for arango clients. func WithSkipAuthentication(ctx context.Context) context.Context { @@ -258,7 +289,9 @@ func createArangodClientAuthentication(ctx context.Context, cli corev1.CoreV1Int // Should we skip using it? if ctx.Value(skipAuthenticationKey{}) == nil { secrets := cli.Secrets(apiObject.GetNamespace()) - s, err := k8sutil.GetTokenSecret(secrets, apiObject.Spec.Authentication.GetJWTSecretName()) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + s, err := k8sutil.GetTokenSecret(ctxChild, secrets, apiObject.Spec.Authentication.GetJWTSecretName()) if err != nil { return nil, errors.WithStack(err) } diff --git a/pkg/util/arangod/dbserver.go b/pkg/util/arangod/dbserver.go index 6e422132a..559ed4932 100644 --- a/pkg/util/arangod/dbserver.go +++ b/pkg/util/arangod/dbserver.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package arangod @@ -36,16 +37,28 @@ import ( // The functions returns an error when the check could not be completed or the dbserver // is not empty, or nil when the dbserver is found to be empty. func IsDBServerEmpty(ctx context.Context, id string, client driver.Client) error { - c, err := client.Cluster(ctx) + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + c, err := client.Cluster(ctxChild) if err != nil { return errors.WithStack(errors.Wrapf(err, "Cannot obtain Cluster")) } - dbs, err := client.Databases(ctx) + + ctxChild, cancel = context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + dbs, err := client.Databases(ctxChild) if err != nil { return errors.WithStack(errors.Wrapf(err, "Cannot fetch databases")) } + + var inventory driver.DatabaseInventory for _, db := range dbs { - inventory, err := c.DatabaseInventory(ctx, db) + err := RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + inventory, err = c.DatabaseInventory(ctxChild, db) + + return err + }) if err != nil { return errors.WithStack(errors.Wrapf(err, "Cannot fetch inventory for %s", db.Name())) } diff --git a/pkg/util/k8sutil/finalizers.go b/pkg/util/k8sutil/finalizers.go index bc88d7c08..4f956b273 100644 --- a/pkg/util/k8sutil/finalizers.go +++ b/pkg/util/k8sutil/finalizers.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil @@ -25,11 +26,12 @@ package k8sutil import ( "context" - "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/rs/zerolog" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + + "github.com/arangodb/kube-arangodb/pkg/util/errors" ) const ( @@ -37,10 +39,14 @@ const ( ) // RemovePodFinalizers removes the given finalizers from the given pod. -func RemovePodFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1.Pod, finalizers []string, ignoreNotFound bool) error { +func RemovePodFinalizers(ctx context.Context, log zerolog.Logger, kubecli kubernetes.Interface, p *v1.Pod, + finalizers []string, ignoreNotFound bool) error { pods := kubecli.CoreV1().Pods(p.GetNamespace()) getFunc := func() (metav1.Object, error) { - result, err := pods.Get(context.Background(), p.GetName(), metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + + result, err := pods.Get(ctxChild, p.GetName(), metav1.GetOptions{}) if err != nil { return nil, errors.WithStack(err) } @@ -48,7 +54,10 @@ func RemovePodFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1 } updateFunc := func(updated metav1.Object) error { updatedPod := updated.(*v1.Pod) - result, err := pods.Update(context.Background(), updatedPod, metav1.UpdateOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + + result, err := pods.Update(ctxChild, updatedPod, metav1.UpdateOptions{}) if err != nil { return errors.WithStack(err) } @@ -62,10 +71,14 @@ func RemovePodFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1 } // RemovePVCFinalizers removes the given finalizers from the given PVC. -func RemovePVCFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1.PersistentVolumeClaim, finalizers []string, ignoreNotFound bool) error { +func RemovePVCFinalizers(ctx context.Context, log zerolog.Logger, kubecli kubernetes.Interface, + p *v1.PersistentVolumeClaim, finalizers []string, ignoreNotFound bool) error { pvcs := kubecli.CoreV1().PersistentVolumeClaims(p.GetNamespace()) getFunc := func() (metav1.Object, error) { - result, err := pvcs.Get(context.Background(), p.GetName(), metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + + result, err := pvcs.Get(ctxChild, p.GetName(), metav1.GetOptions{}) if err != nil { return nil, errors.WithStack(err) } @@ -73,7 +86,10 @@ func RemovePVCFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1 } updateFunc := func(updated metav1.Object) error { updatedPVC := updated.(*v1.PersistentVolumeClaim) - result, err := pvcs.Update(context.Background(), updatedPVC, metav1.UpdateOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + + result, err := pvcs.Update(ctxChild, updatedPVC, metav1.UpdateOptions{}) if err != nil { return errors.WithStack(err) } diff --git a/pkg/util/k8sutil/inspector/inspector.go b/pkg/util/k8sutil/inspector/inspector.go index a111d78d5..5f11f3fe7 100644 --- a/pkg/util/k8sutil/inspector/inspector.go +++ b/pkg/util/k8sutil/inspector/inspector.go @@ -18,11 +18,14 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector import ( + "context" + "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/arangomember" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/persistentvolumeclaim" @@ -37,7 +40,8 @@ import ( ) type Inspector interface { - Refresh(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, c versioned.Interface, namespace string) error + Refresh(ctx context.Context, k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, + c versioned.Interface, namespace string) error pod.Inspector secret.Inspector diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 37da41bdb..49a1cd366 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil @@ -406,10 +407,10 @@ func GetPodSpecChecksum(podSpec core.PodSpec) (string, error) { // CreatePod adds an owner to the given pod and calls the k8s api-server to created it. // If the pod already exists, nil is returned. // If another error occurs, that error is returned. -func CreatePod(kubecli kubernetes.Interface, pod *core.Pod, ns string, owner metav1.OwnerReference) (types.UID, error) { +func CreatePod(ctx context.Context, kubecli kubernetes.Interface, pod *core.Pod, ns string, owner metav1.OwnerReference) (types.UID, error) { AddOwnerRefToObject(pod.GetObjectMeta(), &owner) - if pod, err := kubecli.CoreV1().Pods(ns).Create(context.Background(), pod, metav1.CreateOptions{}); err != nil && !IsAlreadyExists(err) { + if pod, err := kubecli.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil && !IsAlreadyExists(err) { return "", errors.WithStack(err) } else { return pod.UID, nil diff --git a/pkg/util/k8sutil/pvc.go b/pkg/util/k8sutil/pvc.go index c203ff93d..e8be8949a 100644 --- a/pkg/util/k8sutil/pvc.go +++ b/pkg/util/k8sutil/pvc.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil @@ -84,7 +85,7 @@ func ExtractStorageResourceRequirement(resources v1.ResourceRequirements) v1.Res // CreatePersistentVolumeClaim creates a persistent volume claim with given name and configuration. // If the pvc already exists, nil is returned. // If another error occurs, that error is returned. -func CreatePersistentVolumeClaim(pvcs PersistentVolumeClaimInterface, pvcName, deploymentName, ns, storageClassName, role string, enforceAntiAffinity bool, resources v1.ResourceRequirements, vct *v1.PersistentVolumeClaim, finalizers []string, owner metav1.OwnerReference) error { +func CreatePersistentVolumeClaim(ctx context.Context, pvcs PersistentVolumeClaimInterface, pvcName, deploymentName, ns, storageClassName, role string, enforceAntiAffinity bool, resources v1.ResourceRequirements, vct *v1.PersistentVolumeClaim, finalizers []string, owner metav1.OwnerReference) error { labels := LabelsForDeployment(deploymentName, role) volumeMode := v1.PersistentVolumeFilesystem pvc := &v1.PersistentVolumeClaim{ @@ -113,7 +114,7 @@ func CreatePersistentVolumeClaim(pvcs PersistentVolumeClaimInterface, pvcName, d pvc.Spec.StorageClassName = &storageClassName } AddOwnerRefToObject(pvc.GetObjectMeta(), &owner) - if _, err := pvcs.Create(context.Background(), pvc, metav1.CreateOptions{}); err != nil && !IsAlreadyExists(err) { + if _, err := pvcs.Create(ctx, pvc, metav1.CreateOptions{}); err != nil && !IsAlreadyExists(err) { return errors.WithStack(err) } return nil diff --git a/pkg/util/k8sutil/secrets.go b/pkg/util/k8sutil/secrets.go index 1bb219a70..de5b73171 100644 --- a/pkg/util/k8sutil/secrets.go +++ b/pkg/util/k8sutil/secrets.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil @@ -88,8 +89,8 @@ func CreateEncryptionKeySecret(secrets SecretInterface, secretName string, key [ // ValidateCACertificateSecret checks that a secret with given name in given namespace // exists and it contains a 'ca.crt' data field. -func ValidateCACertificateSecret(secrets SecretInterface, secretName string) error { - s, err := secrets.Get(context.Background(), secretName, meta.GetOptions{}) +func ValidateCACertificateSecret(ctx context.Context, secrets SecretInterface, secretName string) error { + s, err := secrets.Get(ctx, secretName, meta.GetOptions{}) if err != nil { return errors.WithStack(err) } @@ -106,8 +107,11 @@ func ValidateCACertificateSecret(secrets SecretInterface, secretName string) err // If the secret does not exists the field is missing, // an error is returned. // Returns: certificate, error -func GetCACertficateSecret(secrets SecretInterface, secretName string) (string, error) { - s, err := secrets.Get(context.Background(), secretName, meta.GetOptions{}) +func GetCACertficateSecret(ctx context.Context, secrets SecretInterface, secretName string) (string, error) { + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + + s, err := secrets.Get(ctxChild, secretName, meta.GetOptions{}) if err != nil { return "", errors.WithStack(err) } @@ -124,8 +128,9 @@ func GetCACertficateSecret(secrets SecretInterface, secretName string) (string, // If the secret does not exists or one of the fields is missing, // an error is returned. // Returns: certificate, private-key, isOwnedByDeployment, error -func GetCASecret(secrets SecretInterface, secretName string, ownerRef *meta.OwnerReference) (string, string, bool, error) { - s, err := secrets.Get(context.Background(), secretName, meta.GetOptions{}) +func GetCASecret(ctx context.Context, secrets SecretInterface, secretName string, + ownerRef *meta.OwnerReference) (string, string, bool, error) { + s, err := secrets.Get(ctx, secretName, meta.GetOptions{}) if err != nil { return "", "", false, errors.WithStack(err) } @@ -155,7 +160,8 @@ func GetCAFromSecret(s *core.Secret, ownerRef *meta.OwnerReference) (string, str } // CreateCASecret creates a secret used to store a PEM encoded CA certificate & private key. -func CreateCASecret(secrets SecretInterface, secretName string, certificate, key string, ownerRef *meta.OwnerReference) error { +func CreateCASecret(ctx context.Context, secrets SecretInterface, secretName string, certificate, key string, + ownerRef *meta.OwnerReference) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ @@ -168,7 +174,7 @@ func CreateCASecret(secrets SecretInterface, secretName string, certificate, key } // Attach secret to owner AddOwnerRefToObject(secret, ownerRef) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + if _, err := secrets.Create(ctx, secret, meta.CreateOptions{}); err != nil { // Failed to create secret return errors.WithStack(err) } @@ -197,7 +203,8 @@ func GetTLSKeyfileFromSecret(s *core.Secret) (string, error) { // CreateTLSKeyfileSecret creates a secret used to store a PEM encoded keyfile // in the format ArangoDB accepts it for its `--ssl.keyfile` option. -func CreateTLSKeyfileSecret(secrets SecretInterface, secretName string, keyfile string, ownerRef *meta.OwnerReference) error { +func CreateTLSKeyfileSecret(ctx context.Context, secrets SecretInterface, secretName string, keyfile string, + ownerRef *meta.OwnerReference) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ @@ -209,7 +216,7 @@ func CreateTLSKeyfileSecret(secrets SecretInterface, secretName string, keyfile } // Attach secret to owner AddOwnerRefToObject(secret, ownerRef) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + if _, err := secrets.Create(ctx, secret, meta.CreateOptions{}); err != nil { // Failed to create secret return errors.WithStack(err) } @@ -218,8 +225,8 @@ func CreateTLSKeyfileSecret(secrets SecretInterface, secretName string, keyfile // ValidateTokenSecret checks that a secret with given name in given namespace // exists and it contains a 'token' data field. -func ValidateTokenSecret(secrets SecretInterface, secretName string) error { - s, err := secrets.Get(context.Background(), secretName, meta.GetOptions{}) +func ValidateTokenSecret(ctx context.Context, secrets SecretInterface, secretName string) error { + s, err := secrets.Get(ctx, secretName, meta.GetOptions{}) if err != nil { return errors.WithStack(err) } @@ -236,8 +243,8 @@ func ValidateTokenFromSecret(s *core.Secret) error { } // GetTokenSecret loads the token secret from a Secret with given name. -func GetTokenSecret(secrets SecretInterface, secretName string) (string, error) { - s, err := secrets.Get(context.Background(), secretName, meta.GetOptions{}) +func GetTokenSecret(ctx context.Context, secrets SecretInterface, secretName string) (string, error) { + s, err := secrets.Get(ctx, secretName, meta.GetOptions{}) if err != nil { return "", errors.WithStack(err) } @@ -256,7 +263,8 @@ func GetTokenFromSecret(s *core.Secret) (string, error) { // CreateTokenSecret creates a secret with given name in given namespace // with a given token as value. -func CreateTokenSecret(secrets SecretInterface, secretName, token string, ownerRef *meta.OwnerReference) error { +func CreateTokenSecret(ctx context.Context, secrets SecretInterface, secretName, token string, + ownerRef *meta.OwnerReference) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ @@ -268,7 +276,7 @@ func CreateTokenSecret(secrets SecretInterface, secretName, token string, ownerR } // Attach secret to owner AddOwnerRefToObject(secret, ownerRef) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + if _, err := secrets.Create(ctx, secret, meta.CreateOptions{}); err != nil { // Failed to create secret return errors.WithStack(err) } @@ -292,8 +300,10 @@ func CreateJWTTokenFromSecret(secret string, claims map[string]interface{}) (str // CreateJWTFromSecret creates a JWT using the secret stored in secretSecretName and stores the // result in a new secret called tokenSecretName -func CreateJWTFromSecret(secrets SecretInterface, tokenSecretName, secretSecretName string, claims map[string]interface{}, ownerRef *meta.OwnerReference) error { - secret, err := GetTokenSecret(secrets, secretSecretName) +func CreateJWTFromSecret(ctx context.Context, secrets SecretInterface, tokenSecretName, secretSecretName string, claims map[string]interface{}, ownerRef *meta.OwnerReference) error { + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + secret, err := GetTokenSecret(ctxChild, secrets, secretSecretName) if err != nil { return errors.WithStack(err) } @@ -307,12 +317,15 @@ func CreateJWTFromSecret(secrets SecretInterface, tokenSecretName, secretSecretN return errors.WithStack(err) } - return CreateTokenSecret(secrets, tokenSecretName, signedToken, ownerRef) + return RunWithTimeout(ctx, func(ctxChild context.Context) error { + return CreateTokenSecret(ctxChild, secrets, tokenSecretName, signedToken, ownerRef) + }) } // CreateBasicAuthSecret creates a secret with given name in given namespace // with a given username and password as value. -func CreateBasicAuthSecret(secrets SecretInterface, secretName, username, password string, ownerRef *meta.OwnerReference) error { +func CreateBasicAuthSecret(ctx context.Context, secrets SecretInterface, secretName, username, password string, + ownerRef *meta.OwnerReference) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ @@ -325,7 +338,11 @@ func CreateBasicAuthSecret(secrets SecretInterface, secretName, username, passwo } // Attach secret to owner AddOwnerRefToObject(secret, ownerRef) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + err := RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Create(ctxChild, secret, meta.CreateOptions{}) + return err + }) + if err != nil { // Failed to create secret return errors.WithStack(err) } diff --git a/pkg/util/k8sutil/services.go b/pkg/util/k8sutil/services.go index ddd1f1ab2..ee8e379ae 100644 --- a/pkg/util/k8sutil/services.go +++ b/pkg/util/k8sutil/services.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil @@ -76,7 +77,8 @@ func CreateExporterClientServiceName(deploymentName string) string { } // CreateExporterService -func CreateExporterService(cachedStatus service.Inspector, svcs ServiceInterface, deployment metav1.Object, owner metav1.OwnerReference) (string, bool, error) { +func CreateExporterService(ctx context.Context, cachedStatus service.Inspector, svcs ServiceInterface, + deployment metav1.Object, owner metav1.OwnerReference) (string, bool, error) { deploymentName := deployment.GetName() svcName := CreateExporterClientServiceName(deploymentName) @@ -104,7 +106,7 @@ func CreateExporterService(cachedStatus service.Inspector, svcs ServiceInterface }, } AddOwnerRefToObject(svc.GetObjectMeta(), &owner) - if _, err := svcs.Create(context.Background(), svc, metav1.CreateOptions{}); IsAlreadyExists(err) { + if _, err := svcs.Create(ctx, svc, metav1.CreateOptions{}); IsAlreadyExists(err) { return svcName, false, nil } else if err != nil { return svcName, false, errors.WithStack(err) @@ -117,7 +119,8 @@ func CreateExporterService(cachedStatus service.Inspector, svcs ServiceInterface // If the service already exists, nil is returned. // If another error occurs, that error is returned. // The returned bool is true if the service is created, or false when the service already existed. -func CreateHeadlessService(svcs ServiceInterface, deployment metav1.Object, owner metav1.OwnerReference) (string, bool, error) { +func CreateHeadlessService(ctx context.Context, svcs ServiceInterface, deployment metav1.Object, + owner metav1.OwnerReference) (string, bool, error) { deploymentName := deployment.GetName() svcName := CreateHeadlessServiceName(deploymentName) ports := []core.ServicePort{ @@ -129,7 +132,7 @@ func CreateHeadlessService(svcs ServiceInterface, deployment metav1.Object, owne } publishNotReadyAddresses := true serviceType := core.ServiceTypeClusterIP - newlyCreated, err := createService(svcs, svcName, deploymentName, deployment.GetNamespace(), ClusterIPNone, "", serviceType, ports, "", nil, publishNotReadyAddresses, owner) + newlyCreated, err := createService(ctx, svcs, svcName, deploymentName, deployment.GetNamespace(), ClusterIPNone, "", serviceType, ports, "", nil, publishNotReadyAddresses, owner) if err != nil { return "", false, errors.WithStack(err) } @@ -140,7 +143,8 @@ func CreateHeadlessService(svcs ServiceInterface, deployment metav1.Object, owne // If the service already exists, nil is returned. // If another error occurs, that error is returned. // The returned bool is true if the service is created, or false when the service already existed. -func CreateDatabaseClientService(svcs ServiceInterface, deployment metav1.Object, single bool, owner metav1.OwnerReference) (string, bool, error) { +func CreateDatabaseClientService(ctx context.Context, svcs ServiceInterface, deployment metav1.Object, single bool, + owner metav1.OwnerReference) (string, bool, error) { deploymentName := deployment.GetName() svcName := CreateDatabaseClientServiceName(deploymentName) ports := []core.ServicePort{ @@ -158,7 +162,7 @@ func CreateDatabaseClientService(svcs ServiceInterface, deployment metav1.Object } serviceType := core.ServiceTypeClusterIP publishNotReadyAddresses := false - newlyCreated, err := createService(svcs, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, "", nil, publishNotReadyAddresses, owner) + newlyCreated, err := createService(ctx, svcs, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, "", nil, publishNotReadyAddresses, owner) if err != nil { return "", false, errors.WithStack(err) } @@ -169,7 +173,9 @@ func CreateDatabaseClientService(svcs ServiceInterface, deployment metav1.Object // If the service already exists, nil is returned. // If another error occurs, that error is returned. // The returned bool is true if the service is created, or false when the service already existed. -func CreateExternalAccessService(svcs ServiceInterface, svcName, role string, deployment metav1.Object, serviceType core.ServiceType, port, nodePort int, loadBalancerIP string, loadBalancerSourceRanges []string, owner metav1.OwnerReference) (string, bool, error) { +func CreateExternalAccessService(ctx context.Context, svcs ServiceInterface, svcName, role string, + deployment metav1.Object, serviceType core.ServiceType, port, nodePort int, loadBalancerIP string, + loadBalancerSourceRanges []string, owner metav1.OwnerReference) (string, bool, error) { deploymentName := deployment.GetName() ports := []core.ServicePort{ core.ServicePort{ @@ -180,7 +186,7 @@ func CreateExternalAccessService(svcs ServiceInterface, svcName, role string, de }, } publishNotReadyAddresses := false - newlyCreated, err := createService(svcs, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, loadBalancerIP, loadBalancerSourceRanges, publishNotReadyAddresses, owner) + newlyCreated, err := createService(ctx, svcs, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, loadBalancerIP, loadBalancerSourceRanges, publishNotReadyAddresses, owner) if err != nil { return "", false, errors.WithStack(err) } @@ -191,8 +197,9 @@ func CreateExternalAccessService(svcs ServiceInterface, svcName, role string, de // If the service already exists, nil is returned. // If another error occurs, that error is returned. // The returned bool is true if the service is created, or false when the service already existed. -func createService(svcs ServiceInterface, svcName, deploymentName, ns, clusterIP, role string, serviceType core.ServiceType, - ports []core.ServicePort, loadBalancerIP string, loadBalancerSourceRanges []string, publishNotReadyAddresses bool, owner metav1.OwnerReference) (bool, error) { +func createService(ctx context.Context, svcs ServiceInterface, svcName, deploymentName, ns, clusterIP, role string, + serviceType core.ServiceType, ports []core.ServicePort, loadBalancerIP string, loadBalancerSourceRanges []string, + publishNotReadyAddresses bool, owner metav1.OwnerReference) (bool, error) { labels := LabelsForDeployment(deploymentName, role) svc := &core.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -211,7 +218,7 @@ func createService(svcs ServiceInterface, svcName, deploymentName, ns, clusterIP }, } AddOwnerRefToObject(svc.GetObjectMeta(), &owner) - if _, err := svcs.Create(context.Background(), svc, metav1.CreateOptions{}); IsAlreadyExists(err) { + if _, err := svcs.Create(ctx, svc, metav1.CreateOptions{}); IsAlreadyExists(err) { return false, nil } else if err != nil { return false, errors.WithStack(err) diff --git a/pkg/util/k8sutil/util.go b/pkg/util/k8sutil/util.go index f9891154a..6ba1aa826 100644 --- a/pkg/util/k8sutil/util.go +++ b/pkg/util/k8sutil/util.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,15 +18,21 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil import ( + "context" + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ) +type TimeoutRunFunc func(ctxChild context.Context) error + const ( // LabelKeyArangoDeployment is the key of the label used to store the ArangoDeployment name in LabelKeyArangoDeployment = "arango_deployment" @@ -43,8 +49,38 @@ const ( // AppName is the fixed value for the "app" label AppName = "arangodb" + + // minDefaultRequestTimeout is minimum default request timeout to k8s. + minDefaultRequestTimeout = time.Second * 3 ) +var requestTimeout = minDefaultRequestTimeout + +// GetRequestTimeout gets request timeout for one call to kubernetes. +func GetRequestTimeout() time.Duration { + return requestTimeout +} + +// RunWithTimeout runs the function with the provided timeout or with default timeout. +func RunWithTimeout(ctx context.Context, run TimeoutRunFunc, timeout ...time.Duration) error { + t := GetRequestTimeout() + if len(timeout) > 0 { + t = timeout[0] + } + + ctxChild, cancel := context.WithTimeout(ctx, t) + defer cancel() + + return run(ctxChild) +} + +// SetRequestTimeout sets request timeout for one call to kubernetes. +func SetRequestTimeout(timeout time.Duration) { + if timeout > minDefaultRequestTimeout { + requestTimeout = timeout + } +} + // AddOwnerRefToObject adds given owner reference to given object func AddOwnerRefToObject(obj metav1.Object, ownerRef *metav1.OwnerReference) { if ownerRef != nil { diff --git a/tests/auth_test.go b/tests/auth_test.go index 5aa6c9fc7..b6390bbbb 100644 --- a/tests/auth_test.go +++ b/tests/auth_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package tests @@ -102,7 +103,7 @@ func TestAuthenticationSingleCustomSecret(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create secret - if err := k8sutil.CreateTokenSecret(secrets, depl.Spec.Authentication.GetJWTSecretName(), "foo", nil); err != nil { + if err := k8sutil.CreateTokenSecret(context.Background(), secrets, depl.Spec.Authentication.GetJWTSecretName(), "foo", nil); err != nil { t.Fatalf("Create JWT secret failed: %v", err) } defer removeSecret(kubecli, depl.Spec.Authentication.GetJWTSecretName(), ns) @@ -152,7 +153,7 @@ func TestAuthenticationNoneSingle(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create deployment - apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl,metav1.CreateOptions{}) + apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl, metav1.CreateOptions{}) if err != nil { t.Fatalf("Create deployment failed: %v", err) } @@ -240,12 +241,12 @@ func TestAuthenticationClusterCustomSecret(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create secret - if err := k8sutil.CreateTokenSecret(secrets, depl.Spec.Authentication.GetJWTSecretName(), "foo", nil); err != nil { + if err := k8sutil.CreateTokenSecret(context.Background(), secrets, depl.Spec.Authentication.GetJWTSecretName(), "foo", nil); err != nil { t.Fatalf("Create JWT secret failed: %v", err) } // Create deployment - apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl,metav1.CreateOptions{}) + apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl, metav1.CreateOptions{}) if err != nil { t.Fatalf("Create deployment failed: %v", err) } @@ -292,7 +293,7 @@ func TestAuthenticationNoneCluster(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create deployment - apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl,metav1.CreateOptions{}) + apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl, metav1.CreateOptions{}) if err != nil { t.Fatalf("Create deployment failed: %v", err) } diff --git a/tests/sync/main.go b/tests/sync/main.go index ceb6c47ae..8aa5df8fb 100644 --- a/tests/sync/main.go +++ b/tests/sync/main.go @@ -143,7 +143,7 @@ func newArangoSyncTestJob(ns, name string) *batchv1.Job { func waitForSyncDeploymentReady(ctx context.Context, ns, name string, kubecli kubernetes.Interface, c versioned.Interface) error { return retry.Retry(func() error { - deployment, err := c.DatabaseV1().ArangoDeployments(ns).Get(name, metav1.GetOptions{}) + deployment, err := c.DatabaseV1().ArangoDeployments(ns).Get(ctx, name, metav1.GetOptions{}) if err != nil { return err } @@ -420,7 +420,7 @@ func mustNewArangoDBSyncClient(ctx context.Context, kubecli kubernetes.Interface ns := deployment.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := deployment.Spec.Sync.Authentication.GetJWTSecretName() - jwtSecret, err := k8sutil.GetTokenSecret(secrets, secretName) + jwtSecret, err := k8sutil.GetTokenSecret(ctx, secrets, secretName) if err != nil { return nil, err } diff --git a/tests/test_util.go b/tests/test_util.go index 964c65d94..6ce3a6167 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package tests @@ -217,7 +218,7 @@ func mustNewArangoSyncClient(ctx context.Context, kubecli kubernetes.Interface, ns := apiObject.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := apiObject.Spec.Sync.Authentication.GetJWTSecretName() - jwtToken, err := k8sutil.GetTokenSecret(secrets, secretName) + jwtToken, err := k8sutil.GetTokenSecret(ctx, secrets, secretName) if err != nil { t.Fatalf("Failed to get sync jwt secret '%s': %s", secretName, err) }