From cf4dcd76cea0afd07b175b278c6a17c166e71e60 Mon Sep 17 00:00:00 2001 From: Slach Date: Sun, 4 Feb 2024 23:01:49 +0400 Subject: [PATCH] improve re-balance disk during download if disk not exists in system.disks. Use least used for `local` disks and `random` for object disks, fix https://github.com/Altinity/clickhouse-backup/issues/561 --- ChangeLog.md | 4 + go.mod | 3 +- go.sum | 122 +-------- pkg/backup/download.go | 206 ++++++++++++-- pkg/backup/download_test.go | 270 +++++++++++++++++++ pkg/backup/list.go | 9 +- pkg/backup/restore.go | 64 ++--- pkg/clickhouse/clickhouse.go | 72 +++-- pkg/clickhouse/clickhouse_test.go | 17 +- pkg/clickhouse/structs.go | 10 +- pkg/filesystemhelper/filesystemhelper.go | 41 +-- pkg/metadata/metadata.go | 10 +- pkg/storage/object_disk/object_disk.go | 5 +- test/integration/custom_entrypoint.sh | 12 + test/integration/docker-compose_advanced.yml | 8 + test/integration/dynamic_settings.sh | 33 ++- test/integration/integration_test.go | 40 ++- 17 files changed, 685 insertions(+), 241 deletions(-) create mode 100644 pkg/backup/download_test.go create mode 100755 test/integration/custom_entrypoint.sh diff --git a/ChangeLog.md b/ChangeLog.md index 5095de89..9e57be27 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,7 @@ +# v2.5.0 +IMPROVEMENTS +- improve re-balance disk during download if disk not exists in system.disks. Use least used for `local` disks and `random` for object disks, fix [561](https://github.com/Altinity/clickhouse-backup/issues/561) + # v2.4.25 BUG FIXES - fix `--restore-table-mapping` corner cases for when destination database contains special characters , fix [820](https://github.com/Altinity/clickhouse-backup/issues/820) diff --git a/go.mod b/go.mod index f7eeaea3..f719e1a4 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/pkg/sftp v1.13.6 github.com/prometheus/client_golang v1.18.0 github.com/puzpuzpuz/xsync v1.5.2 + github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285 github.com/stretchr/testify v1.8.4 github.com/tencentyun/cos-go-sdk-v5 v0.7.45 github.com/urfave/cli v1.22.14 @@ -101,7 +102,6 @@ require ( github.com/kr/fs v0.1.0 // indirect github.com/mattn/go-ieproxy v0.0.11 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mozillazg/go-httpheader v0.4.0 // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect @@ -129,7 +129,6 @@ require ( golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect diff --git a/go.sum b/go.sum index 043fb3c8..be98430c 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM= -cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU= cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -24,8 +22,6 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w= -cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= cloud.google.com/go/storage v1.37.0 h1:WI8CsaFO8Q9KjPVtsZ5Cmi0dXV25zMoX0FklT7c3Jm4= cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -53,18 +49,11 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/ch-go v0.61.0 h1:22JYeFJoFNAU/Vod4etAeUEY28cYt7Ixnwqj1+EUfro= -github.com/ClickHouse/ch-go v0.61.0/go.mod h1:POJBl0MxEMS91Zd0uTgDDt05KfXEjf5KIwW6lNhje9Y= github.com/ClickHouse/ch-go v0.61.1 h1:j5rx3qnvcnYjhnP1IdXE/vdIRQiqgwAzyqOaasA6QCw= github.com/ClickHouse/ch-go v0.61.1/go.mod h1:myxt/JZgy2BYHFGQqzmaIpbfr5CMbs3YHVULaWQj5YU= -github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= -github.com/ClickHouse/clickhouse-go/v2 v2.16.0 h1:rhMfnPewXPnY4Q4lQRGdYuTLRBRKJEIEYHtbUMrzmvI= -github.com/ClickHouse/clickhouse-go/v2 v2.16.0/go.mod h1:J7SPfIxwR+x4mQ+o8MLSe0oY50NNntEqCIjFe/T1VPM= github.com/ClickHouse/clickhouse-go/v2 v2.17.1 h1:ZCmAYWpu75IyEi7+Yrs/uaAjiCGY5wfW5kXo64exkX4= github.com/ClickHouse/clickhouse-go/v2 v2.17.1/go.mod h1:rkGTvFDTLqLIm0ma+13xmcCfr/08Gvs7KmFt1tgiWHQ= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= -github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= -github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0= @@ -78,80 +67,42 @@ github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDw github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v1.23.5 h1:xK6C4udTyDMd82RFvNkDQxtAd00xlzFUtX4fF2nMZyg= -github.com/aws/aws-sdk-go-v2 v1.23.5/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds= github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 h1:Zx9+31KyB8wQna6SXFWOewlgoY5uGdDAu6PTOEU3OQI= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3/go.mod h1:zxbEJhRdKTH1nqS2qu6UJ7zGe25xaHxZXaC2CvuQFnA= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= -github.com/aws/aws-sdk-go-v2/config v1.25.11 h1:RWzp7jhPRliIcACefGkKp03L0Yofmd2p8M25kbiyvno= -github.com/aws/aws-sdk-go-v2/config v1.25.11/go.mod h1:BVUs0chMdygHsQtvaMyEOpW2GIW+ubrxJLgIz/JU29s= github.com/aws/aws-sdk-go-v2/config v1.26.6 h1:Z/7w9bUqlRI0FFQpetVuFYEsjzE3h7fpU6HuGmfPL/o= github.com/aws/aws-sdk-go-v2/config v1.26.6/go.mod h1:uKU6cnDmYCvJ+pxO9S4cWDb2yWWIH5hra+32hVh1MI4= -github.com/aws/aws-sdk-go-v2/credentials v1.16.9 h1:LQo3MUIOzod9JdUK+wxmSdgzLVYUbII3jXn3S/HJZU0= -github.com/aws/aws-sdk-go-v2/credentials v1.16.9/go.mod h1:R7mDuIJoCjH6TxGUc/cylE7Lp/o0bhKVoxdBThsjqCM= github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 h1:FZVFahMyZle6WcogZCOxo6D/lkDA2lqKIn4/ueUmVXw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9/go.mod h1:kjq7REMIkxdtcEC9/4BVXjOsNY5isz6jQbEgk6osRTU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4 h1:TUCNKBd4/JEefsZDxo5deRmrRRPZHqGyBYiUAeBKOWU= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.4/go.mod h1:egDkcl+zsgFqS6VO142bKboip5Pe1sNMwN55Xy38QsM= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15 h1:2MUXyGW6dVaQz6aqycpbdLIH1NMcUI6kW6vQ0RabGYg= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15/go.mod h1:aHbhbR6WEQgHAiRj41EQ2W47yOYwNtIkWTXmcAtYqj8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 h1:8GVZIR0y6JRIUNSYI1xAMF4HDfV8H/bOsZ/8AD/uY5Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8/go.mod h1:rwBfu0SoUkBUZndVgPZKAD9Y2JigaZtRP68unRiYToQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3CDCFNvd9ir5hMjlVStLZWrvM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 h1:n3GDfwqF2tzEkXlv5cuy4iy7LpKDtqDMcNLfZDu9rls= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 h1:abKT+RuM1sdCNZIGIfZpLkvxEX3Rpsto019XG/rkYG8= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8/go.mod h1:Owc4ysUE71JSruVTTa3h4f2pp3E4hlcAtmeNXxDmjj4= github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 h1:5oE2WzJE56/mVveuDZPJESKlg/00AaS2pY2QZcnxg4M= github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10/go.mod h1:FHbKWQtRBYUz4vO5WBWjzMD2by126ny5y/1EoaWoLfI= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 h1:xyfOAYV/ujzZOo01H9+OnyeiRKmTEp6EsITTsmq332Q= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8/go.mod h1:coLeQEoKzW9ViTL2bn0YUlU7K0RYjivKudG74gtd+sI= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 h1:L0ai8WICYHozIKK+OtPzVJBugL7culcuM4E4JOpIEm8= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10/go.mod h1:byqfyxJBshFk0fF9YmK0M0ugIO8OWjzH2T3bPG4eGuA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 h1:EamsKe+ZjkOQjDdHd86/JCEucjFKQ9T0atWKO4s2Lgs= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8/go.mod h1:Q0vV3/csTpbkfKLI5Sb56cJQTCTtJ0ixdb7P+Wedqiw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 h1:ip5ia3JOXl4OAsqeTdrOOmqKgoWiu+t9XSOnRzBwmRs= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8/go.mod h1:kE+aERnK9VQIw1vrk7ElAvhCsgLNzGyCPNg2Qe4Eq4c= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 h1:KOxnQeWy5sXyS37fdKEvAsGHOr9fa/qvwxfJurR/BzE= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10/go.mod h1:jMx5INQFYFYB3lQD9W0D8Ohgq6Wnl7NYOJ2TQndbulI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2 h1:DLSAG8zpJV2pYsU+UPkj1IEZghyBnnUsvIRs6UuXSDU= -github.com/aws/aws-sdk-go-v2/service/s3 v1.47.2/go.mod h1:thjZng67jGsvMyVZnSxlcqKyLwB0XTG8bHIRZPTJ+Bs= github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1 h1:5XNlsBsEvBZBMO6p82y+sqpWg8j5aBCe+5C2GBFgqBQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1/go.mod h1:4qXHrG1Ne3VGIMZPCB8OjH/pLFO94sKABIusjh0KWPU= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.2 h1:xJPydhNm0Hiqct5TVKEuHG7weC0+sOs4MUnd7A5n5F4= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.2/go.mod h1:zxk6y1X2KXThESWMS5CrKRvISD8mbIMab6nZrCGxDG0= github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2 h1:8dU9zqA77C5egbU6yd4hFLaiIdPv3rU+6cp7sz5FjCU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.2/go.mod h1:7Lt5mjQ8x5rVdKqg+sKKDeuwoszDJIIPmkd8BVsEdS0= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.2 h1:fFrLsy08wEbAisqW3KDl/cPHrF43GmV79zXB9EwJiZw= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.2/go.mod h1:7Ld9eTqocTvJqqJ5K/orbSDwmGcpRdlDiLjz2DO+SL8= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= -github.com/aws/smithy-go v1.18.1 h1:pOdBTUfXNazOlxLrgeYalVnuTpKreACHtc62xLwIB3c= -github.com/aws/smithy-go v1.18.1/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= @@ -159,8 +110,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= -github.com/bodgit/sevenzip v1.4.4 h1:cI/PfeqDXESY6mwBn7Q9LyxlVkHaOPyaDev3mR7qzSA= -github.com/bodgit/sevenzip v1.4.4/go.mod h1:G/5VOeQBGI2TsdyF88xpszj2SRvvD6srHdq9gRBQ9Is= github.com/bodgit/sevenzip v1.4.5 h1:HFJQ+nbjppfyf2xbQEJBbmVo+o2kTg1FXV4i7YOx87s= github.com/bodgit/sevenzip v1.4.5/go.mod h1:LAcAg/UQzyjzCQSGBPZFYzoiHMfT6Gk+3tMSjUk3foY= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= @@ -175,6 +124,8 @@ github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 h1:7To3pQ+pZo0i3dsWEbinPNFs5gPSBOsJtx3wTT94VBY= +github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -190,14 +141,14 @@ github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmW github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= -github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-resiliency v1.5.0 h1:dRsaR00whmQD+SgVKlq/vCRFNgtEb5yppyeVos3Yce0= github.com/eapache/go-resiliency v1.5.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -208,8 +159,6 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= -github.com/go-faster/errors v0.7.0 h1:UnD/xusnfUgtEYkgRZohqL2AfmPTwv13NAJwwFFaNYc= -github.com/go-faster/errors v0.7.0/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -218,8 +167,6 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -288,10 +235,6 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= @@ -359,8 +302,6 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM= github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A= @@ -379,13 +320,9 @@ github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= -github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s= -github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/orb v0.11.0 h1:JfVXJUBeH9ifc/OrhBY0lL16QsmPgpCHMlqSSYhcgAA= github.com/paulmach/orb v0.11.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -395,21 +332,19 @@ github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/puzpuzpuz/xsync v1.5.2 h1:yRAP4wqSOZG+/4pxJ08fPTwrfL0IzE/LKQ/cw509qGY= github.com/puzpuzpuz/xsync v1.5.2/go.mod h1:K98BYhX3k1dQ2M63t1YNVDanbwUPmBCAhNmVrrxfiGg= +github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285 h1:d54EL9l+XteliUfUCGsEwwuk65dmmxX85VXF+9T6+50= +github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285/go.mod h1:fxIDly1xtudczrZeOOlfaUvd2OPb2qZAPuWdU2BsBTk= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -477,26 +412,16 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= @@ -514,8 +439,6 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -545,8 +468,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -574,8 +495,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -583,8 +502,6 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -596,8 +513,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -623,17 +538,14 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -691,8 +603,6 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.152.0 h1:t0r1vPnfMc260S2Ci+en7kfCZaLOPs5KI0sVV/6jZrY= -google.golang.org/api v0.152.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY= google.golang.org/api v0.157.0 h1:ORAeqmbrrozeyw5NjnMxh7peHO0UzV4wWYSwZeCUb20= google.golang.org/api v0.157.0/go.mod h1:+z4v4ufbZ1WEpld6yMGHyggs+PmAHiaLNj5ytP3N01g= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -716,22 +626,10 @@ google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4 h1:W12Pwm4urIbRdGhMEg2NM9O3TWKjNcxQhs46V0ypf/k= -google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4/go.mod h1:5RBcpGRxr25RbDzY5w+dmaqpSEvl8Gwl1x2CICf60ic= -google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= -google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= -google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 h1:ZcOkrmX74HbKFYnpPY8Qsw93fC29TbJXspYKaBkSXDQ= -google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4/go.mod h1:k2dtGpRrbsSyKcNPKKI5sstZkrNCZwpU/ns96JoHbGg= -google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 h1:KHBtwE+eQc3+NxpjmRFlQ3pJQ2FNnhhgB9xOV8kyBuU= -google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -743,10 +641,6 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= -google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -761,8 +655,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/backup/download.go b/pkg/backup/download.go index 866c87aa..505d2220 100644 --- a/pkg/backup/download.go +++ b/pkg/backup/download.go @@ -15,6 +15,7 @@ import ( "github.com/eapache/go-resiliency/retrier" "io" "io/fs" + "math/rand" "os" "path" "path/filepath" @@ -210,13 +211,8 @@ func (b *Backuper) Download(backupName string, tablePattern string, partitions [ return fmt.Errorf("one of Download Metadata go-routine return error: %v", err) } if !schemaOnly { - for _, t := range tableMetadataAfterDownload { - for disk := range t.Parts { - if _, diskExists := b.DiskToPathMap[disk]; !diskExists && disk != b.cfg.ClickHouse.EmbeddedBackupDisk { - b.DiskToPathMap[disk] = b.DiskToPathMap["default"] - log.Warnf("table '%s.%s' require disk '%s' that not found in clickhouse table system.disks, you can add nonexistent disks to `disk_mapping` in `clickhouse` config section, data will download to %s", t.Database, t.Table, disk, b.DiskToPathMap["default"]) - } - } + if reBalanceErr := b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload, disks, remoteBackup, log); reBalanceErr != nil { + return reBalanceErr } log.Debugf("prepare table SHADOW concurrent semaphore with concurrency=%d len(tableMetadataAfterDownload)=%d", b.cfg.General.DownloadConcurrency, len(tableMetadataAfterDownload)) dataGroup, dataCtx := errgroup.WithContext(ctx) @@ -304,6 +300,103 @@ func (b *Backuper) Download(backupName string, tablePattern string, partitions [ return nil } +func (b *Backuper) reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload []metadata.TableMetadata, disks []clickhouse.Disk, remoteBackup storage.Backup, log *apexLog.Entry) error { + var disksByStoragePolicyAndType map[string]map[string][]clickhouse.Disk + filterDisksByTypeAndStoragePolicies := func(disk string, diskType string, disks []clickhouse.Disk, remoteBackup storage.Backup, t metadata.TableMetadata) (string, []clickhouse.Disk, error) { + _, ok := remoteBackup.DiskTypes[disk] + if !ok { + return "", nil, fmt.Errorf("disk: %s not found in disk_types section %#v in %s/metadata.json", disk, remoteBackup.DiskTypes, remoteBackup.BackupName) + } + storagePolicy := b.ch.ExtractStoragePolicy(t.Query) + if len(disksByStoragePolicyAndType) == 0 { + disksByStoragePolicyAndType = b.splitDisksByTypeAndStoragePolicy(disks) + } + if _, isTypeExists := disksByStoragePolicyAndType[diskType]; !isTypeExists { + return "", nil, fmt.Errorf("disk: %s, diskType: %s not found in system.disks", disk, diskType) + } + filteredDisks, isPolicyExists := disksByStoragePolicyAndType[diskType][storagePolicy] + if !isPolicyExists || len(filteredDisks) == 0 { + return "", nil, fmt.Errorf("storagePolicy: %s with diskType: %s not found in system.disks", storagePolicy, diskType) + } + return storagePolicy, filteredDisks, nil + } + + updateDiskFreeSize := func(downloadDisk, diskType, storagePolicy string, newFreeSpace uint64) { + for dIdx := range disksByStoragePolicyAndType[diskType][storagePolicy] { + if disksByStoragePolicyAndType[diskType][storagePolicy][dIdx].Name == downloadDisk { + disksByStoragePolicyAndType[diskType][storagePolicy][dIdx].FreeSpace = newFreeSpace + } + } + } + + for i, t := range tableMetadataAfterDownload { + isRebalanced := false + if t.TotalBytes == 0 { + continue + } + totalFiles := 0 + for disk := range t.Files { + totalFiles += len(t.Files[disk]) + } + totalParts := 0 + for disk := range t.Parts { + totalParts += len(t.Parts[disk]) + } + if totalFiles == 0 && totalParts == 0 { + continue + } + partSize := t.TotalBytes / uint64(totalParts) + //re-balance parts + for disk := range t.Parts { + if _, diskExists := b.DiskToPathMap[disk]; !diskExists && disk != b.cfg.ClickHouse.EmbeddedBackupDisk { + diskType := remoteBackup.DiskTypes[disk] + storagePolicy, filteredDisks, err := filterDisksByTypeAndStoragePolicies(disk, diskType, disks, remoteBackup, t) + if err != nil { + return err + } + rebalancedDisks := common.EmptyMap{} + for j := range t.Parts[disk] { + isObjectDisk, downloadDisk, newFreeSpace, reBalanceErr := b.getDownloadDiskForNonExistsDisk(diskType, filteredDisks, partSize) + if reBalanceErr != nil { + return reBalanceErr + } + rebalancedDisks[downloadDisk] = struct{}{} + tableMetadataAfterDownload[i].Parts[disk][j].RebalancedDisk = downloadDisk + isRebalanced = true + if !isObjectDisk { + updateDiskFreeSize(downloadDisk, diskType, storagePolicy, newFreeSpace) + } + //re-balance file depend on part + if t.Files != nil && len(t.Files) > 0 { + if len(t.Files[disk]) == 0 { + return fmt.Errorf("table: `%s`.`%s` part.Name: %s, part.RebalancedDisk: %s, non empty `files` can't find disk: %s", t.Table, t.Database, t.Parts[disk][j].Name, t.Parts[disk][j].RebalancedDisk, disk) + } + for _, fileName := range t.Files[disk] { + if strings.HasPrefix(fileName, disk+"_"+t.Parts[disk][j].Name+".") { + if tableMetadataAfterDownload[i].RebalancedFiles == nil { + tableMetadataAfterDownload[i].RebalancedFiles = map[string]string{} + } + tableMetadataAfterDownload[i].RebalancedFiles[fileName] = downloadDisk + } + } + } + } + rebalancedDisksStr := strings.TrimPrefix( + strings.Replace(fmt.Sprintf("%v", rebalancedDisks), ":{}", "", -1), "map", + ) + log.Warnf("table '%s.%s' require disk '%s' that not found in system.disks, you can add nonexistent disks to `disk_mapping` in `clickhouse` config section, data will download to %v", t.Database, t.Table, disk, rebalancedDisksStr) + } + } + if isRebalanced { + if _, saveErr := t.Save(t.LocalFile, false); saveErr != nil { + return saveErr + } + } + } + + return nil +} + func (b *Backuper) downloadTableMetadataIfNotExists(ctx context.Context, backupName string, log *apexLog.Entry, tableTitle metadata.TableTitle) (*metadata.TableMetadata, error) { metadataLocalFile := path.Join(b.DefaultDataPath, "backup", backupName, "metadata", common.TablePathEncode(tableTitle.Database), fmt.Sprintf("%s.json", common.TablePathEncode(tableTitle.Table))) tm := &metadata.TableMetadata{} @@ -340,6 +433,7 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, } partitionsIdMap, _ = partition.ConvertPartitionsToIdsMapAndNamesList(ctx, b.ch, nil, []metadata.TableMetadata{tableMetadata}, partitions) filterPartsAndFilesByPartitionsFilter(tableMetadata, partitionsIdMap[metadata.TableTitle{Database: tableMetadata.Database, Table: tableMetadata.Table}]) + tableMetadata.LocalFile = localMetadataFile } if isProcessed { size += uint64(processedSize) @@ -394,6 +488,7 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, } written = int64(jsonSize) size += jsonSize + tableMetadata.LocalFile = localMetadataFile } if b.resume { b.resumableState.AppendToState(localMetadataFile, written) @@ -497,7 +592,12 @@ func (b *Backuper) downloadTableData(ctx context.Context, remoteBackup metadata. continue } archiveFile := table.Files[disk][downloadOffset[disk]] - tableLocalDir := b.getLocalBackupDataPathForTable(remoteBackup.BackupName, disk, dbAndTableDir) + diskName := disk + isRebalanced := false + if diskName, isRebalanced = table.RebalancedFiles[archiveFile]; !isRebalanced { + diskName = disk + } + tableLocalDir := b.getLocalBackupDataPathForTable(remoteBackup.BackupName, diskName, dbAndTableDir) downloadOffset[disk] += 1 tableRemoteFile := path.Join(remoteBackup.BackupName, "shadow", common.TablePathEncode(table.Database), common.TablePathEncode(table.Table), archiveFile) dataGroup.Go(func() error { @@ -529,7 +629,7 @@ func (b *Backuper) downloadTableData(ctx context.Context, remoteBackup metadata. for disk, parts := range table.Parts { tableRemotePath := path.Join(remoteBackup.BackupName, "shadow", dbAndTableDir, disk) - diskPath := b.DiskToPathMap[disk] + diskPath, diskExists := b.DiskToPathMap[disk] tableLocalPath := path.Join(diskPath, "backup", remoteBackup.BackupName, "shadow", dbAndTableDir, disk) if b.isEmbedded { tableLocalPath = path.Join(diskPath, remoteBackup.BackupName, "data", dbAndTableDir) @@ -538,6 +638,12 @@ func (b *Backuper) downloadTableData(ctx context.Context, remoteBackup metadata. if part.Required { continue } + if !diskExists { + diskPath, diskExists = b.DiskToPathMap[part.RebalancedDisk] + if !diskExists { + return fmt.Errorf("downloadTableData: table: `%s`.`%s`, disk: %s, part.Name: %s, part.RebalancedDisk: %s not rebalanced", table.Table, table.Database, disk, part.Name, part.RebalancedDisk) + } + } partRemotePath := path.Join(tableRemotePath, part.Name) partLocalPath := path.Join(tableLocalPath, part.Name) dataGroup.Go(func() error { @@ -584,8 +690,19 @@ func (b *Backuper) downloadDiffParts(ctx context.Context, remoteBackup metadata. diffRemoteFilesLock := &sync.Mutex{} for disk, parts := range table.Parts { + diskPath, diskExists := b.DiskToPathMap[disk] for _, part := range parts { - newPath := path.Join(b.DiskToPathMap[disk], "backup", remoteBackup.BackupName, "shadow", dbAndTableDir, disk, part.Name) + if !diskExists && part.RebalancedDisk == "" { + return fmt.Errorf("downloadDiffParts: table: `%s`.`%s`, disk: %s, part.Name: %s, part.RebalancedDisk: `%s` not rebalanced", table.Table, table.Database, disk, part.Name, part.RebalancedDisk) + } + if !diskExists { + diskPath, diskExists = b.DiskToPathMap[part.RebalancedDisk] + if !diskExists { + return fmt.Errorf("downloadDiffParts: table: `%s`.`%s`, disk: %s, part.Name: %s, part.RebalancedDisk: `%s` not rebalanced", table.Table, table.Database, disk, part.Name, part.RebalancedDisk) + } + disk = part.RebalancedDisk + } + newPath := path.Join(diskPath, "backup", remoteBackup.BackupName, "shadow", dbAndTableDir, disk, part.Name) if err := b.checkNewPath(newPath, part); err != nil { return err } @@ -600,6 +717,9 @@ func (b *Backuper) downloadDiffParts(ctx context.Context, remoteBackup metadata. if err != nil && os.IsNotExist(err) { partForDownload := part diskForDownload := disk + if !diskExists { + diskForDownload = part.RebalancedDisk + } downloadDiffGroup.Go(func() error { tableRemoteFiles, err := b.findDiffBackupFilesRemote(downloadDiffCtx, remoteBackup, table, diskForDownload, partForDownload, log) if err != nil { @@ -830,17 +950,22 @@ func (b *Backuper) findDiffFileExist(ctx context.Context, requiredBackup *metada log.WithFields(apexLog.Fields{"tableRemoteFile": tableRemoteFile, "tableRemotePath": tableRemotePath, "part": part.Name}).Debugf("findDiffFileExist not found") return "", "", err } - if tableLocalDir, diskExists := b.DiskToPathMap[localDisk]; !diskExists { - return "", "", fmt.Errorf("`%s` is not found in system.disks", localDisk) - } else { - if path.Ext(tableRemoteFile) == ".txt" { - tableLocalDir = path.Join(tableLocalDir, "backup", requiredBackup.BackupName, "shadow", dbAndTableDir, localDisk, part.Name) - } else { - tableLocalDir = path.Join(tableLocalDir, "backup", requiredBackup.BackupName, "shadow", dbAndTableDir, localDisk) + tableLocalDir, diskExists := b.DiskToPathMap[localDisk] + if !diskExists { + tableLocalDir, diskExists = b.DiskToPathMap[part.RebalancedDisk] + if !diskExists { + return "", "", fmt.Errorf("localDisk:%s, part.Name: %s, part.RebalancedDisk: %s is not found in system.disks", localDisk, part.Name, part.RebalancedDisk) } - log.WithFields(apexLog.Fields{"tableRemoteFile": tableRemoteFile, "tableRemotePath": tableRemotePath, "part": part.Name}).Debugf("findDiffFileExist found") - return tableRemotePath, tableLocalDir, nil + localDisk = part.RebalancedDisk } + + if path.Ext(tableRemoteFile) == ".txt" { + tableLocalDir = path.Join(tableLocalDir, "backup", requiredBackup.BackupName, "shadow", dbAndTableDir, localDisk, part.Name) + } else { + tableLocalDir = path.Join(tableLocalDir, "backup", requiredBackup.BackupName, "shadow", dbAndTableDir, localDisk) + } + log.WithFields(apexLog.Fields{"tableRemoteFile": tableRemoteFile, "tableRemotePath": tableRemotePath, "part": part.Name}).Debugf("findDiffFileExist found") + return tableRemotePath, tableLocalDir, nil } func (b *Backuper) ReadBackupMetadataRemote(ctx context.Context, backupName string) (*metadata.BackupMetadata, error) { @@ -937,3 +1062,46 @@ func (b *Backuper) downloadSingleBackupFile(ctx context.Context, remoteFile stri } return nil } + +// filterDisksByStoragePolicyAndType - https://github.com/Altinity/clickhouse-backup/issues/561 +func (b *Backuper) splitDisksByTypeAndStoragePolicy(disks []clickhouse.Disk) map[string]map[string][]clickhouse.Disk { + disksByTypeAndPolicy := map[string]map[string][]clickhouse.Disk{} + for _, d := range disks { + if !d.IsBackup { + if _, typeExists := disksByTypeAndPolicy[d.Type]; !typeExists { + disksByTypeAndPolicy[d.Type] = map[string][]clickhouse.Disk{} + } + for _, policy := range d.StoragePolicies { + if _, policyExists := disksByTypeAndPolicy[d.Type][policy]; !policyExists { + disksByTypeAndPolicy[d.Type][policy] = []clickhouse.Disk{} + } + disksByTypeAndPolicy[d.Type][policy] = append(disksByTypeAndPolicy[d.Type][policy], d) + } + } + } + return disksByTypeAndPolicy +} + +// getDownloadDiskForNonExistsDisk - https://github.com/Altinity/clickhouse-backup/issues/561 +// allow to Restore to new server with different storage policy, different disk count, +// implements `least_used` for normal disk and `random` for Object disks +func (b *Backuper) getDownloadDiskForNonExistsDisk(notExistsDiskType string, filteredDisks []clickhouse.Disk, partSize uint64) (bool, string, uint64, error) { + // random for non-local disks + if notExistsDiskType != "local" { + roundRobinIdx := rand.Intn(len(filteredDisks)) + return true, filteredDisks[roundRobinIdx].Name, 0, nil + } + // least_used for local + freeSpace := partSize + leastUsedIdx := -1 + for idx, d := range filteredDisks { + if filteredDisks[idx].FreeSpace > freeSpace { + freeSpace = d.FreeSpace + leastUsedIdx = idx + } + } + if leastUsedIdx < 0 { + return false, "", 0, fmt.Errorf("%s free space, not found in system.disks with `local` type", utils.FormatBytes(partSize)) + } + return false, filteredDisks[leastUsedIdx].Name, filteredDisks[leastUsedIdx].FreeSpace - partSize, nil +} diff --git a/pkg/backup/download_test.go b/pkg/backup/download_test.go new file mode 100644 index 00000000..7b1f9e39 --- /dev/null +++ b/pkg/backup/download_test.go @@ -0,0 +1,270 @@ +package backup + +import ( + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" + apexLog "github.com/apex/log" + "github.com/stretchr/testify/assert" + "regexp" + "testing" + "time" +) + +var b = Backuper{ + log: &apexLog.Entry{}, + DefaultDataPath: "/var/lib/clickhouse", + DiskToPathMap: map[string]string{ + "default": "/var/lib/clickhouse", + "s3": "/var/lib/clickhouse/disks/s3", + }, + cfg: &config.Config{ClickHouse: config.ClickHouseConfig{EmbeddedBackupDisk: "backup"}}, + isEmbedded: false, +} + +var baseDisks = []clickhouse.Disk{ + { + Name: "default", + Path: "/var/lib/clickhouse", + Type: "local", + FreeSpace: 1024, + StoragePolicies: []string{"default", "jbod"}, + IsBackup: false, + }, + { + Name: "s3", + Path: "/var/lib/clickhouse/disks/s3", + Type: "s3", + FreeSpace: 0, + StoragePolicies: []string{"s3_only", "jbod"}, + IsBackup: false, + }, + { + Name: "backup_s3", + Path: "/var/lib/clickhouse/disks/backup_s3", + Type: "s3", + FreeSpace: 0, + StoragePolicies: nil, + IsBackup: true, + }, +} +var jbodDisks = []clickhouse.Disk{ + { + Name: "hdd2", + Path: "/hdd2", + Type: "local", + FreeSpace: 1024, + StoragePolicies: []string{"default", "jbod"}, + IsBackup: false, + }, + { + Name: "s3_disk2", + Path: "/var/lib/clickhouse/disks/s3_disk2", + Type: "s3", + FreeSpace: 0, + StoragePolicies: []string{"s3_only", "jbod"}, + IsBackup: false, + }, +} +var log = apexLog.WithField("logger", "test") + +var remoteBackup = storage.Backup{ + BackupMetadata: metadata.BackupMetadata{ + BackupName: "Test", + // doesn't matter which disk path in backup, should use system.disks? + Disks: map[string]string{"default": "/test", "hdd2": "/disk2", "s3": "/s3", "s3_disk2": "/s3_new"}, + DiskTypes: map[string]string{"default": "local", "hdd2": "local", "s3": "s3", "s3_disk2": "s3"}, + ClickhouseBackupVersion: "unknown", + CreationDate: time.Now(), + Tags: "", + ClickHouseVersion: "unknown", + Databases: nil, + Tables: []metadata.TableTitle{ + { + Database: "default", + Table: "test", + }, + }, + Functions: nil, + DataFormat: "tar", + RequiredBackup: "", + }, + Legacy: false, + UploadDate: time.Now(), +} + +func TestReBalanceTablesMetadataIfDiskNotExists_Files_NoErrors(t *testing.T) { + remoteBackup.DataFormat = "tar" + baseTable := metadata.TableMetadata{ + Files: map[string][]string{ + "default": {"default_part_1_1_0.tar", "default_part_2_2_0.tar"}, + "s3": {"s3_part_5_5_0.tar", "s3_part_6_6_0.tar"}, + }, + Parts: map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "s3": {{Name: "part_5_5_0"}, {Name: "part_6_6_0"}}, + }, + RebalancedFiles: nil, + Database: "default", + Table: "test", + Query: "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id", + Size: nil, + TotalBytes: 1000, + Mutations: nil, + MetadataOnly: false, + LocalFile: "/dev/null", + } + var tableMetadataAfterDownload []metadata.TableMetadata + tableMetadataAfterDownload = append(tableMetadataAfterDownload, baseTable) + baseTable.Table = "test2" + baseTable.Query = "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='jbod'" + baseTable.Files = map[string][]string{ + "default": {"default_part_1_1_0.tar", "default_part_2_2_0.tar"}, + "hdd2": {"hdd2_part_3_3_0.tar", "hdd2_part_4_4_0.tar"}, + "s3": {"s3_part_5_5_0.tar", "s3_part_6_6_0.tar"}, + "s3_disk2": {"s3_disk2_part_7_7_0.tar", "s3_disk2_part_8_8_0.tar"}, + } + baseTable.Parts = map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "hdd2": {{Name: "part_3_3_0"}, {Name: "part_4_4_0"}}, + "s3": {{Name: "part_5_5_0"}, {Name: "part_6_6_0"}}, + "s3_disk2": {{Name: "part_7_7_0"}, {Name: "part_8_8_0"}}, + } + + tableMetadataAfterDownload = append(tableMetadataAfterDownload, baseTable) + + assert.NoError(t, b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload, baseDisks, remoteBackup, log)) + //rebalanced table + meta := tableMetadataAfterDownload[1] + assert.Equal(t, 4, len(meta.RebalancedFiles), "expect 4 rebalanced files in %s.%s", meta.Database, meta.Table) + for i, d := range jbodDisks { + for _, p := range meta.Parts[d.Name] { + assert.Equal(t, baseDisks[i].Name, p.RebalancedDisk, "expect rebalanced part:%s", p.Name) + } + } + + //non-rebalanced table + meta = tableMetadataAfterDownload[0] + assert.Equal(t, 0, len(meta.RebalancedFiles), "0 files shall rebalanced") + for _, d := range baseDisks { + for _, p := range meta.Parts[d.Name] { + assert.Equal(t, "", p.RebalancedDisk, "expect no rebalanced part: %s", p.Name) + } + } + +} + +func TestReBalanceTablesMetadataIfDiskNotExists_Parts_NoErrors(t *testing.T) { + remoteBackup.DataFormat = "directory" + baseTable := metadata.TableMetadata{ + Parts: map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "s3": {{Name: "part_5_5_0"}, {Name: "part_6_6_0"}}, + }, + RebalancedFiles: nil, + Database: "default", + Table: "test", + Query: "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id", + Size: nil, + TotalBytes: 1000, + Mutations: nil, + MetadataOnly: false, + LocalFile: "/dev/null", + } + var tableMetadataAfterDownload []metadata.TableMetadata + tableMetadataAfterDownload = append(tableMetadataAfterDownload, baseTable) + baseTable.Table = "test2" + baseTable.Query = "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='jbod'" + baseTable.Parts = map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "hdd2": {{Name: "part_3_3_0"}, {Name: "part_4_4_0"}}, + "s3": {{Name: "part_5_5_0"}, {Name: "part_6_6_0"}}, + "s3_disk2": {{Name: "part_7_7_0"}, {Name: "part_8_8_0"}}, + } + + tableMetadataAfterDownload = append(tableMetadataAfterDownload, baseTable) + + assert.NoError(t, b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload, baseDisks, remoteBackup, log)) + // no files re-balance + for _, meta := range tableMetadataAfterDownload { + assert.Equal(t, 0, len(meta.RebalancedFiles)) + } + // re-balance parts + meta := tableMetadataAfterDownload[1] + for i, d := range jbodDisks { + for _, p := range meta.Parts[d.Name] { + assert.Equal(t, baseDisks[i].Name, p.RebalancedDisk, "expect rebalanced part") + } + } + // non re-balance parts + meta = tableMetadataAfterDownload[0] + for _, d := range baseDisks { + for _, p := range meta.Parts[d.Name] { + assert.Equal(t, "", p.RebalancedDisk, "expect no rebalanced part") + } + } +} + +func TestReBalanceTablesMetadataIfDiskNotExists_CheckErrors(t *testing.T) { + invalidRemoteBackup := remoteBackup + invalidRemoteBackup.DataFormat = DirectoryFormat + invalidTable := metadata.TableMetadata{ + Parts: map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "hdd2": {{Name: "part_3_3_0"}, {Name: "part_4_4_0"}}, + "s3": {{Name: "part_5_5_0"}, {Name: "part_6_6_0"}}, + "s3_disk2": {{Name: "part_7_7_0"}, {Name: "part_8_8_0"}}, + }, + RebalancedFiles: nil, + Database: "default", + Table: "test", + Query: "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id", + Size: nil, + TotalBytes: 1000, + Mutations: nil, + MetadataOnly: false, + LocalFile: "/dev/null", + } + tableMetadataAfterDownload := []metadata.TableMetadata{invalidTable} + // not exists diskType + delete(invalidRemoteBackup.DiskTypes, "hdd2") + err := b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload, baseDisks, invalidRemoteBackup, log) + assert.Error(t, err) + assert.Equal(t, + "disk: hdd2 not found in disk_types section map[string]string{\"default\":\"local\", \"s3\":\"s3\", \"s3_disk2\":\"s3\"} in Test/metadata.json", + err.Error(), + ) + + // invalid disk_type + invalidRemoteBackup.DiskTypes["hdd2"] = "unknown" + err = b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload, baseDisks, invalidRemoteBackup, log) + assert.Error(t, err) + assert.Equal(t, "disk: hdd2, diskType: unknown not found in system.disks", err.Error()) + + // invalid storage_policy + invalidRemoteBackup.DiskTypes["hdd2"] = "local" + invalidTable.Table = "test3" + invalidTable.Query = "CREATE TABLE default.test3(id UInt64) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='invalid'" + tableMetadataAfterDownload = []metadata.TableMetadata{invalidTable} + err = b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload, baseDisks, invalidRemoteBackup, log) + assert.Error(t, err) + matched, matchErr := regexp.MatchString(`storagePolicy: invalid with diskType: \w+ not found in system.disks`, err.Error()) + assert.NoError(t, matchErr) + assert.True(t, matched) + + // no free space + invalidDisks := baseDisks + invalidDisks[0].FreeSpace = 0 + invalidTable.Table = "test4" + invalidTable.Query = "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='default'" + invalidTable.Parts = map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "hdd2": {{Name: "part_3_3_0"}, {Name: "part_4_4_0"}}, + } + tableMetadataAfterDownload = []metadata.TableMetadata{invalidTable} + err = b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload, invalidDisks, invalidRemoteBackup, log) + assert.Error(t, err) + assert.Equal(t, "250B free space, not found in system.disks with `local` type", err.Error()) + +} diff --git a/pkg/backup/list.go b/pkg/backup/list.go index 8367a32c..c6cb11cc 100644 --- a/pkg/backup/list.go +++ b/pkg/backup/list.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/ricochet2200/go-disk-usage/du" "io" "os" "path" @@ -173,7 +174,13 @@ func (b *Backuper) GetLocalBackups(ctx context.Context, disks []clickhouse.Disk) } if disks == nil { disks = []clickhouse.Disk{ - {Name: "default", Path: "/var/lib/clickhouse"}, + { + Name: "default", + Path: "/var/lib/clickhouse", + Type: "local", + FreeSpace: du.NewDiskUsage("/var/lib/clickhouse").Free(), + StoragePolicies: []string{"default"}, + }, } } defaultDataPath, err := b.ch.GetDefaultPath(disks) diff --git a/pkg/backup/restore.go b/pkg/backup/restore.go index e9fd8bee..47f90b5e 100644 --- a/pkg/backup/restore.go +++ b/pkg/backup/restore.go @@ -685,11 +685,6 @@ func (b *Backuper) RestoreData(ctx context.Context, backupName string, tablePatt diskMap[disk.Name] = disk.Path diskTypes[disk.Name] = disk.Type } - for diskName := range backup.Disks { - if _, exists := diskMap[diskName]; !exists { - diskMap[diskName] = backup.Disks[diskName] - } - } for diskName := range backup.DiskTypes { if _, exists := diskTypes[diskName]; !exists { diskTypes[diskName] = backup.DiskTypes[diskName] @@ -737,7 +732,6 @@ func (b *Backuper) restoreDataRegular(ctx context.Context, backupName string, ta if err != nil { return err } - disks = b.adjustDisksFromTablesWithSystemDisks(tablesForRestore, diskMap, diskTypes, disks, log) dstTablesMap := b.prepareDstTablesMap(chTables) missingTables := b.checkMissingTables(tablesForRestore, chTables) @@ -748,6 +742,7 @@ func (b *Backuper) restoreDataRegular(ctx context.Context, backupName string, ta restoreBackupWorkingGroup.SetLimit(int(b.cfg.General.DownloadConcurrency)) for i := range tablesForRestore { + tableRestoreStartTime := time.Now() table := tablesForRestore[i] // need mapped database path and original table.Database for HardlinkBackupPartsToStorage dstDatabase := table.Database @@ -782,7 +777,7 @@ func (b *Backuper) restoreDataRegular(ctx context.Context, backupName string, ta log.Warnf("can't apply mutation %s for table `%s`.`%s` : %v", mutation.Command, tablesForRestore[idx].Database, tablesForRestore[idx].Table, err) } } - log.Info("done") + log.WithField("duration", utils.HumanizeDuration(time.Since(tableRestoreStartTime))).Info("done") return nil }) } @@ -793,7 +788,7 @@ func (b *Backuper) restoreDataRegular(ctx context.Context, backupName string, ta } func (b *Backuper) restoreDataRegularByAttach(ctx context.Context, backupName string, table metadata.TableMetadata, diskMap, diskTypes map[string]string, disks []clickhouse.Disk, dstTable clickhouse.Table, log *apexLog.Entry) error { - if err := filesystemhelper.HardlinkBackupPartsToStorage(backupName, table, disks, dstTable.DataPaths, b.ch, false); err != nil { + if err := filesystemhelper.HardlinkBackupPartsToStorage(backupName, table, disks, diskMap, dstTable.DataPaths, b.ch, false); err != nil { return fmt.Errorf("can't copy data to storage '%s.%s': %v", table.Database, table.Table, err) } log.Debug("data to 'storage' copied") @@ -807,14 +802,14 @@ func (b *Backuper) restoreDataRegularByAttach(ctx context.Context, backupName st } func (b *Backuper) restoreDataRegularByParts(ctx context.Context, backupName string, table metadata.TableMetadata, diskMap, diskTypes map[string]string, disks []clickhouse.Disk, dstTable clickhouse.Table, log *apexLog.Entry) error { - if err := filesystemhelper.HardlinkBackupPartsToStorage(backupName, table, disks, dstTable.DataPaths, b.ch, true); err != nil { + if err := filesystemhelper.HardlinkBackupPartsToStorage(backupName, table, disks, diskMap, dstTable.DataPaths, b.ch, true); err != nil { return fmt.Errorf("can't copy data to detached '%s.%s': %v", table.Database, table.Table, err) } log.Debug("data to 'detached' copied") if err := b.downloadObjectDiskParts(ctx, backupName, table, diskMap, diskTypes); err != nil { return fmt.Errorf("can't restore object_disk server-side copy data parts '%s.%s': %v", table.Database, table.Table, err) } - if err := b.ch.AttachDataParts(table, dstTable, disks); err != nil { + if err := b.ch.AttachDataParts(table, dstTable); err != nil { return fmt.Errorf("can't attach data parts for table '%s.%s': %v", table.Database, table.Table, err) } return nil @@ -853,14 +848,26 @@ func (b *Backuper) downloadObjectDiskParts(ctx context.Context, backupName strin return fmt.Errorf("%s disk doesn't present in diskTypes: %v", diskName, diskTypes) } if diskType == "s3" || diskType == "azure_blob_storage" { - if err = object_disk.InitCredentialsAndConnections(ctx, b.ch, b.cfg, diskName); err != nil { + if _, exists := diskMap[diskName]; !exists { + for _, part := range parts { + if part.RebalancedDisk != "" { + if err = object_disk.InitCredentialsAndConnections(ctx, b.ch, b.cfg, part.RebalancedDisk); err != nil { + return err + } + } + } + } else if err = object_disk.InitCredentialsAndConnections(ctx, b.ch, b.cfg, diskName); err != nil { return err } start := time.Now() downloadObjectDiskPartsWorkingGroup, downloadCtx := errgroup.WithContext(ctx) downloadObjectDiskPartsWorkingGroup.SetLimit(int(b.cfg.General.DownloadConcurrency)) for _, part := range parts { - partPath := path.Join(diskMap[diskName], "backup", backupName, "shadow", dbAndTableDir, diskName, part.Name) + dstDiskName := diskName + if part.RebalancedDisk != "" { + dstDiskName = part.RebalancedDisk + } + partPath := path.Join(diskMap[dstDiskName], "backup", backupName, "shadow", dbAndTableDir, dstDiskName, part.Name) walkErr := filepath.Walk(partPath, func(fPath string, fInfo fs.FileInfo, err error) error { if err != nil { return err @@ -896,8 +903,8 @@ func (b *Backuper) downloadObjectDiskParts(ctx context.Context, backupName strin } else { return fmt.Errorf("incompatible object_disk[%s].Type=%s amd remote_storage: %s", diskName, diskType, b.cfg.General.RemoteStorage) } - if copiedSize, copyObjectErr := object_disk.CopyObject(downloadCtx, b.ch, b.cfg, diskName, storageObject.ObjectSize, srcBucket, srcKey, storageObject.ObjectRelativePath); copyObjectErr != nil { - return fmt.Errorf("object_disk.CopyObject error: %v", err) + if copiedSize, copyObjectErr := object_disk.CopyObject(downloadCtx, dstDiskName, storageObject.ObjectSize, srcBucket, srcKey, storageObject.ObjectRelativePath); copyObjectErr != nil { + return fmt.Errorf("object_disk.CopyObject error: %v", copyObjectErr) } else { atomic.AddInt64(&size, copiedSize) } @@ -954,35 +961,6 @@ func (b *Backuper) prepareDstTablesMap(chTables []clickhouse.Table) map[metadata return dstTablesMap } -func (b *Backuper) adjustDisksFromTablesWithSystemDisks(tablesForRestore ListOfTables, diskMap, diskTypes map[string]string, disks []clickhouse.Disk, log *apexLog.Entry) []clickhouse.Disk { - for _, t := range tablesForRestore { - for disk := range t.Parts { - if _, diskExists := diskMap[disk]; !diskExists { - if diskTypes[disk] != diskTypes["default"] { - log.Fatalf("table '%s.%s' require disk '%s' that not found in clickhouse table system.disks, and have different diskType %s than `default` disk %s", t.Database, t.Table, disk, diskTypes[disk], diskTypes["default"]) - } - log.Warnf("table '%s.%s' require disk '%s' that not found in clickhouse table system.disks, you can add nonexistent disks to `disk_mapping` in `clickhouse` config section, data will restored to %s", t.Database, t.Table, disk, diskMap["default"]) - found := false - for _, d := range disks { - if d.Name == disk { - found = true - break - } - } - if !found { - newDisk := clickhouse.Disk{ - Name: disk, - Path: diskMap["default"], - Type: "local", - } - disks = append(disks, newDisk) - } - } - } - } - return disks -} - func (b *Backuper) changeTablePatternFromRestoreDatabaseMapping(tablePattern string) string { for sourceDb, targetDb := range b.cfg.General.RestoreDatabaseMapping { if tablePattern != "" { diff --git a/pkg/clickhouse/clickhouse.go b/pkg/clickhouse/clickhouse.go index fe6e717a..1eb38340 100644 --- a/pkg/clickhouse/clickhouse.go +++ b/pkg/clickhouse/clickhouse.go @@ -7,9 +7,6 @@ import ( "database/sql" "errors" "fmt" - "github.com/Altinity/clickhouse-backup/v2/pkg/common" - "github.com/ClickHouse/clickhouse-go/v2/lib/driver" - "github.com/antchfx/xmlquery" "os" "path" "path/filepath" @@ -18,10 +15,14 @@ import ( "strings" "time" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" "github.com/Altinity/clickhouse-backup/v2/pkg/config" "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "github.com/antchfx/xmlquery" apexLog "github.com/apex/log" + "github.com/ricochet2200/go-disk-usage/du" ) // ClickHouse - provide @@ -29,7 +30,6 @@ type ClickHouse struct { Config *config.ClickHouseConfig Log *apexLog.Entry conn driver.Conn - disks []Disk version int isPartsColumnPresent int8 IsOpen bool @@ -218,9 +218,11 @@ func (ch *ClickHouse) getDisksFromSystemSettings(ctx context.Context) ([]Disk, e dataPathArray := strings.Split(metadataPath, "/") clickhouseData := path.Join(dataPathArray[:len(dataPathArray)-1]...) return []Disk{{ - Name: "default", - Path: path.Join("/", clickhouseData), - Type: "local", + Name: "default", + Path: path.Join("/", clickhouseData), + Type: "local", + FreeSpace: du.NewDiskUsage(path.Join("/", clickhouseData)).Free(), + StoragePolicies: []string{"default"}, }}, nil } } @@ -251,18 +253,40 @@ func (ch *ClickHouse) getDisksFromSystemDisks(ctx context.Context) ([]Disk, erro case <-ctx.Done(): return nil, ctx.Err() default: - isDiskType := make([]struct { - Present uint64 `ch:"is_disk_type_present"` - }, 0) - if err := ch.SelectContext(ctx, &isDiskType, "SELECT count() is_disk_type_present FROM system.columns WHERE database='system' AND table='disks' AND name='type'"); err != nil { + type DiskFields struct { + DiskTypePresent uint64 `ch:"is_disk_type_present"` + FreeSpacePresent uint64 `ch:"is_free_space_present"` + StoragePolicyPresent uint64 `ch:"is_storage_policy_present"` + } + diskFields := make([]DiskFields, 0) + if err := ch.SelectContext(ctx, &diskFields, + "SELECT countIf(name='type') AS is_disk_type_present, "+ + "countIf(name='free_space') AS is_free_space_present, "+ + "countIf(name='disks') AS is_storage_policy_present "+ + "FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') ", + ); err != nil { return nil, err } diskTypeSQL := "'local'" - if len(isDiskType) > 0 && isDiskType[0].Present > 0 { - diskTypeSQL = "any(type)" + if len(diskFields) > 0 && diskFields[0].DiskTypePresent > 0 { + diskTypeSQL = "any(d.type)" + } + diskFreeSpaceSQL := "0" + if len(diskFields) > 0 && diskFields[0].FreeSpacePresent > 0 { + diskFreeSpaceSQL = "min(d.free_space)" + } + storagePoliciesSQL := "['default']" + joinStoragePoliciesSQL := "" + if len(diskFields) > 0 && diskFields[0].StoragePolicyPresent > 0 { + storagePoliciesSQL = "groupUniqArray(s.policy_name)" + joinStoragePoliciesSQL = " INNER JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name" } var result []Disk - query := fmt.Sprintf("SELECT path, any(name) AS name, %s AS type FROM system.disks GROUP BY path", diskTypeSQL) + query := fmt.Sprintf( + "SELECT d.path, any(d.name) AS name, %s AS type, %s AS free_space, %s AS storage_policies "+ + "FROM system.disks AS d %s GROUP BY d.path", + diskTypeSQL, diskFreeSpaceSQL, storagePoliciesSQL, joinStoragePoliciesSQL, + ) err := ch.SelectContext(ctx, &result, query) return result, err } @@ -703,7 +727,7 @@ func (ch *ClickHouse) FreezeTable(ctx context.Context, table *Table, name string } // AttachDataParts - execute ALTER TABLE ... ATTACH PART command for specific table -func (ch *ClickHouse) AttachDataParts(table metadata.TableMetadata, dstTable Table, disks []Disk) error { +func (ch *ClickHouse) AttachDataParts(table metadata.TableMetadata, dstTable Table) error { if dstTable.Database != "" && dstTable.Database != table.Database { table.Database = dstTable.Database } @@ -714,14 +738,14 @@ func (ch *ClickHouse) AttachDataParts(table metadata.TableMetadata, dstTable Tab if !canContinue { return nil } - for _, disk := range disks { - for _, part := range table.Parts[disk.Name] { + for disk := range table.Parts { + for _, part := range table.Parts[disk] { if !strings.HasSuffix(part.Name, ".proj") { query := fmt.Sprintf("ALTER TABLE `%s`.`%s` ATTACH PART '%s'", table.Database, table.Table, part.Name) if err := ch.Query(query); err != nil { return err } - ch.Log.WithField("table", fmt.Sprintf("%s.%s", table.Database, table.Table)).WithField("disk", disk.Name).WithField("part", part.Name).Debug("attached") + ch.Log.WithField("table", fmt.Sprintf("%s.%s", table.Database, table.Table)).WithField("disk", disk).WithField("part", part.Name).Debug("attached") } } } @@ -1295,3 +1319,15 @@ func (ch *ClickHouse) GetPreprocessedXMLSettings(ctx context.Context, settingsXP } return resultSettings, nil } + +var storagePolicyRE = regexp.MustCompile(`SETTINGS.+storage_policy[^=]*=[^']*'([^']+)'`) + +func (ch *ClickHouse) ExtractStoragePolicy(query string) string { + storagePolicy := "default" + matches := storagePolicyRE.FindStringSubmatch(query) + if len(matches) > 0 { + storagePolicy = matches[1] + } + apexLog.Debugf("extract storage_policy: %s, query: %s", storagePolicy, query) + return storagePolicy +} diff --git a/pkg/clickhouse/clickhouse_test.go b/pkg/clickhouse/clickhouse_test.go index ac2c07c3..13eb7802 100644 --- a/pkg/clickhouse/clickhouse_test.go +++ b/pkg/clickhouse/clickhouse_test.go @@ -10,7 +10,7 @@ import ( func TestCheckTypesConsistency(t *testing.T) { ch := ClickHouse{ - Log: apexLog.WithField("logger", "clickhouse"), + Log: apexLog.WithField("logger", "test"), } table := &Table{ Database: "mydb", @@ -108,3 +108,18 @@ func TestCheckTypesConsistency(t *testing.T) { }) } } + +func TestExtractStoragePolicy(t *testing.T) { + ch := ClickHouse{ + Log: apexLog.WithField("logger", "test"), + } + + testCases := map[string]string{ + "CREATE TABLE `_test.ДБ_atomic__TestIntegrationS3`.test_s3_TestIntegrationS3 UUID '8135780b-0c9a-46a7-94fd-2aebb701eff6' (`id` UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/_test.ДБ_atomic__TestIntegrationS3/test_s3_TestIntegrationS3', '{replica}') ORDER BY id SETTINGS storage_policy = 's3_only', index_granularity = 8192": "s3_only", + "CREATE TABLE test2 SETTINGS storage_policy = 'default'": "default", + "CREATE TABLE test3": "default", + } + for query, policy := range testCases { + assert.Equal(t, policy, ch.ExtractStoragePolicy(query)) + } +} diff --git a/pkg/clickhouse/structs.go b/pkg/clickhouse/structs.go index d865b6ae..7dd7b679 100644 --- a/pkg/clickhouse/structs.go +++ b/pkg/clickhouse/structs.go @@ -38,10 +38,12 @@ type IsSystemTablesFieldPresent struct { } type Disk struct { - Name string `ch:"name"` - Path string `ch:"path"` - Type string `ch:"type"` - IsBackup bool + Name string `ch:"name"` + Path string `ch:"path"` + Type string `ch:"type"` + FreeSpace uint64 `ch:"free_space"` + StoragePolicies []string `ch:"storage_policies"` + IsBackup bool } // Database - Clickhouse system.databases struct diff --git a/pkg/filesystemhelper/filesystemhelper.go b/pkg/filesystemhelper/filesystemhelper.go index 9a460e2c..c7980fb4 100644 --- a/pkg/filesystemhelper/filesystemhelper.go +++ b/pkg/filesystemhelper/filesystemhelper.go @@ -115,25 +115,28 @@ func MkdirAll(path string, ch *clickhouse.ClickHouse, disks []clickhouse.Disk) e } // HardlinkBackupPartsToStorage - copy partitions for specific table to detached folder -func HardlinkBackupPartsToStorage(backupName string, backupTable metadata.TableMetadata, disks []clickhouse.Disk, tableDataPaths []string, ch *clickhouse.ClickHouse, toDetached bool) error { +func HardlinkBackupPartsToStorage(backupName string, backupTable metadata.TableMetadata, disks []clickhouse.Disk, diskMap map[string]string, tableDataPaths []string, ch *clickhouse.ClickHouse, toDetached bool) error { log := apexLog.WithFields(apexLog.Fields{"operation": "HardlinkBackupPartsToStorage"}) start := time.Now() dstDataPaths := clickhouse.GetDisksByPaths(disks, tableDataPaths) dbAndTableDir := path.Join(common.TablePathEncode(backupTable.Database), common.TablePathEncode(backupTable.Table)) - for _, backupDisk := range disks { - backupDiskName := backupDisk.Name - if len(backupTable.Parts[backupDiskName]) == 0 { - log.Debugf("%s disk have no parts", backupDisk.Name) - continue - } - dstParentDir, dstParentDirExists := dstDataPaths[backupDiskName] - if !dstParentDirExists { - return fmt.Errorf("dstDataPaths=%#v, not contains %s", dstDataPaths, backupDiskName) - } - if toDetached { - dstParentDir = filepath.Join(dstParentDir, "detached") - } + for backupDiskName := range backupTable.Parts { for _, part := range backupTable.Parts[backupDiskName] { + dstParentDir, dstParentDirExists := dstDataPaths[backupDiskName] + if !dstParentDirExists && part.RebalancedDisk == "" { + return fmt.Errorf("dstDataPaths=%#v, not contains %s", dstDataPaths, backupDiskName) + } + if !dstParentDirExists && part.RebalancedDisk != "" { + backupDiskName = part.RebalancedDisk + dstParentDir, dstParentDirExists = dstDataPaths[part.RebalancedDisk] + if !dstParentDirExists { + return fmt.Errorf("dstDataPaths=%#v, not contains %s", dstDataPaths, part.RebalancedDisk) + } + } + backupDiskPath := diskMap[backupDiskName] + if toDetached { + dstParentDir = filepath.Join(dstParentDir, "detached") + } dstPartPath := filepath.Join(dstParentDir, part.Name) info, err := os.Stat(dstPartPath) if err != nil { @@ -148,16 +151,16 @@ func HardlinkBackupPartsToStorage(backupName string, backupTable metadata.TableM } else if !info.IsDir() { return fmt.Errorf("'%s' should be directory or absent", dstPartPath) } - partPath := path.Join(backupDisk.Path, "backup", backupName, "shadow", dbAndTableDir, backupDisk.Name, part.Name) + srcPartPath := path.Join(backupDiskPath, "backup", backupName, "shadow", dbAndTableDir, backupDiskName, part.Name) // Legacy backup support - if _, err := os.Stat(partPath); os.IsNotExist(err) { - partPath = path.Join(backupDisk.Path, "backup", backupName, "shadow", dbAndTableDir, part.Name) + if _, err := os.Stat(srcPartPath); os.IsNotExist(err) { + srcPartPath = path.Join(backupDiskPath, "backup", backupName, "shadow", dbAndTableDir, part.Name) } - if err := filepath.Walk(partPath, func(filePath string, info os.FileInfo, err error) error { + if err := filepath.Walk(srcPartPath, func(filePath string, info os.FileInfo, err error) error { if err != nil { return err } - filename := strings.Trim(strings.TrimPrefix(filePath, partPath), "/") + filename := strings.Trim(strings.TrimPrefix(filePath, srcPartPath), "/") dstFilePath := filepath.Join(dstPartPath, filename) if info.IsDir() { log.Debugf("MkDir %s", dstFilePath) diff --git a/pkg/metadata/metadata.go b/pkg/metadata/metadata.go index 98942884..7cade93d 100644 --- a/pkg/metadata/metadata.go +++ b/pkg/metadata/metadata.go @@ -42,6 +42,7 @@ type FunctionsMeta struct { type TableMetadata struct { Files map[string][]string `json:"files,omitempty"` + RebalancedFiles map[string]string `json:"rebalanced_files,omitempty"` Table string `json:"table"` Database string `json:"database"` Parts map[string][]Part `json:"parts"` @@ -52,6 +53,7 @@ type TableMetadata struct { DependenciesDatabase string `json:"dependencies_database,omitempty"` Mutations []MutationMetadata `json:"mutations,omitempty"` MetadataOnly bool `json:"metadata_only"` + LocalFile string `json:"local_file,omitempty"` } type MutationMetadata struct { @@ -60,9 +62,10 @@ type MutationMetadata struct { } type Part struct { - Name string `json:"name"` - Required bool `json:"required,omitempty"` - Partition string `json:"partition,omitempty"` + Name string `json:"name"` + Required bool `json:"required,omitempty"` + RebalancedDisk string `json:"rebalanced_disk,omitempty"` + // @todo remove legacy backup fields // Path string `json:"path"` // TODO: make it relative? look like useless now, can be calculated from Name HashOfAllFiles string `json:"hash_of_all_files,omitempty"` // ??? HashOfUncompressedFiles string `json:"hash_of_uncompressed_files,omitempty"` @@ -70,7 +73,6 @@ type Part struct { PartitionID string `json:"partition_id,omitempty"` ModificationTime *time.Time `json:"modification_time,omitempty"` Size int64 `json:"size,omitempty"` - // bytes_on_disk, data_compressed_bytes, data_uncompressed_bytes } type SplitPartFiles struct { diff --git a/pkg/storage/object_disk/object_disk.go b/pkg/storage/object_disk/object_disk.go index f1e47a49..bb7db128 100644 --- a/pkg/storage/object_disk/object_disk.go +++ b/pkg/storage/object_disk/object_disk.go @@ -637,10 +637,7 @@ func GetFileSize(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Con } */ -func CopyObject(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName string, srcSize int64, srcBucket, srcKey, dstPath string) (int64, error) { - if err := InitCredentialsAndConnections(ctx, ch, cfg, diskName); err != nil { - return 0, err - } +func CopyObject(ctx context.Context, diskName string, srcSize int64, srcBucket, srcKey, dstPath string) (int64, error) { connection, _ := DisksConnections.Load(diskName) remoteStorage := connection.GetRemoteStorage() return remoteStorage.CopyObject(ctx, srcSize, srcBucket, srcKey, dstPath) diff --git a/test/integration/custom_entrypoint.sh b/test/integration/custom_entrypoint.sh new file mode 100755 index 00000000..3a0f5c7f --- /dev/null +++ b/test/integration/custom_entrypoint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# to avoid backward incompatibility ;( +# https://t.me/clickhouse_ru/359960 +# https://t.me/clickhouse_ru/359968 +# https://t.me/clickhouse_ru/362378 + +if [ $# -ne 0 ]; then + /entrypoint.sh "$@" +else + /docker-entrypoint-initdb.d/dynamic_settings.sh + /entrypoint.sh +fi diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml index d4ff0a57..d9a2c2ca 100644 --- a/test/integration/docker-compose_advanced.yml +++ b/test/integration/docker-compose_advanced.yml @@ -227,6 +227,13 @@ services: QA_GCS_OVER_S3_SECRET_KEY: "${QA_GCS_OVER_S3_SECRET_KEY}" QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}" AWS_EC2_METADATA_DISABLED: "true" + + # to avoid backward incompatibility ;( + # https://t.me/clickhouse_ru/359960 + # https://t.me/clickhouse_ru/359968 + # https://t.me/clickhouse_ru/362378 + entrypoint: + - "/custom_entrypoint.sh" volumes: # clickhouse-backup related files requires for some tests - ${CLICKHOUSE_BACKUP_BIN:-../../clickhouse-backup/clickhouse-backup-race}:/usr/bin/clickhouse-backup @@ -252,6 +259,7 @@ services: # for local debug - ./install_delve.sh:/tmp/install_delve.sh # clickhouse configuration + - ./custom_entrypoint.sh:/custom_entrypoint.sh - ./dynamic_settings.sh:/docker-entrypoint-initdb.d/dynamic_settings.sh - ./enable-access_management.xml:/etc/clickhouse-server/users.d/enable-access_management.xml - ./backup-user.xml:/etc/clickhouse-server/users.d/backup-user.xml diff --git a/test/integration/dynamic_settings.sh b/test/integration/dynamic_settings.sh index bfedf4c5..37caf73b 100644 --- a/test/integration/dynamic_settings.sh +++ b/test/integration/dynamic_settings.sh @@ -82,7 +82,9 @@ EOT fi if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.[8-9]|^21\.[0-9]{2} || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[0-9]+ ]]; then - +if [[ -f /var/lib/clickhouse/storage_configuration_s3.xml ]]; then + cp -fv /var/lib/clickhouse/storage_configuration_s3.xml /etc/clickhouse-server/config.d/storage_configuration_s3.xml +else cat < /etc/clickhouse-server/config.d/storage_configuration_s3.xml @@ -112,17 +114,21 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_s3.xml EOT +fi fi if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^22\.[6-9]+ || "${CLICKHOUSE_VERSION}" =~ ^22\.1[0-9]+ || "${CLICKHOUSE_VERSION}" =~ ^2[3-9]\.[1-9]+ ]]; then if [[ "" != "${QA_GCS_OVER_S3_BUCKET}" ]]; then +if [[ -f /var/lib/clickhouse/storage_configuration_gcs.xml ]]; then + cp -fv /var/lib/clickhouse/storage_configuration_gcs.xml /etc/clickhouse-server/config.d/storage_configuration_gcs.xml +else cat < /etc/clickhouse-server/config.d/storage_configuration_gcs.xml - + s3 https://storage.googleapis.com/${QA_GCS_OVER_S3_BUCKET}/clickhouse_backup_disk_gcs_over_s3/${HOSTNAME}/{cluster}/{shard}/ ${QA_GCS_OVER_S3_ACCESS_KEY} @@ -130,13 +136,13 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_gcs.xml false false - + - disk_gcs_over_s3 + disk_gcs @@ -145,12 +151,17 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_gcs.xml EOT +fi fi fi if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.12 || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[0-9]+ ]]; then +if [[ -f /var/lib/clickhouse/storage_configuration_encrypted_s3.xml ]]; then + cp -fv /var/lib/clickhouse/storage_configuration_encrypted_s3.xml /etc/clickhouse-server/config.d/storage_configuration_encrypted_s3.xml +else + cat < /etc/clickhouse-server/config.d/storage_configuration_encrypted_s3.xml @@ -191,6 +202,7 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_encrypted_s3.x EOT +fi fi # embedded s3 backup configuration @@ -267,12 +279,16 @@ EOT mkdir -p /var/lib/clickhouse/disks/backups_azure/ chown -R clickhouse /var/lib/clickhouse/disks/ -cat < /etc/clickhouse-server/config.d/backup_storage_configuration_azure.xml +if [[ -f /var/lib/clickhouse/backup_storage_configuration_azblob.xml ]]; then + cp -fv /var/lib/clickhouse/backup_storage_configuration_azblob.xml /etc/clickhouse-server/config.d/backup_storage_configuration_azblob.xml +else + +cat < /etc/clickhouse-server/config.d/backup_storage_configuration_azblob.xml - + azure_blob_storage http://azure:10000/devstoreaccount1 azure-disk @@ -280,7 +296,7 @@ cat < /etc/clickhouse-server/config.d/backup_storage_configuration_azure.x devstoreaccount1 Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== false - + azure_blob_storage http://azure:10000/devstoreaccount1 @@ -295,7 +311,7 @@ cat < /etc/clickhouse-server/config.d/backup_storage_configuration_azure.x - azure + disk_azure @@ -308,6 +324,7 @@ cat < /etc/clickhouse-server/config.d/backup_storage_configuration_azure.x EOT +fi fi diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index e7059309..44c181fc 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -1957,6 +1957,7 @@ func runMainIntegrationScenario(t *testing.T, remoteStorageType, backupConfig st dropDatabasesFromTestDataDataSet(t, r, ch, databaseList) log.Info("Download") + replaceStorageDiskNameForReBalance(r, ch, remoteStorageType, false) downloadCmd := fmt.Sprintf("clickhouse-backup -c /etc/clickhouse-backup/%s download --resume %s", backupConfig, testBackupName) checkResumeAlreadyProcessed(downloadCmd, testBackupName, "download", r, remoteStorageType) @@ -2009,7 +2010,6 @@ func runMainIntegrationScenario(t *testing.T, remoteStorageType, backupConfig st } else { r.NoError(ch.checkData(t, testDataItem, r)) } - } // test end @@ -2021,6 +2021,40 @@ func runMainIntegrationScenario(t *testing.T, remoteStorageType, backupConfig st } else { fullCleanup(t, r, ch, []string{testBackupName, incrementBackupName}, []string{"remote", "local"}, databaseList, true, true, backupConfig) } + replaceStorageDiskNameForReBalance(r, ch, remoteStorageType, true) +} + +func replaceStorageDiskNameForReBalance(r *require.Assertions, ch *TestClickHouse, remoteStorageType string, isRebalanced bool) { + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "23.3") < 0 { + return + } + if remoteStorageType != "S3" && remoteStorageType != "GCS" && remoteStorageType != "AZBLOB" { + return + } + oldDisk := "disk_" + strings.ToLower(remoteStorageType) + newDisk := oldDisk + "_rebalanced" + if isRebalanced { + oldDisk = "disk_" + strings.ToLower(remoteStorageType) + "_rebalanced" + newDisk = strings.TrimSuffix(oldDisk, "_rebalanced") + } + fileNames := []string{"storage_configuration_" + strings.ToLower(remoteStorageType) + ".xml"} + if remoteStorageType == "S3" { + fileNames = append(fileNames, "storage_configuration_encrypted_"+strings.ToLower(remoteStorageType)+".xml") + } + for _, fileName := range fileNames { + origFile := "/etc/clickhouse-server/config.d/" + fileName + dstFile := "/var/lib/clickhouse/" + fileName + sedCmd := fmt.Sprintf("s/<%s>/<%s>/g; s/<\\/%s>/<\\/%s>/g; s/%s<\\/disk>/%s<\\/disk>/g", oldDisk, newDisk, oldDisk, newDisk, oldDisk, newDisk) + r.NoError(dockerExec("clickhouse", "sed", "-i", sedCmd, origFile)) + r.NoError(dockerExec("clickhouse", "cp", "-vf", origFile, dstFile)) + } + if isRebalanced { + r.NoError(dockerExec("clickhouse", "bash", "-xc", "cp -rfl /var/lib/clickhouse/disks/"+oldDisk+"/*", "/var/lib/clickhouse/disks/"+newDisk+"/")) + r.NoError(dockerExec("clickhouse", "rm", "-rf", "/var/lib/clickhouse/disks/"+oldDisk+"")) + } + ch.chbackend.Close() + r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker-compose", "-f", os.Getenv("COMPOSE_FILE"), "restart", "clickhouse")) + ch.connectWithWait(r, 3*time.Second, 1*time.Second) } func testBackupSpecifiedPartitions(t *testing.T, r *require.Assertions, ch *TestClickHouse, remoteStorageType string, backupConfig string) { @@ -2289,14 +2323,14 @@ func (ch *TestClickHouse) connectWithWait(r *require.Assertions, sleepBefore, ti err := ch.connect(timeOut.String()) if i == 10 { r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker", "logs", "clickhouse")) - out, dockerErr := dockerExecOut("clickhouse", "clickhouse client", "--echo", "-q", "'SELECT version()'") + out, dockerErr := dockerExecOut("clickhouse", "clickhouse", "client", "--echo", "-q", "'SELECT version()'") r.NoError(dockerErr) ch.chbackend.Log.Debug(out) r.NoError(err) } if err != nil { r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker", "ps", "-a")) - if out, dockerErr := dockerExecOut("clickhouse", "clickhouse client", "--echo", "-q", "SELECT version()"); dockerErr == nil { + if out, dockerErr := dockerExecOut("clickhouse", "clickhouse", "client", "--echo", "-q", "SELECT version()"); dockerErr == nil { log.Info(out) } else { log.Warn(out)