Skip to content

Commit 5b25a98

Browse files
1gtm1gtm
authored
[cherry-pick] Dump PVC storage information (#1379) (#1387)
Signed-off-by: hmsayem <hmsayem@appscode.com>
1 parent 1c54759 commit 5b25a98

File tree

24 files changed

+6695
-2856
lines changed

24 files changed

+6695
-2856
lines changed

‎go.mod

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ require (
1313
k8s.io/apimachinery v0.25.3
1414
k8s.io/client-go v0.25.1
1515
k8s.io/klog/v2 v2.80.1
16-
kmodules.xyz/client-go v0.25.19
16+
kmodules.xyz/client-go v0.25.23
1717
kmodules.xyz/custom-resources v0.25.0
18-
kmodules.xyz/offshoot-api v0.25.0
19-
stash.appscode.dev/apimachinery v0.29.0
18+
kmodules.xyz/offshoot-api v0.25.3
19+
stash.appscode.dev/apimachinery v0.29.1-0.20230529131221-1e979c48da10
2020
)
2121

2222
require (
@@ -69,7 +69,7 @@ require (
6969
golang.org/x/text v0.7.0 // indirect
7070
golang.org/x/time v0.1.0 // indirect
7171
gomodules.xyz/clock v0.0.0-20200817085942-06523dba733f // indirect
72-
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
72+
gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
7373
gomodules.xyz/mergo v0.3.13 // indirect
7474
gomodules.xyz/pointer v0.1.0 // indirect
7575
gomodules.xyz/sets v0.2.1 // indirect

‎go.sum

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
117117
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
118118
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
119119
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
120-
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
121120
github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
122121
github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
123122
github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
@@ -672,8 +671,8 @@ gomodules.xyz/flags v0.1.3 h1:jQ06+EfmoMv5NvjXvJon03dOhLU+FF0TQMWN7I6qpzs=
672671
gomodules.xyz/flags v0.1.3/go.mod h1:e+kvBLnqdEWGG670SKOYag1CXStM2Slrxq01OIK3tFs=
673672
gomodules.xyz/go-sh v0.1.0 h1:1BJAuGREh2RhePt7HRrpmjnkbgfpXlCzc42SiyZ5dkc=
674673
gomodules.xyz/go-sh v0.1.0/go.mod h1:N8IrjNiYppUI/rxENYrWD6FOrSxSyEZnIekPEWM7LP0=
675-
gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
676-
gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
674+
gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc=
675+
gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
677676
gomodules.xyz/logs v0.0.6 h1:8+9Wkud5yBPtIvkVszubyTeFxNII30lWODom0+GZD8U=
678677
gomodules.xyz/logs v0.0.6/go.mod h1:Q+fFtZFLEB5q86KmDehXCGuMP72Rv+Rwz0KuVxK+Gi4=
679678
gomodules.xyz/mergo v0.3.13 h1:q6cL/MMXZH/MrR2+yjSihFFq6UifXqjwaqI48B6cMEM=
@@ -874,14 +873,14 @@ k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73 h1:H9TCJUUx+2VA0ZiD9lvtaX8fthFsM
874873
k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
875874
kmodules.xyz/apiversion v0.2.0 h1:vAQYqZFm4xu4pbB1cAdHbFEPES6EQkcR4wc06xdTOWk=
876875
kmodules.xyz/apiversion v0.2.0/go.mod h1:oPX8g8LvlPdPX3Yc5YvCzJHQnw3YF/X4/jdW0b1am80=
877-
kmodules.xyz/client-go v0.25.19 h1:IYWixMIGMqS17LHYyiDxw8U/3ml6igI8QsYAdToYBGc=
878-
kmodules.xyz/client-go v0.25.19/go.mod h1:KWVpgjAj/VNzCzj/BihpJkY39aWIqe8qMp5RuPsuSJA=
876+
kmodules.xyz/client-go v0.25.23 h1:qz5XJYHLVZUowqfRXEJD7JQ4iaLLzQ1O1zPMmsdrkJw=
877+
kmodules.xyz/client-go v0.25.23/go.mod h1:wbdzLEoDYiCPI6dTW0mIAGNwkwFV4lC5BN1FJxiDsbw=
879878
kmodules.xyz/custom-resources v0.25.0 h1:5JQTEuiv6lC/+VVFNKqzfX4YtJCmN5E7mcNtGnHGVQM=
880879
kmodules.xyz/custom-resources v0.25.0/go.mod h1:ULwzvLmOqZJcPSXKI7iLclYL5eYRlKx8Nbex28Ht19E=
881880
kmodules.xyz/objectstore-api v0.25.1-0.20221104003322-f0289b5b6ca2 h1:efc0glYeBw+ok5s5ZecKdB9zgnRo/IvsLlSaQUPQjZE=
882881
kmodules.xyz/objectstore-api v0.25.1-0.20221104003322-f0289b5b6ca2/go.mod h1:X5aCkyU91p9TOn4jcWw0cfcJL0HCKd/Z6FJHdzKz1ZU=
883-
kmodules.xyz/offshoot-api v0.25.0 h1:Svq9da/+sg5afOjpgo9vx2J/Lu90Mo0aFxkdQmgKnGI=
884-
kmodules.xyz/offshoot-api v0.25.0/go.mod h1:ysEBn7LJuT3+s8ynAQA/OG0BSsJugXa6KGtDLMRjlKo=
882+
kmodules.xyz/offshoot-api v0.25.3 h1:KOGzW+TRHJvZ/KauvVQfKHztHF3HTHQ0ibbCHB3sWRg=
883+
kmodules.xyz/offshoot-api v0.25.3/go.mod h1:PUk4EuJFhhyQykCflHj7EgXcljGIqs9vi0IN0RpxtY4=
885884
kmodules.xyz/prober v0.25.0 h1:R5uRLHJEvEtEoogj+vaTAob0Btph6+PX5IlS6hPh8PA=
886885
kmodules.xyz/prober v0.25.0/go.mod h1:z4RTnjaajNQa/vPltsiOnO3xI716I/ziD2ac2Exm+1M=
887886
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
@@ -895,5 +894,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kF
895894
sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
896895
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
897896
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
898-
stash.appscode.dev/apimachinery v0.29.0 h1:36ijvg31s2UDQIAN2lnYY76E9dydUADutTkkErqokAQ=
899-
stash.appscode.dev/apimachinery v0.29.0/go.mod h1:7ao2U0Jgntc10uRf9KRmzXuabO12Nm7+2WpcRE/ZXWo=
897+
stash.appscode.dev/apimachinery v0.29.1-0.20230529131221-1e979c48da10 h1:QOMoITdH8rqfDD1biUw334dpWieMXl4hzBcgkd1//gk=
898+
stash.appscode.dev/apimachinery v0.29.1-0.20230529131221-1e979c48da10/go.mod h1:7ao2U0Jgntc10uRf9KRmzXuabO12Nm7+2WpcRE/ZXWo=

‎pkg/backup.go

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,14 @@ package pkg
1919
import (
2020
"context"
2121
"fmt"
22+
"os"
2223
"path/filepath"
2324
"time"
2425

26+
"stash.appscode.dev/apimachinery/apis"
2527
api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1"
2628
stash "stash.appscode.dev/apimachinery/client/clientset/versioned"
29+
"stash.appscode.dev/apimachinery/pkg/invoker"
2730
"stash.appscode.dev/apimachinery/pkg/restic"
2831
api_util "stash.appscode.dev/apimachinery/pkg/util"
2932

@@ -34,6 +37,7 @@ import (
3437
"k8s.io/client-go/kubernetes"
3538
"k8s.io/client-go/tools/clientcmd"
3639
"k8s.io/klog/v2"
40+
meta_util "kmodules.xyz/client-go/meta"
3741
appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
3842
appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
3943
v1 "kmodules.xyz/offshoot-api/api/v1"
@@ -235,6 +239,9 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res
235239
return nil, err
236240
}
237241

242+
if err := opt.dumpPVCStorageLimit(targetRef); err != nil {
243+
return nil, fmt.Errorf("failed to dump pvc storage limit info %w", err)
244+
}
238245
// dumped data has been stored in the interim data dir. Now, we will backup this directory using Stash.
239246
opt.backupOptions.BackupPaths = []string{opt.interimDataDir}
240247

@@ -243,6 +250,41 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res
243250
if err != nil {
244251
return nil, err
245252
}
246-
247253
return resticWrapper.RunBackup(opt.backupOptions, targetRef)
248254
}
255+
256+
func (opt *esOptions) dumpPVCStorageLimit(targetRef api_v1beta1.TargetRef) error {
257+
bs, err := opt.stashClient.StashV1beta1().BackupSessions(opt.namespace).Get(context.TODO(), opt.backupSessionName, metav1.GetOptions{})
258+
if err != nil {
259+
return err
260+
}
261+
262+
bi, err := invoker.NewBackupInvoker(opt.stashClient, bs.Spec.Invoker.Kind, bs.Spec.Invoker.Name, opt.namespace)
263+
if err != nil {
264+
return err
265+
}
266+
267+
var pvcName string
268+
for _, info := range bi.GetTargetInfo() {
269+
if info.Target != nil && targetMatched(info.Target.Ref, targetRef.Kind, targetRef.Name, targetRef.Namespace) {
270+
if info.InterimVolumeTemplate == nil {
271+
return nil
272+
}
273+
pvcName = meta_util.ValidNameWithPrefix(opt.backupSessionName, info.InterimVolumeTemplate.Name)
274+
}
275+
}
276+
277+
klog.Infoln("Dumping pvc storage limit info")
278+
pvc, err := opt.kubeClient.CoreV1().PersistentVolumeClaims(opt.namespace).Get(context.TODO(), pvcName, metav1.GetOptions{})
279+
if err != nil {
280+
return err
281+
}
282+
storageSize := pvc.Spec.Resources.Requests.Storage().String()
283+
284+
// Write PVC storage limit info to file
285+
return os.WriteFile(filepath.Join(opt.interimDataDir, apis.ESMetaFile), []byte(storageSize), os.ModePerm)
286+
}
287+
288+
func targetMatched(tref api_v1beta1.TargetRef, expectedKind, expectedName, expectedNamespace string) bool {
289+
return tref.Kind == expectedKind && tref.Namespace == expectedNamespace && tref.Name == expectedName
290+
}

‎pkg/restore.go

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@ package pkg
1919
import (
2020
"context"
2121
"fmt"
22+
"os"
2223
"path/filepath"
2324

25+
"stash.appscode.dev/apimachinery/apis"
2426
api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1"
2527
"stash.appscode.dev/apimachinery/pkg/restic"
2628

@@ -211,9 +213,13 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re
211213
return nil, err
212214
}
213215

216+
// delete the metadata file as it is not required for restoring the dumps
217+
if err := clearFile(filepath.Join(opt.interimDataDir, apis.ESMetaFile)); err != nil {
218+
return nil, err
219+
}
220+
214221
// run separate shell to restore indices
215222
// klog.Infoln("Performing multielasticdump on", hostname)
216-
217223
session.sh.ShowCMD = false
218224
session.setUserArgs(opt.esArgs)
219225
session.sh.Command(session.cmd.Name, session.cmd.Args...) // xref: multielasticdump: https://github.com/taskrabbit/elasticsearch-dump#multielasticdump
@@ -223,3 +229,12 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re
223229
}
224230
return restoreOutput, nil
225231
}
232+
233+
func clearFile(filepath string) error {
234+
if _, err := os.Stat(filepath); err == nil {
235+
if err := os.Remove(filepath); err != nil {
236+
return fmt.Errorf("unable to clean file: %v. Reason: %v", filepath, err)
237+
}
238+
}
239+
return nil
240+
}

‎pkg/utils.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ func clearDir(dir string) error {
148148
if err := os.RemoveAll(dir); err != nil {
149149
return fmt.Errorf("unable to clean datadir: %v. Reason: %v", dir, err)
150150
}
151+
151152
return os.MkdirAll(dir, os.ModePerm)
152153
}
153154

‎vendor/gomodules.xyz/jsonpatch/v2/jsonpatch.go

Lines changed: 34 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package jsonpatch
22

33
import (
4-
"bytes"
54
"encoding/json"
65
"fmt"
76
"reflect"
@@ -24,21 +23,28 @@ func (j *Operation) Json() string {
2423
}
2524

2625
func (j *Operation) MarshalJSON() ([]byte, error) {
27-
var b bytes.Buffer
28-
b.WriteString("{")
29-
b.WriteString(fmt.Sprintf(`"op":"%s"`, j.Operation))
30-
b.WriteString(fmt.Sprintf(`,"path":"%s"`, j.Path))
31-
// Consider omitting Value for non-nullable operations.
32-
if j.Value != nil || j.Operation == "replace" || j.Operation == "add" {
33-
v, err := json.Marshal(j.Value)
34-
if err != nil {
35-
return nil, err
36-
}
37-
b.WriteString(`,"value":`)
38-
b.Write(v)
39-
}
40-
b.WriteString("}")
41-
return b.Bytes(), nil
26+
// Ensure for add and replace we emit `value: null`
27+
if j.Value == nil && (j.Operation == "replace" || j.Operation == "add") {
28+
return json.Marshal(struct {
29+
Operation string `json:"op"`
30+
Path string `json:"path"`
31+
Value interface{} `json:"value"`
32+
}{
33+
Operation: j.Operation,
34+
Path: j.Path,
35+
})
36+
}
37+
// otherwise just marshal normally. We cannot literally do json.Marshal(j) as it would be recursively
38+
// calling this function.
39+
return json.Marshal(struct {
40+
Operation string `json:"op"`
41+
Path string `json:"path"`
42+
Value interface{} `json:"value,omitempty"`
43+
}{
44+
Operation: j.Operation,
45+
Path: j.Path,
46+
Value: j.Value,
47+
})
4248
}
4349

4450
type ByPath []Operation
@@ -149,9 +155,6 @@ func makePath(path string, newPart interface{}) string {
149155
if path == "" {
150156
return "/" + key
151157
}
152-
if strings.HasSuffix(path, "/") {
153-
return path + key
154-
}
155158
return path + "/" + key
156159
}
157160

@@ -211,22 +214,18 @@ func handleValues(av, bv interface{}, p string, patch []Operation) ([]Operation,
211214
}
212215
case []interface{}:
213216
bt := bv.([]interface{})
214-
if isSimpleArray(at) && isSimpleArray(bt) {
215-
patch = append(patch, compareEditDistance(at, bt, p)...)
216-
} else {
217-
n := min(len(at), len(bt))
218-
for i := len(at) - 1; i >= n; i-- {
219-
patch = append(patch, NewOperation("remove", makePath(p, i), nil))
220-
}
221-
for i := n; i < len(bt); i++ {
222-
patch = append(patch, NewOperation("add", makePath(p, i), bt[i]))
223-
}
224-
for i := 0; i < n; i++ {
225-
var err error
226-
patch, err = handleValues(at[i], bt[i], makePath(p, i), patch)
227-
if err != nil {
228-
return nil, err
229-
}
217+
n := min(len(at), len(bt))
218+
for i := len(at) - 1; i >= n; i-- {
219+
patch = append(patch, NewOperation("remove", makePath(p, i), nil))
220+
}
221+
for i := n; i < len(bt); i++ {
222+
patch = append(patch, NewOperation("add", makePath(p, i), bt[i]))
223+
}
224+
for i := 0; i < n; i++ {
225+
var err error
226+
patch, err = handleValues(at[i], bt[i], makePath(p, i), patch)
227+
if err != nil {
228+
return nil, err
230229
}
231230
}
232231
default:
@@ -235,100 +234,9 @@ func handleValues(av, bv interface{}, p string, patch []Operation) ([]Operation,
235234
return patch, nil
236235
}
237236

238-
func isBasicType(a interface{}) bool {
239-
switch a.(type) {
240-
case string, float64, bool:
241-
default:
242-
return false
243-
}
244-
return true
245-
}
246-
247-
func isSimpleArray(a []interface{}) bool {
248-
for i := range a {
249-
switch a[i].(type) {
250-
case string, float64, bool:
251-
default:
252-
val := reflect.ValueOf(a[i])
253-
if val.Kind() == reflect.Map {
254-
for _, k := range val.MapKeys() {
255-
av := val.MapIndex(k)
256-
if av.Kind() == reflect.Ptr || av.Kind() == reflect.Interface {
257-
if av.IsNil() {
258-
continue
259-
}
260-
av = av.Elem()
261-
}
262-
if av.Kind() != reflect.String && av.Kind() != reflect.Float64 && av.Kind() != reflect.Bool {
263-
return false
264-
}
265-
}
266-
return true
267-
}
268-
return false
269-
}
270-
}
271-
return true
272-
}
273-
274-
// https://en.wikipedia.org/wiki/Wagner%E2%80%93Fischer_algorithm
275-
// Adapted from https://github.com/texttheater/golang-levenshtein
276-
func compareEditDistance(s, t []interface{}, p string) []Operation {
277-
m := len(s)
278-
n := len(t)
279-
280-
d := make([][]int, m+1)
281-
for i := 0; i <= m; i++ {
282-
d[i] = make([]int, n+1)
283-
d[i][0] = i
284-
}
285-
for j := 0; j <= n; j++ {
286-
d[0][j] = j
287-
}
288-
289-
for j := 1; j <= n; j++ {
290-
for i := 1; i <= m; i++ {
291-
if reflect.DeepEqual(s[i-1], t[j-1]) {
292-
d[i][j] = d[i-1][j-1] // no op required
293-
} else {
294-
del := d[i-1][j] + 1
295-
add := d[i][j-1] + 1
296-
rep := d[i-1][j-1] + 1
297-
d[i][j] = min(rep, min(add, del))
298-
}
299-
}
300-
}
301-
302-
return backtrace(s, t, p, m, n, d)
303-
}
304-
305237
func min(x int, y int) int {
306238
if y < x {
307239
return y
308240
}
309241
return x
310242
}
311-
312-
func backtrace(s, t []interface{}, p string, i int, j int, matrix [][]int) []Operation {
313-
if i > 0 && matrix[i-1][j]+1 == matrix[i][j] {
314-
op := NewOperation("remove", makePath(p, i-1), nil)
315-
return append([]Operation{op}, backtrace(s, t, p, i-1, j, matrix)...)
316-
}
317-
if j > 0 && matrix[i][j-1]+1 == matrix[i][j] {
318-
op := NewOperation("add", makePath(p, i), t[j-1])
319-
return append([]Operation{op}, backtrace(s, t, p, i, j-1, matrix)...)
320-
}
321-
if i > 0 && j > 0 && matrix[i-1][j-1]+1 == matrix[i][j] {
322-
if isBasicType(s[0]) {
323-
op := NewOperation("replace", makePath(p, i-1), t[j-1])
324-
return append([]Operation{op}, backtrace(s, t, p, i-1, j-1, matrix)...)
325-
}
326-
327-
p2, _ := handleValues(s[i-1], t[j-1], makePath(p, i-1), []Operation{})
328-
return append(p2, backtrace(s, t, p, i-1, j-1, matrix)...)
329-
}
330-
if i > 0 && j > 0 && matrix[i-1][j-1] == matrix[i][j] {
331-
return backtrace(s, t, p, i-1, j-1, matrix)
332-
}
333-
return []Operation{}
334-
}

0 commit comments

Comments
 (0)