Skip to content

Commit ac4829c

Browse files
ishtiaqhimelishtiaqhimel
authored
Add support for creating space during dashboard restore (#1544)
Signed-off-by: Md. Ishtiaq Islam <ishtiaq@appscode.com>
1 parent e79d7c3 commit ac4829c

File tree

194 files changed

+54403
-1107
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

194 files changed

+54403
-1107
lines changed

‎go.mod

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ toolchain go1.22.2
77
require (
88
github.com/spf13/cobra v1.8.0
99
go.bytebuilders.dev/license-verifier/kubernetes v0.14.0
10+
golang.org/x/text v0.14.0
1011
gomodules.xyz/flags v0.1.3
1112
gomodules.xyz/go-sh v0.1.0
1213
gomodules.xyz/logs v0.0.7
@@ -18,8 +19,8 @@ require (
1819
kmodules.xyz/client-go v0.29.13
1920
kmodules.xyz/custom-resources v0.29.1
2021
kmodules.xyz/offshoot-api v0.29.0
21-
kubedb.dev/apimachinery v0.41.0
22-
kubedb.dev/db-client-go v0.0.11-0.20240208083800-50462091d436
22+
kubedb.dev/apimachinery v0.44.0
23+
kubedb.dev/db-client-go v0.0.15-0.20240419064300-f4fab3d22391
2324
sigs.k8s.io/controller-runtime v0.17.2
2425
stash.appscode.dev/apimachinery v0.34.0
2526
)
@@ -29,6 +30,7 @@ require (
2930
github.com/PuerkitoBio/purell v1.2.1 // indirect
3031
github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect
3132
github.com/beorn7/perks v1.0.1 // indirect
33+
github.com/blang/semver/v4 v4.0.0 // indirect
3234
github.com/cert-manager/cert-manager v1.13.3 // indirect
3335
github.com/cespare/xxhash/v2 v2.2.0 // indirect
3436
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
@@ -81,7 +83,6 @@ require (
8183
golang.org/x/oauth2 v0.15.0 // indirect
8284
golang.org/x/sys v0.18.0 // indirect
8385
golang.org/x/term v0.18.0 // indirect
84-
golang.org/x/text v0.14.0 // indirect
8586
golang.org/x/time v0.5.0 // indirect
8687
gomodules.xyz/clock v0.0.0-20200817085942-06523dba733f // indirect
8788
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
@@ -104,6 +105,7 @@ require (
104105
kmodules.xyz/monitoring-agent-api v0.29.0 // indirect
105106
kmodules.xyz/objectstore-api v0.29.1 // indirect
106107
kmodules.xyz/prober v0.29.0 // indirect
108+
kubeops.dev/petset v0.0.5 // indirect
107109
sigs.k8s.io/gateway-api v0.8.0 // indirect
108110
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
109111
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect

‎go.sum

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
3131
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
3232
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
3333
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
34+
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
35+
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
3436
github.com/cert-manager/cert-manager v1.13.3 h1:3R4G0RI7K0OkTZhWlVOC5SGZMYa2NwqmQJoyKydrz/M=
3537
github.com/cert-manager/cert-manager v1.13.3/go.mod h1:BM2+Pt/NmSv1Zr25/MHv6BgIEF9IUxA1xAjp80qkxgc=
3638
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -332,8 +334,8 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
332334
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
333335
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
334336
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
335-
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
336-
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
337+
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
338+
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
337339
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
338340
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
339341
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -587,10 +589,12 @@ kmodules.xyz/offshoot-api v0.29.0 h1:GHLhxxT9jU1N8+FvOCCeJNyU5g0duYS46UGrs6AHNLY
587589
kmodules.xyz/offshoot-api v0.29.0/go.mod h1:5NxhBblXoDHWStx9HCDJR2KFTwYjEZ7i1Id3jelIunw=
588590
kmodules.xyz/prober v0.29.0 h1:Ex7m4F9rH7uWNNJlLgP63ROOM+nUATJkC2L5OQ7nwMg=
589591
kmodules.xyz/prober v0.29.0/go.mod h1:UtK+HKyI1lFLEKX+HFLyOCVju6TO93zv3kwGpzqmKOo=
590-
kubedb.dev/apimachinery v0.41.0 h1:VbGQnH3YL7ICFvnCjAumlnL3HeZzg5F4F+flpjOSnG4=
591-
kubedb.dev/apimachinery v0.41.0/go.mod h1:rNWsbBzdnZA8G2FE8igi+nsGnlWqYurC+i3RFFDwluc=
592-
kubedb.dev/db-client-go v0.0.11-0.20240208083800-50462091d436 h1:JmSnrw+IdzbkgisjDQ5X0oo5jSts4OhBLHgEw6SL3qs=
593-
kubedb.dev/db-client-go v0.0.11-0.20240208083800-50462091d436/go.mod h1:sadvZUXo7tWz/75gdW7wpqeR25bi9kI/h2djyYLjN0A=
592+
kubedb.dev/apimachinery v0.44.0 h1:R7tiR35tmjJxNlpHUS8PVPmSrnFMfLycDE0c9XEU1/A=
593+
kubedb.dev/apimachinery v0.44.0/go.mod h1:7daaaWragCFLV38plrrJtsOuzinBSX3enMpliqlm3Uo=
594+
kubedb.dev/db-client-go v0.0.15-0.20240419064300-f4fab3d22391 h1:QqKQl3YVwOXcmHsCo5k6DSCW7qeUFJlhn7/XyJ4DmuA=
595+
kubedb.dev/db-client-go v0.0.15-0.20240419064300-f4fab3d22391/go.mod h1:Vv6x2vfdOGzjWa7iOOuMYyABXXAIkJpJ5eZ79GcgNqc=
596+
kubeops.dev/petset v0.0.5 h1:VVXi39JhjondlbHyZ98z0MLp6VCmiCMinL59K48Y2zA=
597+
kubeops.dev/petset v0.0.5/go.mod h1:ijtKT1HlAht2vBEZj5LW7C00XEs3B0d1VdCQgd5V4cA=
594598
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
595599
sigs.k8s.io/gateway-api v0.8.0 h1:isQQ3Jx2qFP7vaA3ls0846F0Amp9Eq14P08xbSwVbQg=
596600
sigs.k8s.io/gateway-api v0.8.0/go.mod h1:okOnjPNBFbIS/Rw9kAhuIUaIkLhTKEu+ARIuXk2dgaM=

‎pkg/backup.go

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package pkg
1818

1919
import (
2020
"context"
21+
"encoding/json"
2122
"fmt"
2223
"io"
2324
"net/http"
@@ -313,8 +314,17 @@ func (opt *esOptions) dumpDashboardObjects(appBinding *appcatalog.AppBinding) er
313314
return err
314315
}
315316

317+
data, err := json.Marshal(spaces)
318+
if err != nil {
319+
return fmt.Errorf("failed to marshal spaces: %w", err)
320+
}
321+
322+
if err = os.WriteFile(filepath.Join(opt.interimDataDir, SpacesInfoFile), data, os.ModePerm); err != nil {
323+
return fmt.Errorf("failed to write dashboard: %w", err)
324+
}
325+
316326
for _, space := range spaces {
317-
response, err := dashboardClient.ExportSavedObjects(space)
327+
response, err := dashboardClient.ExportSavedObjects(space.Id)
318328
if err != nil {
319329
return err
320330
}
@@ -328,7 +338,7 @@ func (opt *esOptions) dumpDashboardObjects(appBinding *appcatalog.AppBinding) er
328338
return fmt.Errorf("failed to export dashboard saved objects %s", string(body))
329339
}
330340

331-
if err = os.WriteFile(opt.getDashboardFilePath(space), body, os.ModePerm); err != nil {
341+
if err = os.WriteFile(opt.getDashboardFilePath(space.Id), body, os.ModePerm); err != nil {
332342
return err
333343
}
334344
}

‎pkg/restore.go

Lines changed: 74 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,22 @@ package pkg
1818

1919
import (
2020
"context"
21+
"encoding/json"
2122
"fmt"
2223
"io"
2324
"net/http"
2425
"os"
2526
"path/filepath"
27+
"strings"
2628

2729
"stash.appscode.dev/apimachinery/apis"
2830
api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1"
2931
"stash.appscode.dev/apimachinery/pkg/restic"
3032

3133
"github.com/spf13/cobra"
3234
license "go.bytebuilders.dev/license-verifier/kubernetes"
35+
"golang.org/x/text/cases"
36+
"golang.org/x/text/language"
3337
"gomodules.xyz/flags"
3438
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3539
"k8s.io/client-go/kubernetes"
@@ -38,6 +42,7 @@ import (
3842
appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
3943
appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
4044
v1 "kmodules.xyz/offshoot-api/api/v1"
45+
"kubedb.dev/db-client-go/elasticsearchdashboard"
4146
)
4247

4348
func NewCmdRestore() *cobra.Command {
@@ -257,13 +262,24 @@ func (opt *esOptions) restoreDashboardObjects(appBinding *appcatalog.AppBinding)
257262
return err
258263
}
259264

260-
spaces, err := dashboardClient.ListSpaces()
265+
spaces, err := opt.getSpaces()
266+
if err != nil {
267+
return err
268+
}
269+
270+
existingSpaces, err := dashboardClient.ListSpaces()
261271
if err != nil {
262272
return err
263273
}
264274

265275
for _, space := range spaces {
266-
response, err := dashboardClient.ImportSavedObjects(space, opt.getDashboardFilePath(space))
276+
if !isExist(existingSpaces, space.Id) {
277+
if err = dashboardClient.CreateSpace(space); err != nil {
278+
return fmt.Errorf("failed to create space %s: %w", space.Id, err)
279+
}
280+
}
281+
282+
response, err := dashboardClient.ImportSavedObjects(space.Id, opt.getDashboardFilePath(space.Id))
267283
if err != nil {
268284
return err
269285
}
@@ -278,10 +294,65 @@ func (opt *esOptions) restoreDashboardObjects(appBinding *appcatalog.AppBinding)
278294
}
279295

280296
// delete the dashboard file(s) as it is not required for restoring the dumps
281-
if err = clearFile(opt.getDashboardFilePath(space)); err != nil {
297+
if err = clearFile(opt.getDashboardFilePath(space.Id)); err != nil {
282298
return err
283299
}
284300
}
285301

286302
return nil
287303
}
304+
305+
func (opt *esOptions) getSpaces() ([]elasticsearchdashboard.Space, error) {
306+
if _, err := os.Stat(filepath.Join(opt.interimDataDir, SpacesInfoFile)); os.IsNotExist(err) {
307+
spaces := make([]elasticsearchdashboard.Space, 0)
308+
if err = filepath.Walk(opt.interimDataDir, func(path string, info os.FileInfo, err error) error {
309+
if err != nil {
310+
return err
311+
}
312+
313+
if filepath.Ext(info.Name()) == DashboardSavedObjectsExt {
314+
id, _ := strings.CutSuffix(info.Name(), DashboardSavedObjectsExt)
315+
spaces = append(spaces, elasticsearchdashboard.Space{
316+
Id: id,
317+
Name: cases.Title(language.English).String(strings.Replace(id, "-", " ", -1)),
318+
})
319+
}
320+
321+
return nil
322+
}); err != nil {
323+
return nil, err
324+
}
325+
326+
if len(spaces) == 0 {
327+
return nil, fmt.Errorf("no spaces found in interim data directory")
328+
}
329+
330+
return spaces, nil
331+
} else {
332+
data, err := os.ReadFile(filepath.Join(opt.interimDataDir, SpacesInfoFile))
333+
if err != nil {
334+
return nil, fmt.Errorf("failed to read spaces info %w", err)
335+
}
336+
337+
var spaces []elasticsearchdashboard.Space
338+
if err = json.Unmarshal(data, &spaces); err != nil {
339+
return nil, fmt.Errorf("failed to unmarshal spaces info %w", err)
340+
}
341+
342+
// delete the spaces info file as it is not required for restoring the dumps
343+
if err = clearFile(filepath.Join(opt.interimDataDir, SpacesInfoFile)); err != nil {
344+
return nil, err
345+
}
346+
347+
return spaces, nil
348+
}
349+
}
350+
351+
func isExist(existingSpaces []elasticsearchdashboard.Space, spaceId string) bool {
352+
for _, space := range existingSpaces {
353+
if space.Id == spaceId {
354+
return true
355+
}
356+
}
357+
return false
358+
}

‎pkg/utils.go

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,13 @@ import (
5050
)
5151

5252
const (
53-
ESUser = "ADMIN_USERNAME"
54-
ESPassword = "ADMIN_PASSWORD"
55-
MultiElasticDumpCMD = "multielasticdump"
56-
ESCACertFile = "root.pem"
57-
ESAuthFile = "auth.txt"
53+
ESUser = "ADMIN_USERNAME"
54+
ESPassword = "ADMIN_PASSWORD"
55+
MultiElasticDumpCMD = "multielasticdump"
56+
ESCACertFile = "root.pem"
57+
ESAuthFile = "auth.txt"
58+
SpacesInfoFile = "spaces.json"
59+
DashboardSavedObjectsExt = ".ndjson"
5860
)
5961

6062
type esOptions struct {
@@ -294,5 +296,5 @@ func (opt esOptions) getDashboardClient(appBinding *appcatalog.AppBinding) (*es_
294296
}
295297

296298
func (opt esOptions) getDashboardFilePath(space string) string {
297-
return filepath.Join(opt.interimDataDir, space+".ndjson")
299+
return filepath.Join(opt.interimDataDir, space+DashboardSavedObjectsExt)
298300
}

‎vendor/github.com/blang/semver/v4/LICENSE

Lines changed: 22 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/blang/semver/v4/json.go

Lines changed: 23 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)