Skip to content

Commit c88df7a

Browse files
Dipta Dastamalsaha
Dipta Das
authored and
tamalsaha
committed
Run restic commands using docker (#754)
1 parent 8e24d32 commit c88df7a

File tree

7 files changed

+312
-70
lines changed

7 files changed

+312
-70
lines changed

‎cli/cli.go

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"path/filepath"
55

66
cs "github.com/appscode/stash/client/clientset/versioned"
7+
docker_image "github.com/appscode/stash/pkg/docker"
78
"github.com/spf13/cobra"
89
"k8s.io/client-go/kubernetes"
910
"k8s.io/client-go/rest"
@@ -13,8 +14,8 @@ import (
1314
)
1415

1516
const (
16-
cliScratchDir = "/tmp/stash-cli/scratch"
1717
cliSecretDir = "/tmp/stash-cli/secret"
18+
cliConfigDir = "/tmp/stash-cli/config"
1819
)
1920

2021
type stashCLIController struct {
@@ -23,6 +24,14 @@ type stashCLIController struct {
2324
stashClient cs.Interface
2425
}
2526

27+
var (
28+
image = docker_image.Docker{
29+
Registry: docker_image.ACRegistry,
30+
Image: docker_image.ImageStash,
31+
Tag: "latest", // TODO: update default release tag
32+
}
33+
)
34+
2635
func NewCLICmd() *cobra.Command {
2736
var cmd = &cobra.Command{
2837
Use: "cli",

‎cli/download.go

Lines changed: 79 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -4,23 +4,29 @@ import (
44
"fmt"
55
"io/ioutil"
66
"os"
7+
"os/exec"
8+
"os/user"
79
"path/filepath"
810

911
"github.com/appscode/go/flags"
1012
"github.com/appscode/go/log"
13+
"github.com/appscode/stash/pkg/cmds/docker"
1114
"github.com/appscode/stash/pkg/restic"
1215
"github.com/appscode/stash/pkg/util"
1316
"github.com/spf13/cobra"
17+
core "k8s.io/api/core/v1"
1418
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1519
)
1620

1721
func NewDownloadCmd() *cobra.Command {
1822
var (
19-
kubeConfig string
20-
repositoryName string
21-
namespace string
22-
restoreOpt = restic.RestoreOptions{
23-
SourceHost: restic.DefaultHost,
23+
kubeConfig string
24+
repositoryName string
25+
namespace string
26+
localDestination string
27+
restoreOpt = restic.RestoreOptions{
28+
SourceHost: restic.DefaultHost,
29+
Destination: docker.DestinationDir,
2430
}
2531
)
2632

@@ -42,55 +48,37 @@ func NewDownloadCmd() *cobra.Command {
4248
if err != nil {
4349
return err
4450
}
45-
4651
// unlock local backend
4752
if repository.Spec.Backend.Local != nil {
4853
return fmt.Errorf("can't restore from repository with local backend")
4954
}
50-
51-
// get source repository secret
55+
// get repository secret
5256
secret, err := c.kubeClient.CoreV1().Secrets(namespace).Get(repository.Spec.Backend.StorageSecretName, metav1.GetOptions{})
5357
if err != nil {
5458
return err
5559
}
5660

57-
// cleanup whole scratch/secret dir at the end
58-
defer os.RemoveAll(cliScratchDir)
59-
defer os.RemoveAll(cliSecretDir)
60-
61-
// write repository secrets in a temp dir
62-
if err := os.MkdirAll(cliSecretDir, 0755); err != nil {
63-
return err
64-
}
65-
for key, value := range secret.Data {
66-
if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil {
67-
return err
68-
}
69-
}
70-
7161
// configure restic wrapper
7262
extraOpt := util.ExtraOptions{
73-
SecretDir: cliSecretDir,
63+
SecretDir: docker.SecretDir,
7464
EnableCache: false,
75-
ScratchDir: cliScratchDir,
65+
ScratchDir: docker.ScratchDir,
7666
}
7767
setupOpt, err := util.SetupOptionsForRepository(*repository, extraOpt)
7868
if err != nil {
79-
return fmt.Errorf("setup option for repository fail")
69+
return fmt.Errorf("setup option for repository failed")
8070
}
81-
resticWrapper, err := restic.NewResticWrapper(setupOpt)
82-
if err != nil {
71+
72+
// write secret and config
73+
// cleanup whole config/secret dir at the end
74+
defer os.RemoveAll(cliSecretDir)
75+
defer os.RemoveAll(cliConfigDir)
76+
if err = prepareDockerVolumeForRestore(*secret, setupOpt, restoreOpt); err != nil {
8377
return err
8478
}
85-
// if destination flag not specified, restore in current directory
86-
if restoreOpt.Destination == "" {
87-
restoreOpt.Destination, err = os.Getwd()
88-
if err != nil {
89-
return err
90-
}
91-
}
92-
// run restore
93-
if _, err = resticWrapper.RunRestore(restoreOpt); err != nil {
79+
80+
// run restore inside docker
81+
if err = runRestoreViaDocker(localDestination); err != nil {
9482
return err
9583
}
9684
log.Infof("Repository %s/%s restored in path %s", namespace, repositoryName, restoreOpt.Destination)
@@ -101,12 +89,65 @@ func NewDownloadCmd() *cobra.Command {
10189
cmd.Flags().StringVar(&kubeConfig, "kubeconfig", kubeConfig, "Path of the Kube config file.")
10290
cmd.Flags().StringVar(&repositoryName, "repository", repositoryName, "Name of the Repository.")
10391
cmd.Flags().StringVar(&namespace, "namespace", "default", "Namespace of the Repository.")
92+
cmd.Flags().StringVar(&localDestination, "destination", localDestination, "Destination path where snapshot will be restored.")
10493

105-
cmd.Flags().StringVar(&restoreOpt.Destination, "destination", restoreOpt.Destination, "Destination path where snapshot will be restored.")
10694
cmd.Flags().StringVar(&restoreOpt.SourceHost, "host", restoreOpt.SourceHost, "Name of the source host machine")
10795
cmd.Flags().StringSliceVar(&restoreOpt.RestoreDirs, "directories", restoreOpt.RestoreDirs, "List of directories to be restored")
108-
// TODO: only allow a single snapshot ?
10996
cmd.Flags().StringSliceVar(&restoreOpt.Snapshots, "snapshots", restoreOpt.Snapshots, "List of snapshots to be restored")
11097

98+
cmd.Flags().StringVar(&image.Registry, "docker-registry", image.Registry, "Docker image registry for unlock job")
99+
cmd.Flags().StringVar(&image.Tag, "image-tag", image.Tag, "Stash image tag for unlock job")
100+
111101
return cmd
112102
}
103+
104+
func prepareDockerVolumeForRestore(secret core.Secret, setupOpt restic.SetupOptions, restoreOpt restic.RestoreOptions) error {
105+
// write repository secrets
106+
if err := os.MkdirAll(cliSecretDir, 0755); err != nil {
107+
return err
108+
}
109+
for key, value := range secret.Data {
110+
if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil {
111+
return err
112+
}
113+
}
114+
// write restic options
115+
err := docker.WriteSetupOptionToFile(&setupOpt, filepath.Join(cliConfigDir, docker.SetupOptionsFile))
116+
if err != nil {
117+
return err
118+
}
119+
return docker.WriteRestoreOptionToFile(&restoreOpt, filepath.Join(cliConfigDir, docker.RestoreOptionsFile))
120+
}
121+
122+
func runRestoreViaDocker(localDestination string) error {
123+
// get current user
124+
currentUser, err := user.Current()
125+
if err != nil {
126+
return err
127+
}
128+
// if destination flag is not specified, restore in current directory
129+
if localDestination == "" {
130+
if localDestination, err = os.Getwd(); err != nil {
131+
return err
132+
}
133+
}
134+
// create local destination dir
135+
if err := os.MkdirAll(localDestination, 0755); err != nil {
136+
return err
137+
}
138+
args := []string{
139+
"run",
140+
"--rm",
141+
"-u", currentUser.Uid,
142+
"-v", cliConfigDir + ":" + docker.ConfigDir,
143+
"-v", cliSecretDir + ":" + docker.SecretDir,
144+
"-v", localDestination + ":" + docker.DestinationDir,
145+
image.ToContainerImage(),
146+
"docker",
147+
"download-snapshots",
148+
}
149+
log.Infoln("Running docker with args:", args)
150+
out, err := exec.Command("docker", args...).CombinedOutput()
151+
log.Infoln("Output:", string(out))
152+
return err
153+
}

‎cli/unlock_repository.go

Lines changed: 52 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,16 @@ import (
44
"fmt"
55
"io/ioutil"
66
"os"
7+
"os/exec"
8+
"os/user"
79
"path/filepath"
810

911
"github.com/appscode/go/flags"
1012
"github.com/appscode/go/log"
1113
"github.com/appscode/go/types"
1214
"github.com/appscode/stash/apis/stash/v1alpha1"
1315
stash_scheme "github.com/appscode/stash/client/clientset/versioned/scheme"
14-
"github.com/appscode/stash/pkg/docker"
16+
"github.com/appscode/stash/pkg/cmds/docker"
1517
"github.com/appscode/stash/pkg/restic"
1618
"github.com/appscode/stash/pkg/util"
1719
"github.com/spf13/cobra"
@@ -30,14 +32,6 @@ const (
3032
unlockJobSecretVolume = "secret-volume"
3133
)
3234

33-
var (
34-
image = docker.Docker{
35-
Registry: docker.ACRegistry,
36-
Image: docker.ImageStash,
37-
Tag: "latest", // TODO: update default release tag
38-
}
39-
)
40-
4135
func NewUnlockRepositoryCmd() *cobra.Command {
4236
var (
4337
kubeConfig string
@@ -63,49 +57,40 @@ func NewUnlockRepositoryCmd() *cobra.Command {
6357
if err != nil {
6458
return err
6559
}
66-
6760
// unlock local backend
6861
if repository.Spec.Backend.Local != nil {
6962
if err = unlockLocalRepo(c, repository); err != nil {
7063
return fmt.Errorf("can't unlock repository for local backend, reason: %s", err)
7164
}
7265
return nil
7366
}
74-
7567
// get source repository secret
7668
secret, err := c.kubeClient.CoreV1().Secrets(namespace).Get(repository.Spec.Backend.StorageSecretName, metav1.GetOptions{})
7769
if err != nil {
7870
return err
7971
}
8072

81-
// cleanup whole scratch/secret dir at the end
82-
defer os.RemoveAll(cliScratchDir)
83-
defer os.RemoveAll(cliSecretDir)
84-
85-
// write repository secrets in a temp dir
86-
if err := os.MkdirAll(cliSecretDir, 0755); err != nil {
87-
return err
88-
}
89-
for key, value := range secret.Data {
90-
if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil {
91-
return err
92-
}
93-
}
94-
73+
// configure restic wrapper
9574
extraOpt := util.ExtraOptions{
96-
SecretDir: cliSecretDir,
75+
SecretDir: docker.SecretDir,
9776
EnableCache: false,
98-
ScratchDir: cliScratchDir,
77+
ScratchDir: docker.ScratchDir,
9978
}
10079
setupOpt, err := util.SetupOptionsForRepository(*repository, extraOpt)
10180
if err != nil {
102-
return fmt.Errorf("setup option for repository fail")
81+
return fmt.Errorf("setup option for repository failed")
10382
}
104-
resticWrapper, err := restic.NewResticWrapper(setupOpt)
105-
if err != nil {
83+
84+
// write secret and config
85+
// cleanup whole config/secret dir at the end
86+
defer os.RemoveAll(cliSecretDir)
87+
defer os.RemoveAll(cliConfigDir)
88+
if err = prepareDockerVolumeForUnlock(*secret, setupOpt); err != nil {
10689
return err
10790
}
108-
if err = resticWrapper.UnlockRepository(); err != nil {
91+
92+
// run unlock inside docker
93+
if err = runUnlockViaDocker(); err != nil {
10994
return err
11095
}
11196
log.Infof("Repository %s/%s unlocked", namespace, repositoryName)
@@ -123,6 +108,42 @@ func NewUnlockRepositoryCmd() *cobra.Command {
123108
return cmd
124109
}
125110

111+
func prepareDockerVolumeForUnlock(secret core.Secret, setupOpt restic.SetupOptions) error {
112+
// write repository secrets
113+
if err := os.MkdirAll(cliSecretDir, 0755); err != nil {
114+
return err
115+
}
116+
for key, value := range secret.Data {
117+
if err := ioutil.WriteFile(filepath.Join(cliSecretDir, key), value, 0755); err != nil {
118+
return err
119+
}
120+
}
121+
// write restic setup options
122+
return docker.WriteSetupOptionToFile(&setupOpt, filepath.Join(cliConfigDir, docker.SetupOptionsFile))
123+
}
124+
125+
func runUnlockViaDocker() error {
126+
// get current user
127+
currentUser, err := user.Current()
128+
if err != nil {
129+
return err
130+
}
131+
args := []string{
132+
"run",
133+
"--rm",
134+
"-u", currentUser.Uid,
135+
"-v", cliConfigDir + ":" + docker.ConfigDir,
136+
"-v", cliSecretDir + ":" + docker.SecretDir,
137+
image.ToContainerImage(),
138+
"docker",
139+
"unlock-repository",
140+
}
141+
log.Infoln("Running docker with args:", args)
142+
out, err := exec.Command("docker", args...).CombinedOutput()
143+
log.Infoln("Output:", string(out))
144+
return err
145+
}
146+
126147
func unlockLocalRepo(c *stashCLIController, repo *v1alpha1.Repository) error {
127148
_, path, err := util.GetBucketAndPrefix(&repo.Spec.Backend)
128149
if err != nil {

0 commit comments

Comments
 (0)