diff --git a/Makefile b/Makefile index e38e6b2000..e9326cbbf1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ COVERAGE_PACKAGES=github.com/kopia/kopia/repo/...,github.com/kopia/kopia/fs/...,github.com/kopia/kopia/snapshot/... TEST_FLAGS?= KOPIA_INTEGRATION_EXE=$(CURDIR)/dist/integration/kopia.exe +TESTING_ACTION_EXE=$(CURDIR)/dist/integration/testingaction.exe FIO_DOCKER_TAG=ljishen/fio export BOTO_PATH=$(CURDIR)/tools/.boto @@ -219,8 +220,12 @@ vtest: $(gotestsum) build-integration-test-binary: go build -o $(KOPIA_INTEGRATION_EXE) -tags testing github.com/kopia/kopia +$(TESTING_ACTION_EXE): tests/testingaction/main.go + go build -o $(TESTING_ACTION_EXE) -tags testing github.com/kopia/kopia/tests/testingaction + integration-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE) -integration-tests: build-integration-test-binary $(gotestsum) +integration-tests: export TESTING_ACTION_EXE ?= $(TESTING_ACTION_EXE) +integration-tests: build-integration-test-binary $(gotestsum) $(TESTING_ACTION_EXE) $(GO_TEST) $(TEST_FLAGS) -count=1 -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/end_to_end_test endurance-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE) diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index dbe5bd55c6..c72fb7210e 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -86,6 +86,10 @@ func setPolicyFromFlags(ctx context.Context, p *policy.Policy, changeCount *int) return errors.Wrap(err, "scheduling policy") } + if err := setActionsFromFlags(ctx, &p.Actions, changeCount); err != nil { + return errors.Wrap(err, "actions policy") + } + // It's not really a list, just optional boolean, last one wins. for _, inherit := range *policySetInherit { *changeCount++ diff --git a/cli/command_policy_set_actions.go b/cli/command_policy_set_actions.go new file mode 100644 index 0000000000..b65137df30 --- /dev/null +++ b/cli/command_policy_set_actions.go @@ -0,0 +1,116 @@ +package cli + +import ( + "context" + "encoding/csv" + "fmt" + "io/ioutil" + "strings" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/snapshot/policy" +) + +const maxScriptLength = 32000 + +var ( + policySetBeforeFolderActionCommand = policySetCommand.Flag("before-folder-action", "Path to before-folder action command ('none' to remove)").Default("-").PlaceHolder("COMMAND").String() + policySetAfterFolderActionCommand = policySetCommand.Flag("after-folder-action", "Path to after-folder action command ('none' to remove)").Default("-").PlaceHolder("COMMAND").String() + policySetBeforeSnapshotRootActionCommand = policySetCommand.Flag("before-snapshot-root-action", "Path to before-snapshot-root action command ('none' to remove or 'inherit')").Default("-").PlaceHolder("COMMAND").String() + policySetAfterSnapshotRootActionCommand = policySetCommand.Flag("after-snapshot-root-action", "Path to after-snapshot-root action command ('none' to remove or 'inherit')").Default("-").PlaceHolder("COMMAND").String() + policySetActionCommandTimeout = policySetCommand.Flag("action-command-timeout", "Max time allowed for a action to run in seconds").Default("5m").Duration() + policySetActionCommandMode = policySetCommand.Flag("action-command-mode", "Action command mode").Default("essential").Enum("essential", "optional", "async") + policySetPersistActionScript = policySetCommand.Flag("persist-action-script", "Persist action script").Bool() +) + +func setActionsFromFlags(ctx context.Context, p *policy.ActionsPolicy, changeCount *int) error { + if err := setActionCommandFromFlags(ctx, "before-folder", &p.BeforeFolder, *policySetBeforeFolderActionCommand, changeCount); err != nil { + return errors.Wrap(err, "invalid before-folder-action") + } + + if err := setActionCommandFromFlags(ctx, "after-folder", &p.AfterFolder, *policySetAfterFolderActionCommand, changeCount); err != nil { + return errors.Wrap(err, "invalid after-folder-action") + } + + if err := setActionCommandFromFlags(ctx, "before-snapshot-root", &p.BeforeSnapshotRoot, *policySetBeforeSnapshotRootActionCommand, changeCount); err != nil { + return errors.Wrap(err, "invalid before-snapshot-root-action") + } + + if err := setActionCommandFromFlags(ctx, "after-snapshot-root", &p.AfterSnapshotRoot, *policySetAfterSnapshotRootActionCommand, changeCount); err != nil { + return errors.Wrap(err, "invalid after-snapshot-root-action") + } + + return nil +} + +func setActionCommandFromFlags(ctx context.Context, actionName string, cmd **policy.ActionCommand, value string, changeCount *int) error { + if value == "-" { + // not set + return nil + } + + if value == "" { + log(ctx).Infof(" - removing %v action", actionName) + + *changeCount++ + + *cmd = nil + + return nil + } + + *cmd = &policy.ActionCommand{ + TimeoutSeconds: int(policySetActionCommandTimeout.Seconds()), + Mode: *policySetActionCommandMode, + } + + *changeCount++ + + if *policySetPersistActionScript { + script, err := ioutil.ReadFile(value) //nolint:gosec + if err != nil { + return err + } + + if len(script) > maxScriptLength { + return errors.Errorf("action script file (%v) too long: %v, max allowed %d", value, len(script), maxScriptLength) + } + + log(ctx).Infof(" - setting %v (%v) action script from file %v (%v bytes) with timeout %v", actionName, *policySetActionCommandMode, value, len(script), *policySetActionCommandTimeout) + + (*cmd).Script = string(script) + + return nil + } + + // parse path as CSV as if space was the separator, this automatically takes care of quotations + r := csv.NewReader(strings.NewReader(value)) + r.Comma = ' ' // space + + fields, err := r.Read() + if err != nil { + return errors.Wrapf(err, "error parsing %v command", actionName) + } + + (*cmd).Command = fields[0] + (*cmd).Arguments = fields[1:] + + if len((*cmd).Arguments) == 0 { + log(ctx).Infof(" - setting %v (%v) action command to %v and timeout %v", actionName, *policySetActionCommandMode, quoteArguments((*cmd).Command), *policySetActionCommandTimeout) + } else { + log(ctx).Infof(" - setting %v (%v) action command to %v with arguments %v and timeout %v", actionName, *policySetActionCommandMode, quoteArguments((*cmd).Command), quoteArguments((*cmd).Arguments...), *policySetActionCommandTimeout) + } + + return nil +} + +func quoteArguments(s ...string) string { + var result []string + + for _, v := range s { + result = append(result, fmt.Sprintf("\"%v\"", v)) + } + + return strings.Join(result, " ") +} diff --git a/cli/command_policy_show.go b/cli/command_policy_show.go index f21375e0ba..a9e00917a5 100644 --- a/cli/command_policy_show.go +++ b/cli/command_policy_show.go @@ -3,11 +3,13 @@ package cli import ( "context" "fmt" + "strings" "github.com/pkg/errors" "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" ) @@ -44,10 +46,10 @@ func showPolicy(ctx context.Context, rep repo.Repository) error { return nil } -func getDefinitionPoint(parents []*policy.Policy, match func(p *policy.Policy) bool) string { - for i, p := range parents { +func getDefinitionPoint(target snapshot.SourceInfo, parents []*policy.Policy, match func(p *policy.Policy) bool) string { + for _, p := range parents { if match(p) { - if i == 0 { + if p.Target() == target { return "(defined for this target)" } @@ -84,38 +86,40 @@ func printPolicy(p *policy.Policy, parents []*policy.Policy) { printSchedulingPolicy(p, parents) printStdout("\n") printCompressionPolicy(p, parents) + printStdout("\n") + printActions(p, parents) } func printRetentionPolicy(p *policy.Policy, parents []*policy.Policy) { printStdout("Retention:\n") printStdout(" Annual snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepAnnual), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepAnnual != nil })) printStdout(" Monthly snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepMonthly), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepMonthly != nil })) printStdout(" Weekly snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepWeekly), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepWeekly != nil })) printStdout(" Daily snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepDaily), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepDaily != nil })) printStdout(" Hourly snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepHourly), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepHourly != nil })) printStdout(" Latest snapshots: %3v %v\n", valueOrNotSet(p.RetentionPolicy.KeepLatest), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.RetentionPolicy.KeepLatest != nil })) } @@ -125,7 +129,7 @@ func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) { printStdout(" Ignore cache directories: %5v %v\n", p.FilesPolicy.IgnoreCacheDirectoriesOrDefault(true), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.FilesPolicy.IgnoreCacheDirs != nil })) @@ -137,7 +141,7 @@ func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) { for _, rule := range p.FilesPolicy.IgnoreRules { rule := rule - printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool { + printStdout(" %-30v %v\n", rule, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return containsString(pol.FilesPolicy.IgnoreRules, rule) })) } @@ -148,7 +152,7 @@ func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) { for _, dotFile := range p.FilesPolicy.DotIgnoreFiles { dotFile := dotFile - printStdout(" %-30v %v\n", dotFile, getDefinitionPoint(parents, func(pol *policy.Policy) bool { + printStdout(" %-30v %v\n", dotFile, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return containsString(pol.FilesPolicy.DotIgnoreFiles, dotFile) })) } @@ -156,14 +160,14 @@ func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) { if maxSize := p.FilesPolicy.MaxFileSize; maxSize > 0 { printStdout(" Ignore files above: %10v %v\n", units.BytesStringBase2(maxSize), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.FilesPolicy.MaxFileSize != 0 })) } printStdout(" Scan one filesystem only: %5v %v\n", p.FilesPolicy.OneFileSystemOrDefault(false), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.FilesPolicy.OneFileSystem != nil })) } @@ -173,13 +177,13 @@ func printErrorHandlingPolicy(p *policy.Policy, parents []*policy.Policy) { printStdout(" Ignore file read errors: %5v %v\n", p.ErrorHandlingPolicy.IgnoreFileErrorsOrDefault(false), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.ErrorHandlingPolicy.IgnoreFileErrors != nil })) printStdout(" Ignore directory read errors: %5v %v\n", p.ErrorHandlingPolicy.IgnoreDirectoryErrorsOrDefault(false), - getDefinitionPoint(parents, func(pol *policy.Policy) bool { + getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.ErrorHandlingPolicy.IgnoreDirectoryErrors != nil })) } @@ -190,7 +194,7 @@ func printSchedulingPolicy(p *policy.Policy, parents []*policy.Policy) { any := false if p.SchedulingPolicy.Interval() != 0 { - printStdout(" Snapshot interval: %10v %v\n", p.SchedulingPolicy.Interval(), getDefinitionPoint(parents, func(pol *policy.Policy) bool { + printStdout(" Snapshot interval: %10v %v\n", p.SchedulingPolicy.Interval(), getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.SchedulingPolicy.Interval() != 0 })) @@ -202,7 +206,7 @@ func printSchedulingPolicy(p *policy.Policy, parents []*policy.Policy) { for _, tod := range p.SchedulingPolicy.TimesOfDay { tod := tod - printStdout(" %9v %v\n", tod, getDefinitionPoint(parents, func(pol *policy.Policy) bool { + printStdout(" %9v %v\n", tod, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { for _, t := range pol.SchedulingPolicy.TimesOfDay { if t == tod { return true @@ -224,7 +228,7 @@ func printSchedulingPolicy(p *policy.Policy, parents []*policy.Policy) { func printCompressionPolicy(p *policy.Policy, parents []*policy.Policy) { if p.CompressionPolicy.CompressorName != "" && p.CompressionPolicy.CompressorName != "none" { printStdout("Compression:\n") - printStdout(" Compressor: %q %v\n", p.CompressionPolicy.CompressorName, getDefinitionPoint(parents, func(pol *policy.Policy) bool { + printStdout(" Compressor: %q %v\n", p.CompressionPolicy.CompressorName, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return pol.CompressionPolicy.CompressorName != "" })) } else { @@ -238,7 +242,7 @@ func printCompressionPolicy(p *policy.Policy, parents []*policy.Policy) { for _, rule := range p.CompressionPolicy.OnlyCompress { rule := rule - printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool { + printStdout(" %-30v %v\n", rule, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return containsString(pol.CompressionPolicy.OnlyCompress, rule) })) } @@ -248,7 +252,7 @@ func printCompressionPolicy(p *policy.Policy, parents []*policy.Policy) { for _, rule := range p.CompressionPolicy.NeverCompress { rule := rule - printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool { + printStdout(" %-30v %v\n", rule, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { return containsString(pol.CompressionPolicy.NeverCompress, rule) })) } @@ -269,6 +273,60 @@ func printCompressionPolicy(p *policy.Policy, parents []*policy.Policy) { } } +func printActions(p *policy.Policy, parents []*policy.Policy) { + var anyActions bool + + if h := p.Actions.BeforeSnapshotRoot; h != nil { + printStdout("Run command before snapshot root: %v\n", getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { + return pol.Actions.BeforeSnapshotRoot == h + })) + + printActionCommand(h) + + anyActions = true + } + + if h := p.Actions.AfterSnapshotRoot; h != nil { + printStdout("Run command after snapshot root: %v\n", getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool { + return pol.Actions.AfterSnapshotRoot == h + })) + printActionCommand(h) + + anyActions = true + } + + if h := p.Actions.BeforeFolder; h != nil { + printStdout("Run command before this folder: (non-inheritable)\n") + + printActionCommand(h) + + anyActions = true + } + + if h := p.Actions.AfterFolder; h != nil { + printStdout("Run command after this folder: (non-inheritable)\n") + printActionCommand(h) + + anyActions = true + } + + if !anyActions { + printStdout("No actions defined.\n") + } +} + +func printActionCommand(h *policy.ActionCommand) { + if h.Script != "" { + printStdout(" Embedded Script: %q\n", h.Script) + } else { + printStdout(" Command: %v %v\n", h.Command, strings.Join(h.Arguments, " ")) + } + + printStdout(" Mode: %v\n", h.Mode) + printStdout(" Timeout: %v\n", h.TimeoutSeconds) + printStdout("\n") +} + func valueOrNotSet(p *int) string { if p == nil { return "-" diff --git a/fs/entry.go b/fs/entry.go index dd9e5d775d..8f1b70715a 100644 --- a/fs/entry.go +++ b/fs/entry.go @@ -14,6 +14,7 @@ type Entry interface { os.FileInfo Owner() OwnerInfo Device() DeviceInfo + LocalFilesystemPath() string // returns full local filesystem path or "" if not a local filesystem } // OwnerInfo describes owner of a filesystem entry. diff --git a/fs/localfs/local_fs.go b/fs/localfs/local_fs.go index 5dba9a9bb7..15102aac8e 100644 --- a/fs/localfs/local_fs.go +++ b/fs/localfs/local_fs.go @@ -77,6 +77,10 @@ func (e *filesystemEntry) Device() fs.DeviceInfo { return e.device } +func (e *filesystemEntry) LocalFilesystemPath() string { + return e.fullPath() +} + var _ os.FileInfo = (*filesystemEntry)(nil) func newEntry(fi os.FileInfo, parentDir string) filesystemEntry { diff --git a/internal/mockfs/mockfs.go b/internal/mockfs/mockfs.go index 14a3bcdc3b..163ae1fb7a 100644 --- a/internal/mockfs/mockfs.go +++ b/internal/mockfs/mockfs.go @@ -69,6 +69,10 @@ func (e *entry) Device() fs.DeviceInfo { return e.device } +func (e *entry) LocalFilesystemPath() string { + return "" +} + // Directory is mock in-memory implementation of fs.Directory. type Directory struct { entry diff --git a/repo/content/content_manager.go b/repo/content/content_manager.go index d8fbed0e43..bd62851177 100644 --- a/repo/content/content_manager.go +++ b/repo/content/content_manager.go @@ -778,7 +778,7 @@ func setupCaches(ctx context.Context, m *Manager, caching *CachingOptions) error } if caching.ownWritesCache == nil { - // this is test hook to allow test to specify custom cache + // this is test action to allow test to specify custom cache caching.ownWritesCache, err = newOwnWritesCache(ctx, caching, m.timeNow) if err != nil { return errors.Wrap(err, "unable to initialize own writes cache") diff --git a/snapshot/policy/actions_policy.go b/snapshot/policy/actions_policy.go new file mode 100644 index 0000000000..f06c48fb1e --- /dev/null +++ b/snapshot/policy/actions_policy.go @@ -0,0 +1,46 @@ +package policy + +// ActionsPolicy describes actions to be invoked when taking snapshots. +type ActionsPolicy struct { + // command runs once before and after the folder it's attached to (not inherited). + BeforeFolder *ActionCommand `json:"beforeFolder,omitempty"` + AfterFolder *ActionCommand `json:"afterFolder,omitempty"` + + // commands run once before and after each snapshot root (can be inherited). + BeforeSnapshotRoot *ActionCommand `json:"beforeSnapshotRoot,omitempty"` + AfterSnapshotRoot *ActionCommand `json:"afterSnapshotRoot,omitempty"` +} + +// ActionCommand configures a action command. +type ActionCommand struct { + // command + args to run + Command string `json:"path,omitempty"` + Arguments []string `json:"args,omitempty"` + + // alternatively inline script to run using either Unix shell or cmd.exe on Windows. + Script string `json:"script,omitempty"` + + TimeoutSeconds int `json:"timeout,omitempty"` + Mode string `json:"mode,omitempty"` // essential,optional,async +} + +// Merge applies default values from the provided policy. +// nolint:gocritic +func (p *ActionsPolicy) Merge(src ActionsPolicy) { + if p.BeforeSnapshotRoot == nil { + p.BeforeSnapshotRoot = src.BeforeSnapshotRoot + } + + if p.AfterSnapshotRoot == nil { + p.AfterSnapshotRoot = src.AfterSnapshotRoot + } +} + +// MergeNonInheritable copies non-inheritable properties from the provided actions policy. +func (p *ActionsPolicy) MergeNonInheritable(src ActionsPolicy) { + p.BeforeFolder = src.BeforeFolder + p.AfterFolder = src.AfterFolder +} + +// defaultActionsPolicy is the default actions policy. +var defaultActionsPolicy = ActionsPolicy{} diff --git a/snapshot/policy/policy.go b/snapshot/policy/policy.go index c82e874bec..ffb322cac7 100644 --- a/snapshot/policy/policy.go +++ b/snapshot/policy/policy.go @@ -19,6 +19,7 @@ type Policy struct { ErrorHandlingPolicy ErrorHandlingPolicy `json:"errorHandling,omitempty"` SchedulingPolicy SchedulingPolicy `json:"scheduling,omitempty"` CompressionPolicy CompressionPolicy `json:"compression,omitempty"` + Actions ActionsPolicy `json:"actions"` NoParent bool `json:"noParent,omitempty"` } @@ -63,6 +64,7 @@ func MergePolicies(policies []*Policy) *Policy { merged.ErrorHandlingPolicy.Merge(p.ErrorHandlingPolicy) merged.SchedulingPolicy.Merge(p.SchedulingPolicy) merged.CompressionPolicy.Merge(p.CompressionPolicy) + merged.Actions.Merge(p.Actions) } // Merge default expiration policy. @@ -71,6 +73,11 @@ func MergePolicies(policies []*Policy) *Policy { merged.ErrorHandlingPolicy.Merge(defaultErrorHandlingPolicy) merged.SchedulingPolicy.Merge(defaultSchedulingPolicy) merged.CompressionPolicy.Merge(defaultCompressionPolicy) + merged.Actions.Merge(defaultActionsPolicy) + + if len(policies) > 0 { + merged.Actions.MergeNonInheritable(policies[0].Actions) + } return &merged } diff --git a/snapshot/policy/policy_tree.go b/snapshot/policy/policy_tree.go index 94ff721614..4cd69b6f88 100644 --- a/snapshot/policy/policy_tree.go +++ b/snapshot/policy/policy_tree.go @@ -1,6 +1,8 @@ package policy -import "strings" +import ( + "strings" +) // DefaultPolicy is a default policy returned by policy tree in absence of other policies. var DefaultPolicy = &Policy{ @@ -9,6 +11,7 @@ var DefaultPolicy = &Policy{ CompressionPolicy: defaultCompressionPolicy, ErrorHandlingPolicy: defaultErrorHandlingPolicy, SchedulingPolicy: defaultSchedulingPolicy, + Actions: defaultActionsPolicy, } // Tree represents a node in the policy tree, where a policy can be diff --git a/snapshot/snapshotfs/all_sources.go b/snapshot/snapshotfs/all_sources.go index 38fea79ad6..a2b4dfb2a1 100644 --- a/snapshot/snapshotfs/all_sources.go +++ b/snapshot/snapshotfs/all_sources.go @@ -47,6 +47,10 @@ func (s *repositoryAllSources) Sys() interface{} { return nil } +func (s *repositoryAllSources) LocalFilesystemPath() string { + return "" +} + func (s *repositoryAllSources) Child(ctx context.Context, name string) (fs.Entry, error) { return fs.ReadDirAndFindChild(ctx, s, name) } diff --git a/snapshot/snapshotfs/repofs.go b/snapshot/snapshotfs/repofs.go index c7ede1c974..857a3cfdea 100644 --- a/snapshot/snapshotfs/repofs.go +++ b/snapshot/snapshotfs/repofs.go @@ -79,6 +79,10 @@ func (e *repositoryEntry) DirEntry() *snapshot.DirEntry { return e.metadata } +func (e *repositoryEntry) LocalFilesystemPath() string { + return "" +} + type repositoryDirectory struct { repositoryEntry summary *fs.DirectorySummary diff --git a/snapshot/snapshotfs/source_directories.go b/snapshot/snapshotfs/source_directories.go index 5eae0768c5..53fb8b1abb 100644 --- a/snapshot/snapshotfs/source_directories.go +++ b/snapshot/snapshotfs/source_directories.go @@ -47,6 +47,10 @@ func (s *sourceDirectories) Device() fs.DeviceInfo { return fs.DeviceInfo{} } +func (s *sourceDirectories) LocalFilesystemPath() string { + return "" +} + func (s *sourceDirectories) Child(ctx context.Context, name string) (fs.Entry, error) { return fs.ReadDirAndFindChild(ctx, s, name) } diff --git a/snapshot/snapshotfs/source_snapshots.go b/snapshot/snapshotfs/source_snapshots.go index 3944c65e1a..f287330f80 100644 --- a/snapshot/snapshotfs/source_snapshots.go +++ b/snapshot/snapshotfs/source_snapshots.go @@ -51,6 +51,10 @@ func (s *sourceSnapshots) Device() fs.DeviceInfo { return fs.DeviceInfo{} } +func (s *sourceSnapshots) LocalFilesystemPath() string { + return "" +} + func safeName(path string) string { path = strings.TrimLeft(path, "/") return strings.Replace(path, "/", "_", -1) diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index 20838b301d..be31284b89 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -372,7 +372,7 @@ func (u *Uploader) periodicallyCheckpoint(ctx context.Context, cp *checkpointReg return } - // test hook + // test action if u.checkpointFinished != nil { u.checkpointFinished <- struct{}{} } @@ -395,7 +395,22 @@ func (u *Uploader) uploadDirWithCheckpointing(ctx context.Context, rootDir fs.Di cancelCheckpointer := u.periodicallyCheckpoint(ctx, &cp, &snapshot.Manifest{Source: sourceInfo}) defer cancelCheckpointer() - return uploadDirInternal(ctx, u, rootDir, policyTree, previousDirs, ".", &dmb, &cp) + var hc actionContext + + localDirPathOrEmpty := rootDir.LocalFilesystemPath() + + overrideDir, err := executeBeforeFolderAction(ctx, "before-snapshot-root", policyTree.EffectivePolicy().Actions.BeforeSnapshotRoot, localDirPathOrEmpty, &hc) + if err != nil { + return nil, dirReadError{errors.Wrap(err, "error executing before-snapshot-root action")} + } + + if overrideDir != nil { + rootDir = overrideDir + } + + defer executeAfterFolderAction(ctx, "after-snapshot-root", policyTree.EffectivePolicy().Actions.AfterSnapshotRoot, localDirPathOrEmpty, &hc) + + return uploadDirInternal(ctx, u, rootDir, policyTree, previousDirs, localDirPathOrEmpty, ".", &dmb, &cp) } func (u *Uploader) foreachEntryUnlessCanceled(ctx context.Context, parallel int, relativePath string, entries fs.Entries, cb func(ctx context.Context, entry fs.Entry, entryRelativePath string) error) error { @@ -563,8 +578,16 @@ func isDir(e *snapshot.DirEntry) bool { return e.Type == snapshot.EntryTypeDirectory } -func (u *Uploader) processChildren(ctx context.Context, parentDirCheckpointRegistry *checkpointRegistry, parentDirBuilder *dirManifestBuilder, relativePath string, entries fs.Entries, policyTree *policy.Tree, previousEntries []fs.Entries) error { - if err := u.processSubdirectories(ctx, parentDirCheckpointRegistry, parentDirBuilder, relativePath, entries, policyTree, previousEntries); err != nil { +func (u *Uploader) processChildren( + ctx context.Context, + parentDirCheckpointRegistry *checkpointRegistry, + parentDirBuilder *dirManifestBuilder, + localDirPathOrEmpty, relativePath string, + entries fs.Entries, + policyTree *policy.Tree, + previousEntries []fs.Entries, +) error { + if err := u.processSubdirectories(ctx, parentDirCheckpointRegistry, parentDirBuilder, localDirPathOrEmpty, relativePath, entries, policyTree, previousEntries); err != nil { return err } @@ -575,7 +598,15 @@ func (u *Uploader) processChildren(ctx context.Context, parentDirCheckpointRegis return nil } -func (u *Uploader) processSubdirectories(ctx context.Context, parentDirCheckpointRegistry *checkpointRegistry, parentDirBuilder *dirManifestBuilder, relativePath string, entries fs.Entries, policyTree *policy.Tree, previousEntries []fs.Entries) error { +func (u *Uploader) processSubdirectories( + ctx context.Context, + parentDirCheckpointRegistry *checkpointRegistry, + parentDirBuilder *dirManifestBuilder, + localDirPathOrEmpty, relativePath string, + entries fs.Entries, + policyTree *policy.Tree, + previousEntries []fs.Entries, +) error { // for now don't process subdirectories in parallel, we need a mechanism to // prevent explosion of parallelism const parallelism = 1 @@ -598,7 +629,12 @@ func (u *Uploader) processSubdirectories(ctx context.Context, parentDirCheckpoin childDirBuilder := &dirManifestBuilder{} - de, err := uploadDirInternal(ctx, u, dir, policyTree.Child(entry.Name()), previousDirs, entryRelativePath, childDirBuilder, parentDirCheckpointRegistry) + childLocalDirPathOrEmpty := "" + if localDirPathOrEmpty != "" { + childLocalDirPathOrEmpty = filepath.Join(localDirPathOrEmpty, entry.Name()) + } + + de, err := uploadDirInternal(ctx, u, dir, policyTree.Child(entry.Name()), previousDirs, childLocalDirPathOrEmpty, entryRelativePath, childDirBuilder, parentDirCheckpointRegistry) if errors.Is(err, errCanceled) { return err } @@ -795,7 +831,7 @@ func uploadDirInternal( directory fs.Directory, policyTree *policy.Tree, previousDirs []fs.Directory, - dirRelativePath string, + localDirPathOrEmpty, dirRelativePath string, thisDirBuilder *dirManifestBuilder, thisCheckpointRegistry *checkpointRegistry, ) (*snapshot.DirEntry, error) { @@ -804,6 +840,26 @@ func uploadDirInternal( u.Progress.StartedDirectory(dirRelativePath) defer u.Progress.FinishedDirectory(dirRelativePath) + var definedActions policy.ActionsPolicy + + if p := policyTree.DefinedPolicy(); p != nil { + definedActions = p.Actions + } + + var hc actionContext + defer cleanupActionContext(ctx, &hc) + + overrideDir, herr := executeBeforeFolderAction(ctx, "before-folder", definedActions.BeforeFolder, localDirPathOrEmpty, &hc) + if herr != nil { + return nil, dirReadError{errors.Wrap(herr, "error executing before-folder action")} + } + + defer executeAfterFolderAction(ctx, "after-folder", definedActions.AfterFolder, localDirPathOrEmpty, &hc) + + if overrideDir != nil { + directory = overrideDir + } + t0 := u.repo.Time() entries, direrr := directory.Readdir(ctx) log(ctx).Debugf("finished reading directory %v in %v", dirRelativePath, u.repo.Time().Sub(t0)) @@ -842,7 +898,7 @@ func uploadDirInternal( }) defer thisCheckpointRegistry.removeCheckpointCallback(directory) - if err := u.processChildren(ctx, childCheckpointRegistry, thisDirBuilder, dirRelativePath, entries, policyTree, prevEntries); err != nil && !errors.Is(err, errCanceled) { + if err := u.processChildren(ctx, childCheckpointRegistry, thisDirBuilder, localDirPathOrEmpty, dirRelativePath, entries, policyTree, prevEntries); err != nil && !errors.Is(err, errCanceled) { return nil, err } diff --git a/snapshot/snapshotfs/upload_actions.go b/snapshot/snapshotfs/upload_actions.go new file mode 100644 index 0000000000..5271c39260 --- /dev/null +++ b/snapshot/snapshotfs/upload_actions.go @@ -0,0 +1,242 @@ +package snapshotfs + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/fs/localfs" + "github.com/kopia/kopia/snapshot/policy" +) + +const ( + actionCommandTimeout = 3 * time.Minute + actionScriptPermissions = 0o700 +) + +// actionContext carries state between before/after actions. +type actionContext struct { + ActionsEnabled bool + SnapshotID string + SourcePath string + SnapshotPath string + WorkDir string +} + +func (hc *actionContext) envars() []string { + return []string{ + fmt.Sprintf("KOPIA_SNAPSHOT_ID=%v", hc.SnapshotID), + fmt.Sprintf("KOPIA_SOURCE_PATH=%v", hc.SourcePath), + fmt.Sprintf("KOPIA_SNAPSHOT_PATH=%v", hc.SnapshotPath), + } +} + +func (hc *actionContext) ensureInitialized(dirPathOrEmpty string) error { + if dirPathOrEmpty == "" { + return nil + } + + if hc.ActionsEnabled { + // already initialized + return nil + } + + var randBytes [8]byte + + if _, err := rand.Read(randBytes[:]); err != nil { + return errors.Wrap(err, "error reading random bytes") + } + + hc.SnapshotID = fmt.Sprintf("%x", randBytes[:]) + hc.SourcePath = dirPathOrEmpty + hc.SnapshotPath = hc.SourcePath + + wd, err := ioutil.TempDir("", "kopia-action") + if err != nil { + return err + } + + hc.WorkDir = wd + hc.ActionsEnabled = true + + return nil +} + +func actionScriptExtension() string { + if runtime.GOOS == "windows" { + return ".cmd" + } + + return ".sh" +} + +// prepareCommandForAction prepares *exec.Cmd that will run the provided action command in the provided +// working directory. +func prepareCommandForAction(ctx context.Context, actionType string, h *policy.ActionCommand, workDir string) (*exec.Cmd, context.CancelFunc, error) { + timeout := actionCommandTimeout + if h.TimeoutSeconds != 0 { + timeout = time.Duration(h.TimeoutSeconds) * time.Second + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + + var c *exec.Cmd + + switch { + case h.Script != "": + scriptFile := filepath.Join(workDir, actionType+actionScriptExtension()) + if err := ioutil.WriteFile(scriptFile, []byte(h.Script), actionScriptPermissions); err != nil { + cancel() + + return nil, nil, err + } + + switch { + case runtime.GOOS == "windows": + c = exec.CommandContext(ctx, "cmd.exe", "/c", scriptFile) // nolint:gosec + case strings.HasPrefix(h.Script, "#!"): + // on unix if a script starts with #!, it will run under designated interpreter + c = exec.CommandContext(ctx, scriptFile) // nolint:gosec + default: + c = exec.CommandContext(ctx, "sh", "-e", scriptFile) // nolint:gosec + } + + case h.Command != "": + c = exec.CommandContext(ctx, h.Command, h.Arguments...) // nolint:gosec + + default: + cancel() + + return nil, nil, errors.Errorf("action did not provide either script nor command to run") + } + + // all actions run inside temporary working directory + c.Dir = workDir + + return c, cancel, nil +} + +// runActionCommand executes the action command passing the provided inputs as environment +// variables. It analyzes the standard output of the command looking for 'key=value' +// where the key is present in the provided outputs map and sets the corresponding map value. +func runActionCommand( + ctx context.Context, + actionType string, + h *policy.ActionCommand, + inputs []string, + captures map[string]string, + workDir string, +) error { + cmd, cancel, err := prepareCommandForAction(ctx, actionType, h, workDir) + if err != nil { + return errors.Wrap(err, "error preparing command") + } + + defer cancel() + + cmd.Env = append(os.Environ(), inputs...) + cmd.Stderr = os.Stderr + + if h.Mode == "async" { + return cmd.Start() + } + + v, err := cmd.Output() + if err != nil { + if h.Mode == "essential" { + return err + } + + log(ctx).Warningf("error running non-essential action command: %v", err) + } + + return parseCaptures(v, captures) +} + +// parseCaptures analyzes given byte array and updated the provided map values whenever +// map keys match lines inside the byte array. The lines must be formatted as k=v. +func parseCaptures(v []byte, captures map[string]string) error { + s := bufio.NewScanner(bytes.NewReader(v)) + for s.Scan() { + l := strings.SplitN(s.Text(), "=", 2) + if len(l) <= 1 { + continue + } + + key, value := l[0], l[1] + if _, ok := captures[key]; ok { + captures[key] = value + } + } + + return s.Err() +} + +func executeBeforeFolderAction(ctx context.Context, actionType string, h *policy.ActionCommand, dirPathOrEmpty string, hc *actionContext) (fs.Directory, error) { + if h == nil { + return nil, nil + } + + if err := hc.ensureInitialized(dirPathOrEmpty); err != nil { + return nil, errors.Wrap(err, "error initializing action context") + } + + if !hc.ActionsEnabled { + return nil, nil + } + + log(ctx).Debugf("running action %v on %v %#v", actionType, hc.SourcePath, *h) + + captures := map[string]string{ + "KOPIA_SNAPSHOT_PATH": "", + } + + if err := runActionCommand(ctx, actionType, h, hc.envars(), captures, hc.WorkDir); err != nil { + return nil, errors.Wrapf(err, "error running '%v' action", actionType) + } + + if p := captures["KOPIA_SNAPSHOT_PATH"]; p != "" { + hc.SnapshotPath = p + return localfs.Directory(hc.SnapshotPath) + } + + return nil, nil +} + +func executeAfterFolderAction(ctx context.Context, actionType string, h *policy.ActionCommand, dirPathOrEmpty string, hc *actionContext) { + if h == nil { + return + } + + if err := hc.ensureInitialized(dirPathOrEmpty); err != nil { + log(ctx).Warningf("error initializing action context: %v", err) + } + + if !hc.ActionsEnabled { + return + } + + if err := runActionCommand(ctx, actionType, h, hc.envars(), nil, hc.WorkDir); err != nil { + log(ctx).Warningf("error running '%v' action: %v", actionType, err) + } +} + +func cleanupActionContext(ctx context.Context, hc *actionContext) { + if hc.WorkDir != "" { + if err := os.RemoveAll(hc.WorkDir); err != nil { + log(ctx).Debugf("unable to remove action working directory: %v", err) + } + } +} diff --git a/snapshot/snapshotfs/upload_test.go b/snapshot/snapshotfs/upload_test.go index 375ee10808..a4e02f7b61 100644 --- a/snapshot/snapshotfs/upload_test.go +++ b/snapshot/snapshotfs/upload_test.go @@ -309,7 +309,7 @@ func TestUploadWithCheckpointing(t *testing.T) { Path: "path", } - // inject a hook into mock filesystem to trigger and wait for checkpoints at few places. + // inject a action into mock filesystem to trigger and wait for checkpoints at few places. // the places are not important, what's important that those are 3 separate points in time. dirsToCheckpointAt := []*mockfs.Directory{ th.sourceDir.Subdir("d1"), diff --git a/tests/end_to_end_test/snapshot_actions_test.go b/tests/end_to_end_test/snapshot_actions_test.go new file mode 100644 index 0000000000..c047971300 --- /dev/null +++ b/tests/end_to_end_test/snapshot_actions_test.go @@ -0,0 +1,317 @@ +package endtoend_test + +import ( + "bufio" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/kopia/kopia/tests/testenv" +) + +func TestSnapshotActionsBeforeSnapshotRoot(t *testing.T) { + t.Parallel() + + th := os.Getenv("TESTING_ACTION_EXE") + if th == "" { + t.Skip("TESTING_ACTION_EXE verifyNoError be set") + } + + e := testenv.NewCLITest(t) + + defer e.RunAndExpectSuccess(t, "repo", "disconnect") + + e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-hostname=foo", "--override-username=foo") + e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir2) + + envFile1 := filepath.Join(e.LogsDir, "env1.txt") + + // set a action before-snapshot-root that fails and which saves the environment to a file. + e.RunAndExpectSuccess(t, + "policy", "set", sharedTestDataDir1, + "--before-snapshot-root-action", + th+" --exit-code=3 --save-env="+envFile1) + + // this prevents the snapshot from being created + e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1) + + envFile2 := filepath.Join(e.LogsDir, "env2.txt") + + // now set a action before-snapshot-root that succeeds and saves environment to a different file + e.RunAndExpectSuccess(t, + "policy", "set", sharedTestDataDir1, + "--before-snapshot-root-action", + th+" --save-env="+envFile2) + + // snapshot now succeeds. + e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) + + env1 := mustReadEnvFile(t, envFile1) + env2 := mustReadEnvFile(t, envFile2) + + // make sure snapshot IDs are different between two attempts + if id1, id2 := env1["KOPIA_SNAPSHOT_ID"], env2["KOPIA_SNAPSHOT_ID"]; id1 == id2 { + t.Errorf("KOPIA_SNAPSHOT_ID passed to action was not different between runs %v", id1) + } + + // Now set up the action again, in optional mode, + e.RunAndExpectSuccess(t, + "policy", "set", sharedTestDataDir1, + "--before-snapshot-root-action", + th+" --exit-code=3", + "--action-command-mode=optional") + + // this will not prevent snapshot creation. + e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) + + // Now set up the action again, in async mode and pass --sleep so that the command takes some time. + // because the action is async it will not wait for the command. + e.RunAndExpectSuccess(t, + "policy", "set", sharedTestDataDir1, + "--before-snapshot-root-action", + th+" --exit-code=3 --sleep=30s", + "--action-command-mode=async") + + t0 := time.Now() + + // at this point the data is all cached so this will be quick, definitely less than 30s, + // async action failure will not prevent snapshot success. + e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) + + if dur := time.Since(t0); dur > 30*time.Second { + t.Errorf("command did not execute asynchronously (took %v)", dur) + } + + // Now set up essential action with a timeout of 3s and have the action sleep for 30s + e.RunAndExpectSuccess(t, + "policy", "set", sharedTestDataDir1, + "--before-snapshot-root-action", + th+" --sleep=30s", + "--action-command-timeout=3s") + + t0 = time.Now() + + // the action will be killed after 3s and cause a failure. + e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1) + + if dur := time.Since(t0); dur > 30*time.Second { + t.Errorf("command did not apply timeout (took %v)", dur) + } + + // Now set up essential action that will cause redirection to an alternative folder which does not exist. + e.RunAndExpectSuccess(t, + "policy", "set", sharedTestDataDir1, + "--before-snapshot-root-action", + th+" --stdout-file="+tmpfileWithContents(t, "KOPIA_SNAPSHOT_PATH=/no/such/directory\n")) + + e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1) + + // Now set up essential action that will cause redirection to an alternative folder which does exist. + e.RunAndExpectSuccess(t, + "policy", "set", sharedTestDataDir1, + "--before-snapshot-root-action", + th+" --stdout-file="+tmpfileWithContents(t, "KOPIA_SNAPSHOT_PATH="+sharedTestDataDir2+"\n")) + + e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) + + // since we redirected to sharedTestDataDir2 the object ID of last snapshot of sharedTestDataDir1 + // will be the same as snapshots of sharedTestDataDir2 + snaps1 := e.ListSnapshotsAndExpectSuccess(t, sharedTestDataDir1)[0].Snapshots + snaps2 := e.ListSnapshotsAndExpectSuccess(t, sharedTestDataDir2)[0].Snapshots + + if snaps1[0].ObjectID == snaps2[0].ObjectID { + t.Fatal("failed sanity check - snapshots are the same") + } + + if got, want := snaps1[len(snaps1)-1].ObjectID, snaps2[0].ObjectID; got != want { + t.Fatalf("invalid snapshot ID after redirection %v, wanted %v", got, want) + } + + // not setup the same redirection but in async mode - will be ignored because Kopia does not wait for asynchronous + // actions at all or parse their output. + e.RunAndExpectSuccess(t, + "policy", "set", sharedTestDataDir1, + "--before-snapshot-root-action", + th+" --stdout-file="+tmpfileWithContents(t, "KOPIA_SNAPSHOT_PATH="+sharedTestDataDir2+"\n"), + "--action-command-mode=async") + e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) + + // verify redirection had no effect - last snapshot will be the same as the first one + snaps1 = e.ListSnapshotsAndExpectSuccess(t, sharedTestDataDir1)[0].Snapshots + if got, want := snaps1[len(snaps1)-1].ObjectID, snaps1[0].ObjectID; got != want { + t.Fatalf("invalid snapshot ID after async action %v, wanted %v", got, want) + } +} + +func TestSnapshotActionsBeforeAfterFolder(t *testing.T) { + t.Parallel() + + th := os.Getenv("TESTING_ACTION_EXE") + if th == "" { + t.Skip("TESTING_ACTION_EXE verifyNoError be set") + } + + e := testenv.NewCLITest(t) + + e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") + + // create directory structure + rootDir := t.TempDir() + sd1 := filepath.Join(rootDir, "subdir1") + sd2 := filepath.Join(rootDir, "subdir2") + sd11 := filepath.Join(rootDir, "subdir1", "subdir1") + sd12 := filepath.Join(rootDir, "subdir1", "subdir2") + + verifyNoError(t, os.Mkdir(sd1, 0700)) + verifyNoError(t, os.Mkdir(sd2, 0700)) + verifyNoError(t, os.Mkdir(sd11, 0700)) + verifyNoError(t, os.Mkdir(sd12, 0700)) + + actionRanDir := t.TempDir() + + actionRanFileBeforeRoot := filepath.Join(actionRanDir, "before-root") + actionRanFileAfterRoot := filepath.Join(actionRanDir, "before-root") + actionRanFileBeforeSD1 := filepath.Join(actionRanDir, "before-sd1") + actionRanFileAfterSD1 := filepath.Join(actionRanDir, "before-sd1") + actionRanFileBeforeSD11 := filepath.Join(actionRanDir, "before-sd11") + actionRanFileAfterSD11 := filepath.Join(actionRanDir, "before-sd11") + actionRanFileBeforeSD2 := filepath.Join(actionRanDir, "before-sd2") + actionRanFileAfterSD2 := filepath.Join(actionRanDir, "before-sd2") + + // setup actions that will write a marker file when the action is executed. + // + // We are not setting a policy on 'sd12' to ensure it's not inherited + // from sd1. If it was inherited, the action would fail since it refuses to create the + // file if one already exists. + e.RunAndExpectSuccess(t, "policy", "set", rootDir, + "--before-folder-action", th+" --create-file="+actionRanFileBeforeRoot) + e.RunAndExpectSuccess(t, "policy", "set", rootDir, + "--after-folder-action", th+" --create-file="+actionRanFileAfterRoot) + e.RunAndExpectSuccess(t, "policy", "set", sd1, + "--before-folder-action", th+" --create-file="+actionRanFileBeforeSD1) + e.RunAndExpectSuccess(t, "policy", "set", sd1, + "--after-folder-action", th+" --create-file="+actionRanFileAfterSD1) + e.RunAndExpectSuccess(t, "policy", "set", sd2, + "--before-folder-action", th+" --create-file="+actionRanFileBeforeSD2) + e.RunAndExpectSuccess(t, "policy", "set", sd2, + "--after-folder-action", th+" --create-file="+actionRanFileAfterSD2) + e.RunAndExpectSuccess(t, "policy", "set", sd11, + "--before-folder-action", th+" --create-file="+actionRanFileBeforeSD11) + e.RunAndExpectSuccess(t, "policy", "set", sd11, + "--after-folder-action", th+" --create-file="+actionRanFileAfterSD11) + + e.RunAndExpectSuccess(t, "snapshot", "create", rootDir) + + verifyFileExists(t, actionRanFileBeforeRoot) + verifyFileExists(t, actionRanFileAfterRoot) + verifyFileExists(t, actionRanFileBeforeSD1) + verifyFileExists(t, actionRanFileBeforeSD11) + verifyFileExists(t, actionRanFileAfterSD11) + verifyFileExists(t, actionRanFileAfterSD1) + verifyFileExists(t, actionRanFileBeforeSD2) + verifyFileExists(t, actionRanFileAfterSD2) + + // the action will fail to run the next time since all 'actionRan*' files already exist. + e.RunAndExpectFailure(t, "snapshot", "create", rootDir) +} + +func TestSnapshotActionsEmbeddedScript(t *testing.T) { + t.Parallel() + + e := testenv.NewCLITest(t) + + e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") + + var ( + successScript = tmpfileWithContents(t, "echo Hello world!") + successScript2 string + failingScript string + goodRedirectScript = tmpfileWithContents(t, "echo KOPIA_SNAPSHOT_PATH="+sharedTestDataDir2) + badRedirectScript = tmpfileWithContents(t, "echo KOPIA_SNAPSHOT_PATH=/no/such/directory") + ) + + if runtime.GOOS == "windows" { + failingScript = tmpfileWithContents(t, "exit /b 1") + successScript2 = tmpfileWithContents(t, "echo Hello world!") + } else { + failingScript = tmpfileWithContents(t, "#!/bin/sh\nexit 1") + successScript2 = tmpfileWithContents(t, "#!/bin/sh\necho Hello world!") + } + + e.RunAndExpectSuccess(t, "policy", "set", sharedTestDataDir1, "--before-folder-action", successScript, "--persist-action-script") + e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) + + e.RunAndExpectSuccess(t, "policy", "set", sharedTestDataDir1, "--before-folder-action", goodRedirectScript, "--persist-action-script") + e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) + + e.RunAndExpectSuccess(t, "policy", "set", sharedTestDataDir1, "--before-folder-action", successScript2, "--persist-action-script") + e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1) + + snaps1 := e.ListSnapshotsAndExpectSuccess(t, sharedTestDataDir1)[0].Snapshots + if snaps1[0].ObjectID == snaps1[1].ObjectID { + t.Fatalf("redirection did not happen!") + } + + e.RunAndExpectSuccess(t, "policy", "set", sharedTestDataDir1, "--before-folder-action", badRedirectScript, "--persist-action-script") + e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1) + + e.RunAndExpectSuccess(t, "policy", "set", sharedTestDataDir1, "--before-folder-action", failingScript, "--persist-action-script") + e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1) +} + +func tmpfileWithContents(t *testing.T, contents string) string { + f, err := ioutil.TempFile("", "kopia-test") + verifyNoError(t, err) + + f.WriteString(contents) + f.Close() + + t.Cleanup(func() { os.Remove(f.Name()) }) + + return f.Name() +} + +func verifyFileExists(t *testing.T, fname string) { + t.Helper() + + _, err := os.Stat(fname) + if err != nil { + t.Fatal(err) + } +} + +func verifyNoError(t *testing.T, err error) { + t.Helper() + + if err != nil { + t.Fatal(err) + } +} + +func mustReadEnvFile(t *testing.T, fname string) map[string]string { + f, err := os.Open(fname) + + verifyNoError(t, err) + + defer f.Close() + s := bufio.NewScanner(f) + + m := map[string]string{} + + for s.Scan() { + parts := strings.SplitN(s.Text(), "=", 2) + if len(parts) == 2 { + m[parts[0]] = parts[1] + } + } + + verifyNoError(t, s.Err()) + + return m +} diff --git a/tests/testingaction/main.go b/tests/testingaction/main.go new file mode 100644 index 0000000000..d10018b0f4 --- /dev/null +++ b/tests/testingaction/main.go @@ -0,0 +1,115 @@ +// Command testingaction implements a action that is used in various tests. +package main + +import ( + "bufio" + "flag" + "io" + "io/ioutil" + "log" + "os" + "strings" + "time" + + "github.com/pkg/errors" +) + +var ( + exitCode = flag.Int("exit-code", 0, "Exit code") + sleepDuration = flag.Duration("sleep", 0, "Sleep duration") + saveEnvironmentToFile = flag.String("save-env", "", "Save environment to file (key=value).") + copyFilesSpec = flag.String("copy-files", "", "Copy files based on spec in the provided file (each line containing 'source => destination')") + createFile = flag.String("create-file", "", "Create empty file with a given name") + writeToStdout = flag.String("stdout-file", "", "Copy contents of the provided file to stdout.") + writeToStderr = flag.String("stderr-file", "", "Copy contents of the provided file to stderr.") +) + +func main() { + flag.Parse() + + if fn := *saveEnvironmentToFile; fn != "" { + if err := ioutil.WriteFile(fn, []byte(strings.Join(os.Environ(), "\n")), 0600); err != nil { + log.Fatalf("error writing environment file: %v", err) + } + } + + if fn := *writeToStdout; fn != "" { + if err := writeFileTo(os.Stdout, fn); err != nil { + log.Fatalf("error writing to stdout: %v", err) + } + } + + if fn := *writeToStderr; fn != "" { + if err := writeFileTo(os.Stderr, fn); err != nil { + log.Fatalf("error writing to stderr: %v", err) + } + } + + if fn := *copyFilesSpec; fn != "" { + if err := copyFiles(fn); err != nil { + log.Fatalf("unable to copy files: %v", err) + } + } + + if fn := *createFile; fn != "" { + if _, err := os.Stat(fn); !os.IsNotExist(err) { + log.Fatalf("unexpected file found: %v", fn) + } + + if err := ioutil.WriteFile(fn, nil, 0600); err != nil { + log.Fatalf("unable to create file: %v", err) + } + } + + time.Sleep(*sleepDuration) + os.Exit(*exitCode) +} + +func writeFileTo(dst io.Writer, fn string) error { + f, err := os.Open(fn) + if err != nil { + return err + } + + defer f.Close() + + io.Copy(dst, f) + + return nil +} + +func copyFiles(specFile string) error { + f, err := os.Open(specFile) + if err != nil { + return errors.Wrap(err, "unable to open spec file") + } + + defer f.Close() + + s := bufio.NewScanner(f) + for s.Scan() { + parts := strings.Split(s.Text(), " => ") + if len(parts) != 2 { + continue + } + + src := os.ExpandEnv(parts[0]) + dst := os.ExpandEnv(parts[1]) + + if err := copyFile(src, dst); err != nil { + return errors.Wrap(err, "copy file error") + } + } + + return s.Err() +} + +func copyFile(src, dst string) error { + df, err := os.Create(dst) + if err != nil { + return err + } + defer df.Close() + + return writeFileTo(df, src) +}