From a0f56644b35461dab6ac93a54eb5600b6b2ee811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hermann?= Date: Fri, 13 Mar 2026 19:08:05 +0100 Subject: [PATCH] feat: cli store configuration --- cmd/cloudstic/cmd_backup.go | 78 ++++----- cmd/cloudstic/cmd_check.go | 7 +- cmd/cloudstic/cmd_forget.go | 22 ++- cmd/cloudstic/cmd_init.go | 6 +- cmd/cloudstic/completion.go | 104 ++++++------ cmd/cloudstic/completion_test.go | 22 ++- cmd/cloudstic/flags.go | 54 +++---- cmd/cloudstic/store.go | 254 ++++++++++++++++++++---------- cmd/cloudstic/store_test.go | 102 ++++++++++++ cmd/cloudstic/usage.go | 65 +++++--- docs/user-guide.md | 186 ++++++++++------------ e2e/e2e_test.go | 31 ++-- e2e/local.go | 4 +- e2e/minio.go | 7 +- e2e/portable_darwin.go | 2 +- e2e/portable_linux.go | 2 +- e2e/sftp.go | 12 +- internal/core/models.go | 11 +- internal/engine/backup.go | 1 - internal/engine/policy.go | 11 +- internal/engine/policy_test.go | 28 ++++ pkg/source/gdrive.go | 1 - scripts/benchmark/affinity.sh | 12 +- scripts/benchmark/run.sh | 6 +- scripts/test_hamt.sh | 4 +- scripts/test_repack.sh | 8 +- scripts/test_selective_restore.sh | 2 +- 27 files changed, 620 insertions(+), 422 deletions(-) diff --git a/cmd/cloudstic/cmd_backup.go b/cmd/cloudstic/cmd_backup.go index 6317b35..d86198f 100644 --- a/cmd/cloudstic/cmd_backup.go +++ b/cmd/cloudstic/cmd_backup.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "flag" "fmt" - "os" "strings" "time" @@ -18,14 +17,17 @@ import ( type backupArgs struct { g *globalFlags - sourceType string - sourcePath string + sourceURI string driveID string rootFolder string dryRun bool excludeFile string skipNativeFiles bool volumeUUID string + googleCreds string + googleTokenFile string + onedriveClientID string + onedriveTokenFile string tags stringArrayFlags excludes stringArrayFlags } @@ -34,25 +36,31 @@ func parseBackupArgs() *backupArgs { fs := flag.NewFlagSet("backup", flag.ExitOnError) a := &backupArgs{} a.g = addGlobalFlags(fs) - sourceType := fs.String("source", envDefault("CLOUDSTIC_SOURCE", "gdrive"), "source type (gdrive, gdrive-changes, local, onedrive, onedrive-changes)") - sourcePath := fs.String("source-path", envDefault("CLOUDSTIC_SOURCE_PATH", "."), "Local source path (if source=local)") + sourceURI := fs.String("source", envDefault("CLOUDSTIC_SOURCE", "gdrive"), "Source URI: local:, sftp://[user@]host[:port]/, gdrive, gdrive-changes, onedrive, onedrive-changes") driveID := fs.String("drive-id", envDefault("CLOUDSTIC_DRIVE_ID", ""), "Shared drive ID for gdrive source (omit for My Drive)") rootFolder := fs.String("root-folder", envDefault("CLOUDSTIC_ROOT_FOLDER", ""), "Root folder ID for gdrive source (defaults to entire drive)") dryRun := fs.Bool("dry-run", false, "Scan source and report changes without writing to the store") skipNativeFiles := fs.Bool("skip-native-files", false, "Exclude Google-native files (Docs, Sheets, Slides, etc.) from the backup") excludeFile := fs.String("exclude-file", "", "Path to file with exclude patterns (one per line, gitignore syntax)") volumeUUID := fs.String("volume-uuid", envDefault("CLOUDSTIC_VOLUME_UUID", ""), "Override volume UUID for local source (enables cross-machine incremental backup)") + googleCreds := fs.String("google-credentials", envDefault("GOOGLE_APPLICATION_CREDENTIALS", ""), "Path to Google service account credentials JSON file") + googleTokenFile := fs.String("google-token-file", envDefault("GOOGLE_TOKEN_FILE", ""), "Path to Google OAuth token file") + onedriveClientID := fs.String("onedrive-client-id", envDefault("ONEDRIVE_CLIENT_ID", ""), "OneDrive OAuth client ID") + onedriveTokenFile := fs.String("onedrive-token-file", envDefault("ONEDRIVE_TOKEN_FILE", ""), "Path to OneDrive OAuth token file") fs.Var(&a.tags, "tag", "Tag to apply to the snapshot (can be specified multiple times)") fs.Var(&a.excludes, "exclude", "Exclude pattern (gitignore syntax, repeatable)") mustParse(fs) - a.sourceType = *sourceType - a.sourcePath = *sourcePath + a.sourceURI = *sourceURI a.driveID = *driveID a.rootFolder = *rootFolder a.dryRun = *dryRun a.skipNativeFiles = *skipNativeFiles a.excludeFile = *excludeFile a.volumeUUID = *volumeUUID + a.googleCreds = *googleCreds + a.googleTokenFile = *googleTokenFile + a.onedriveClientID = *onedriveClientID + a.onedriveTokenFile = *onedriveTokenFile return a } @@ -66,7 +74,7 @@ func (r *runner) runBackup() int { ctx := context.Background() - src, err := initSource(ctx, a.sourceType, a.sourcePath, a.driveID, a.rootFolder, a.skipNativeFiles, a.volumeUUID, a.g, excludePatterns) + src, err := initSource(ctx, a.sourceURI, a.driveID, a.rootFolder, a.skipNativeFiles, a.volumeUUID, a.googleCreds, a.googleTokenFile, a.onedriveClientID, a.onedriveTokenFile, a.g, excludePatterns) if err != nil { return r.fail("Failed to init source: %v", err) } @@ -138,32 +146,30 @@ func (r *runner) printBackupSummary(res *engine.RunResult) { } } -func initSource(ctx context.Context, sourceType, sourcePath, driveID, rootFolder string, skipNativeFiles bool, volumeUUID string, g *globalFlags, excludePatterns []string) (source.Source, error) { - switch sourceType { +func initSource(ctx context.Context, sourceURI, driveID, rootFolder string, skipNativeFiles bool, volumeUUID, googleCreds, googleTokenFile, onedriveClientID, onedriveTokenFile string, g *globalFlags, excludePatterns []string) (source.Source, error) { + uri, err := parseSourceURI(sourceURI) + if err != nil { + return nil, err + } + + switch uri.scheme { case "local": opts := []source.LocalOption{source.WithLocalExcludePatterns(excludePatterns)} if volumeUUID != "" { opts = append(opts, source.WithVolumeUUID(volumeUUID)) } - return source.NewLocalSource(sourcePath, opts...), nil + return source.NewLocalSource(uri.path, opts...), nil case "sftp": - sftpHost, sftpOpts := g.sftpSourceOpts(g.sourceSFTPHost, g.sourceSFTPPort, g.sourceSFTPUser, g.sourceSFTPPassword, g.sourceSFTPKey, &sourcePath) - if sftpHost == "" { - return nil, fmt.Errorf("--sftp-host is required for sftp source") - } - if sourcePath == "" { - return nil, fmt.Errorf("-source-path is required for sftp source") - } + sftpOpts := g.buildSFTPSourceOpts(uri) sftpOpts = append(sftpOpts, source.WithSFTPExcludePatterns(excludePatterns)) - return source.NewSFTPSource(sftpHost, sftpOpts...) + return source.NewSFTPSource(uri.host, sftpOpts...) case "gdrive": - creds := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") // optional; uses built-in OAuth client when empty - tokenPath, err := resolveTokenPath("GOOGLE_TOKEN_FILE", "google_token.json") + tokenPath, err := resolveTokenPath(googleTokenFile, "google_token.json") if err != nil { return nil, err } gdriveOpts := []source.GDriveOption{ - source.WithCredsPath(creds), + source.WithCredsPath(googleCreds), source.WithTokenPath(tokenPath), source.WithDriveID(driveID), source.WithRootFolderID(rootFolder), @@ -174,13 +180,12 @@ func initSource(ctx context.Context, sourceType, sourcePath, driveID, rootFolder } return source.NewGDriveSource(ctx, gdriveOpts...) case "gdrive-changes": - creds := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") // optional; uses built-in OAuth client when empty - tokenPath, err := resolveTokenPath("GOOGLE_TOKEN_FILE", "google_token.json") + tokenPath, err := resolveTokenPath(googleTokenFile, "google_token.json") if err != nil { return nil, err } gdriveOpts := []source.GDriveOption{ - source.WithCredsPath(creds), + source.WithCredsPath(googleCreds), source.WithTokenPath(tokenPath), source.WithDriveID(driveID), source.WithRootFolderID(rootFolder), @@ -191,38 +196,35 @@ func initSource(ctx context.Context, sourceType, sourcePath, driveID, rootFolder } return source.NewGDriveChangeSource(ctx, gdriveOpts...) case "onedrive": - clientID := os.Getenv("ONEDRIVE_CLIENT_ID") // optional; uses built-in OAuth client when empty - tokenPath, err := resolveTokenPath("ONEDRIVE_TOKEN_FILE", "onedrive_token.json") + tokenPath, err := resolveTokenPath(onedriveTokenFile, "onedrive_token.json") if err != nil { return nil, err } return source.NewOneDriveSource(ctx, - source.WithOneDriveClientID(clientID), + source.WithOneDriveClientID(onedriveClientID), source.WithOneDriveTokenPath(tokenPath), source.WithOneDriveExcludePatterns(excludePatterns), ) case "onedrive-changes": - clientID := os.Getenv("ONEDRIVE_CLIENT_ID") // optional; uses built-in OAuth client when empty - tokenPath, err := resolveTokenPath("ONEDRIVE_TOKEN_FILE", "onedrive_token.json") + tokenPath, err := resolveTokenPath(onedriveTokenFile, "onedrive_token.json") if err != nil { return nil, err } return source.NewOneDriveChangeSource(ctx, - source.WithOneDriveClientID(clientID), + source.WithOneDriveClientID(onedriveClientID), source.WithOneDriveTokenPath(tokenPath), source.WithOneDriveExcludePatterns(excludePatterns), ) default: - return nil, fmt.Errorf("unsupported source type: %s", sourceType) + return nil, fmt.Errorf("unsupported source: %s", uri.scheme) } } -// resolveTokenPath returns an absolute path for a token file. If the -// environment variable envKey is set, that value is used as-is. Otherwise -// the filename is placed inside the cloudstic config directory. -func resolveTokenPath(envKey, defaultFilename string) (string, error) { - if v := os.Getenv(envKey); v != "" { - return v, nil +// resolveTokenPath returns the token file path to use. If explicit is non-empty +// it is used as-is; otherwise the filename is placed in the cloudstic config dir. +func resolveTokenPath(explicit, defaultFilename string) (string, error) { + if explicit != "" { + return explicit, nil } return paths.TokenPath(defaultFilename) } diff --git a/cmd/cloudstic/cmd_check.go b/cmd/cloudstic/cmd_check.go index fea857e..32c8f87 100644 --- a/cmd/cloudstic/cmd_check.go +++ b/cmd/cloudstic/cmd_check.go @@ -19,12 +19,11 @@ func parseCheckArgs() *checkArgs { a := &checkArgs{} a.g = addGlobalFlags(fs) readData := fs.Bool("read-data", false, "Re-hash all chunk data for full byte-level verification") - snapshotFlag := fs.String("snapshot", "", "Check a specific snapshot (default: all)") mustParse(fs) a.readData = *readData - a.snapshotRef = *snapshotFlag - if a.snapshotRef == "" && fs.NArg() > 0 { - a.snapshotRef = fs.Arg(0) + a.snapshotRef = fs.Arg(0) + if a.snapshotRef == "" { + a.snapshotRef = "latest" } return a } diff --git a/cmd/cloudstic/cmd_forget.go b/cmd/cloudstic/cmd_forget.go index 99f8180..6c8922b 100644 --- a/cmd/cloudstic/cmd_forget.go +++ b/cmd/cloudstic/cmd_forget.go @@ -23,8 +23,8 @@ type forgetArgs struct { keepYearly int filterTags stringArrayFlags filterSource string - filterAccount string filterPath string + filterAccount string groupBy string snapshotID string hasPolicy bool @@ -43,9 +43,8 @@ func parseForgetArgs() *forgetArgs { keepMonthly := fs.Int("keep-monthly", 0, "Keep n monthly snapshots") keepYearly := fs.Int("keep-yearly", 0, "Keep n yearly snapshots") fs.Var(&a.filterTags, "tag", "Filter by tag (can be specified multiple times)") - filterSource := fs.String("source", "", "Filter by source type") + filterSource := fs.String("source", "", "Filter by source URI (e.g. local:./docs, gdrive)") filterAccount := fs.String("account", "", "Filter by account") - filterPath := fs.String("path", "", "Filter by path") groupBy := fs.String("group-by", "source,account,path", "Group snapshots by fields (comma-separated)") mustParse(fs) a.prune = *prune @@ -56,10 +55,23 @@ func parseForgetArgs() *forgetArgs { a.keepWeekly = *keepWeekly a.keepMonthly = *keepMonthly a.keepYearly = *keepYearly - a.filterSource = *filterSource a.filterAccount = *filterAccount - a.filterPath = *filterPath a.groupBy = *groupBy + if *filterSource != "" { + // Allow bare source type keywords (e.g. "local", "sftp") without a path for type-only filtering. + switch *filterSource { + case "local", "sftp", "gdrive", "gdrive-changes", "onedrive", "onedrive-changes": + a.filterSource = *filterSource + default: + parts, err := parseSourceURI(*filterSource) + if err != nil { + fmt.Fprintf(os.Stderr, "Invalid -source filter: %v\n", err) + os.Exit(1) + } + a.filterSource = parts.scheme + a.filterPath = parts.path + } + } a.hasPolicy = a.keepLast > 0 || a.keepHourly > 0 || a.keepDaily > 0 || a.keepWeekly > 0 || a.keepMonthly > 0 || a.keepYearly > 0 a.snapshotID = fs.Arg(0) diff --git a/cmd/cloudstic/cmd_init.go b/cmd/cloudstic/cmd_init.go index 6b9a919..d8cb4c1 100644 --- a/cmd/cloudstic/cmd_init.go +++ b/cmd/cloudstic/cmd_init.go @@ -23,7 +23,7 @@ func parseInitArgs() *initArgs { fs := flag.NewFlagSet("init", flag.ExitOnError) a := &initArgs{} a.g = addGlobalFlags(fs) - recovery := fs.Bool("recovery", false, "Generate a recovery key (24-word seed phrase) during init") + recovery := fs.Bool("add-recovery-key", false, "Generate a recovery key (24-word seed phrase) during init") noEncryption := fs.Bool("no-encryption", false, "Create an unencrypted repository (NOT recommended)") adoptSlots := fs.Bool("adopt-slots", false, "Initialize by adopting existing key slots if found (prevents error if already has slots)") mustParse(fs) @@ -53,11 +53,11 @@ func (r *runner) runInit() int { if err != nil { return r.fail("Error: %v", err) } - *a.g.encryptionPassword = pw + *a.g.password = pw kc, _ = a.g.buildKeychain(context.Background()) } else { _, _ = fmt.Fprintln(r.errOut, "Error: encryption is required by default.") - _, _ = fmt.Fprintln(r.errOut, "Provide --encryption-password or --encryption-key to encrypt your repository.") + _, _ = fmt.Fprintln(r.errOut, "Provide --password or --encryption-key to encrypt your repository.") _, _ = fmt.Fprintln(r.errOut, "To create an unencrypted repository, pass --no-encryption (not recommended).") return 1 } diff --git a/cmd/cloudstic/completion.go b/cmd/cloudstic/completion.go index e921299..fb35329 100644 --- a/cmd/cloudstic/completion.go +++ b/cmd/cloudstic/completion.go @@ -43,7 +43,7 @@ _cloudstic() { local commands="init backup restore list ls prune forget diff break-lock key cat completion version help" - local global_flags="-store -store-path -store-prefix -s3-endpoint -s3-region -s3-access-key -s3-secret-key -sftp-host -sftp-port -sftp-user -sftp-password -sftp-key -source-sftp-host -source-sftp-port -source-sftp-user -source-sftp-password -source-sftp-key -store-sftp-host -store-sftp-port -store-sftp-user -store-sftp-password -store-sftp-key -encryption-key -encryption-password -recovery-key -kms-key-arn -kms-region -kms-endpoint -enable-packfile -verbose -quiet -debug" + local global_flags="-store -s3-endpoint -s3-region -s3-access-key -s3-secret-key -source-sftp-password -source-sftp-key -store-sftp-password -store-sftp-key -encryption-key -password -recovery-key -kms-key-arn -kms-region -kms-endpoint -disable-packfile -prompt -verbose -quiet -debug" # Identify the subcommand local cmd="" @@ -53,7 +53,7 @@ _cloudstic() { -*) # skip flags and their values case "${words[i]}" in - -store|-store-path|-store-prefix|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-sftp-host|-sftp-port|-sftp-user|-sftp-password|-sftp-key|-source-sftp-host|-source-sftp-port|-source-sftp-user|-source-sftp-password|-source-sftp-key|-store-sftp-host|-store-sftp-port|-store-sftp-user|-store-sftp-password|-store-sftp-key|-encryption-key|-encryption-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-source-path|-drive-id|-root-folder|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-snapshot|-account|-path|-json) + -store|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-drive-id|-root-folder|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account|-json) ((i++)) ;; esac ;; @@ -74,15 +74,15 @@ _cloudstic() { local cmd_flags="" case "$cmd" in init) - cmd_flags="-recovery -no-encryption -adopt-slots" ;; + cmd_flags="-add-recovery-key -no-encryption -adopt-slots" ;; backup) - cmd_flags="-source -source-path -drive-id -root-folder -tag -dry-run" ;; + cmd_flags="-source -drive-id -root-folder -skip-native-files -google-credentials -google-token-file -onedrive-client-id -onedrive-token-file -tag -dry-run" ;; restore) cmd_flags="-output -dry-run" ;; prune) cmd_flags="-dry-run" ;; forget) - cmd_flags="-prune -dry-run -keep-last -keep-hourly -keep-daily -keep-weekly -keep-monthly -keep-yearly -tag -source -account -path -group-by" ;; + cmd_flags="-prune -dry-run -keep-last -keep-hourly -keep-daily -keep-weekly -keep-monthly -keep-yearly -tag -source -account -group-by" ;; cat) cmd_flags="-json -raw" ;; completion) @@ -111,6 +111,8 @@ _cloudstic() { ;; list) cmd_flags="-group" ;; + check) + cmd_flags="-read-data" ;; ls|diff|break-lock|version|help) cmd_flags="" ;; esac @@ -123,12 +125,14 @@ _cloudstic() { # Value completions for specific flags case "$prev" in -store) - COMPREPLY=($(compgen -W "local b2 s3 sftp" -- "$cur")) + # URI completion hint: show scheme prefixes + COMPREPLY=($(compgen -W "local: s3: b2: sftp://" -- "$cur")) return ;; -source) - COMPREPLY=($(compgen -W "local sftp gdrive gdrive-changes onedrive onedrive-changes" -- "$cur")) + # URI completion hint: show scheme prefixes and bare keywords + COMPREPLY=($(compgen -W "local: sftp:// gdrive gdrive-changes onedrive onedrive-changes" -- "$cur")) return ;; - -source-path|-store-path|-sftp-key|-source-sftp-key|-store-sftp-key|-output) + -source-sftp-key|-store-sftp-key|-output) _filedir return ;; esac @@ -165,35 +169,23 @@ _cloudstic() { local -a global_flags global_flags=( - '-store[Storage backend]:backend:(local b2 s3 sftp)' - '-store-path[Local/SFTP path or bucket name]:path:_files' - '-store-prefix[Key prefix for B2/S3 objects]:prefix:' + '-store[Storage backend URI (local:, s3:[/], b2:[/], sftp://[user@]host[:port]/)]:uri:' '-s3-endpoint[S3 compatible endpoint URL]:url:' '-s3-region[S3 region]:region:' '-s3-access-key[S3 access key ID]:key:' '-s3-secret-key[S3 secret access key]:secret:' - '-sftp-host[SFTP server hostname]:host:_hosts' - '-sftp-port[SFTP server port]:port:' - '-sftp-user[SFTP username]:user:_users' - '-sftp-password[SFTP password]:password:' - '-sftp-key[Path to SSH private key]:key:_files' - '-source-sftp-host[Override SFTP source hostname]:host:_hosts' - '-source-sftp-port[Override SFTP source port]:port:' - '-source-sftp-user[Override SFTP source username]:user:' - '-source-sftp-password[Override SFTP source password]:password:' - '-source-sftp-key[Override SFTP source private key]:key:_files' - '-store-sftp-host[Override SFTP store hostname]:host:_hosts' - '-store-sftp-port[Override SFTP store port]:port:' - '-store-sftp-user[Override SFTP store username]:user:' - '-store-sftp-password[Override SFTP store password]:password:' - '-store-sftp-key[Override SFTP store private key]:key:_files' + '-source-sftp-password[SFTP source password]:password:' + '-source-sftp-key[Path to SSH private key for SFTP source]:key:_files' + '-store-sftp-password[SFTP store password]:password:' + '-store-sftp-key[Path to SSH private key for SFTP store]:key:_files' '-encryption-key[Platform key (hex-encoded)]:key:' - '-encryption-password[Password for encryption]:password:' + '-password[Repository password]:password:' '-recovery-key[Recovery key (24-word mnemonic)]:words:' '-kms-key-arn[AWS KMS key ARN]:arn:' '-kms-region[AWS KMS region]:region:' '-kms-endpoint[Custom AWS KMS endpoint]:url:' - '-enable-packfile[Bundle small objects into packs]' + '-disable-packfile[Disable bundling small objects into packs]' + '-prompt[Prompt for password interactively]' '-verbose[Log detailed operations]' '-quiet[Suppress progress bars]' '-debug[Log every store request]' @@ -207,7 +199,7 @@ _cloudstic() { -*) # Skip flags with values case "${words[i]}" in - -store|-store-path|-store-prefix|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-sftp-host|-sftp-port|-sftp-user|-sftp-password|-sftp-key|-source-sftp-host|-source-sftp-port|-source-sftp-user|-source-sftp-password|-source-sftp-key|-store-sftp-host|-store-sftp-port|-store-sftp-user|-store-sftp-password|-source-sftp-key|-store-sftp-host|-store-sftp-port|-store-sftp-user|-store-sftp-password|-store-sftp-key|-encryption-key|-encryption-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-source-path|-drive-id|-root-folder|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-snapshot|-account|-path) + -store|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-drive-id|-root-folder|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account) (( i++ )) ;; esac ;; @@ -228,16 +220,20 @@ _cloudstic() { case "$cmd" in init) _arguments $global_flags \ - '-recovery[Generate a 24-word recovery key]' \ + '-add-recovery-key[Generate a 24-word recovery key]' \ '-no-encryption[Create an unencrypted repository]' \ '-adopt-slots[Adopt existing key slots]' ;; backup) _arguments $global_flags \ - '-source[Source type]:type:(local sftp gdrive gdrive-changes onedrive onedrive-changes)' \ - '-source-path[Path to source directory]:path:_files' \ + '-source[Source URI]:uri:(local: sftp:// gdrive gdrive-changes onedrive onedrive-changes)' \ '-drive-id[Shared drive ID]:id:' \ '-root-folder[Root folder ID]:id:' \ + '-skip-native-files[Exclude Google-native files]' \ + '-google-credentials[Google service account credentials JSON]:path:_files' \ + '-google-token-file[Google OAuth token file]:path:_files' \ + '-onedrive-client-id[OneDrive OAuth client ID]:id:' \ + '-onedrive-token-file[OneDrive OAuth token file]:path:_files' \ '*-tag[Tag for the snapshot]:tag:' \ '-dry-run[Scan without writing]' ;; @@ -270,9 +266,8 @@ _cloudstic() { '-keep-monthly[Keep N monthly snapshots]:count:' \ '-keep-yearly[Keep N yearly snapshots]:count:' \ '*-tag[Filter by tag]:tag:' \ - '-source[Filter by source type]:type:' \ + '-source[Filter by source URI (e.g. local:./docs, gdrive)]:uri:' \ '-account[Filter by account]:account:' \ - '-path[Filter by path]:path:' \ '-group-by[Group snapshots by fields]:fields:' \ ':snapshot ID:' ;; @@ -355,49 +350,41 @@ complete -c cloudstic -n __fish_use_subcommand -a version -d 'Print version info complete -c cloudstic -n __fish_use_subcommand -a help -d 'Show usage information' # Global flags (available for all subcommands) -complete -c cloudstic -l store -x -a 'local b2 s3 sftp' -d 'Storage backend' -complete -c cloudstic -l store-path -r -F -d 'Local/SFTP path or bucket name' -complete -c cloudstic -l store-prefix -x -d 'Key prefix for B2/S3 objects' +complete -c cloudstic -l store -x -d 'Storage backend URI (local:, s3:[/], b2:[/], sftp://[user@]host[:port]/)' complete -c cloudstic -l s3-endpoint -x -d 'S3 compatible endpoint URL' complete -c cloudstic -l s3-region -x -d 'S3 region' complete -c cloudstic -l s3-access-key -x -d 'S3 access key ID' complete -c cloudstic -l s3-secret-key -x -d 'S3 secret access key' -complete -c cloudstic -l sftp-host -x -d 'SFTP server hostname' -complete -c cloudstic -l sftp-port -x -d 'SFTP server port' -complete -c cloudstic -l sftp-user -x -d 'SFTP username' -complete -c cloudstic -l sftp-password -x -d 'SFTP password' -complete -c cloudstic -l sftp-key -r -F -d 'Path to SSH private key' -complete -c cloudstic -l source-sftp-host -x -d 'Override: SFTP source hostname' -complete -c cloudstic -l source-sftp-port -x -d 'Override: SFTP source port' -complete -c cloudstic -l source-sftp-user -x -d 'Override: SFTP source username' -complete -c cloudstic -l source-sftp-password -x -d 'Override: SFTP source password' -complete -c cloudstic -l source-sftp-key -r -F -d 'Override: SFTP source private key' -complete -c cloudstic -l store-sftp-host -x -d 'Override: SFTP store hostname' -complete -c cloudstic -l store-sftp-port -x -d 'Override: SFTP store port' -complete -c cloudstic -l store-sftp-user -x -d 'Override: SFTP store username' -complete -c cloudstic -l store-sftp-password -x -d 'Override: SFTP store password' -complete -c cloudstic -l store-sftp-key -r -F -d 'Override: SFTP store private key' +complete -c cloudstic -l source-sftp-password -x -d 'SFTP source password' +complete -c cloudstic -l source-sftp-key -r -F -d 'Path to SSH private key for SFTP source' +complete -c cloudstic -l store-sftp-password -x -d 'SFTP store password' +complete -c cloudstic -l store-sftp-key -r -F -d 'Path to SSH private key for SFTP store' complete -c cloudstic -l encryption-key -x -d 'Platform key (hex-encoded)' -complete -c cloudstic -l encryption-password -x -d 'Password for encryption' +complete -c cloudstic -l password -x -d 'Repository password' complete -c cloudstic -l recovery-key -x -d 'Recovery key (24-word mnemonic)' complete -c cloudstic -l kms-key-arn -x -d 'AWS KMS key ARN' complete -c cloudstic -l kms-region -x -d 'AWS KMS region' complete -c cloudstic -l kms-endpoint -x -d 'Custom AWS KMS endpoint' -complete -c cloudstic -l enable-packfile -d 'Bundle small objects into packs' +complete -c cloudstic -l disable-packfile -d 'Disable bundling small objects into packs' +complete -c cloudstic -l prompt -d 'Prompt for password interactively' complete -c cloudstic -l verbose -d 'Log detailed operations' complete -c cloudstic -l quiet -d 'Suppress progress bars' complete -c cloudstic -l debug -d 'Log every store request' # init -complete -c cloudstic -n '__fish_seen_subcommand_from init' -l recovery -d 'Generate a 24-word recovery key' +complete -c cloudstic -n '__fish_seen_subcommand_from init' -l add-recovery-key -d 'Generate a 24-word recovery key' complete -c cloudstic -n '__fish_seen_subcommand_from init' -l no-encryption -d 'Create an unencrypted repository' complete -c cloudstic -n '__fish_seen_subcommand_from init' -l adopt-slots -d 'Adopt existing key slots' # backup -complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l source -x -a 'local sftp gdrive gdrive-changes onedrive onedrive-changes' -d 'Source type' -complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l source-path -r -F -d 'Path to source directory' +complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l source -x -a 'local: sftp:// gdrive gdrive-changes onedrive onedrive-changes' -d 'Source URI' complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l drive-id -x -d 'Shared drive ID' complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l root-folder -x -d 'Root folder ID' +complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l skip-native-files -d 'Exclude Google-native files' +complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l google-credentials -r -F -d 'Google service account credentials JSON' +complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l google-token-file -r -F -d 'Google OAuth token file' +complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l onedrive-client-id -x -d 'OneDrive OAuth client ID' +complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l onedrive-token-file -r -F -d 'OneDrive OAuth token file' complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l tag -x -d 'Tag for the snapshot' complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l dry-run -d 'Scan without writing' @@ -421,9 +408,8 @@ complete -c cloudstic -n '__fish_seen_subcommand_from forget' -l keep-weekly -x complete -c cloudstic -n '__fish_seen_subcommand_from forget' -l keep-monthly -x -d 'Keep N monthly snapshots' complete -c cloudstic -n '__fish_seen_subcommand_from forget' -l keep-yearly -x -d 'Keep N yearly snapshots' complete -c cloudstic -n '__fish_seen_subcommand_from forget' -l tag -x -d 'Filter by tag' -complete -c cloudstic -n '__fish_seen_subcommand_from forget' -l source -x -d 'Filter by source type' +complete -c cloudstic -n '__fish_seen_subcommand_from forget' -l source -x -d 'Filter by source URI (e.g. local:./docs, gdrive)' complete -c cloudstic -n '__fish_seen_subcommand_from forget' -l account -x -d 'Filter by account' -complete -c cloudstic -n '__fish_seen_subcommand_from forget' -l path -x -d 'Filter by path' complete -c cloudstic -n '__fish_seen_subcommand_from forget' -l group-by -x -d 'Group snapshots by fields' # key subcommands diff --git a/cmd/cloudstic/completion_test.go b/cmd/cloudstic/completion_test.go index 9498002..a7956b1 100644 --- a/cmd/cloudstic/completion_test.go +++ b/cmd/cloudstic/completion_test.go @@ -25,11 +25,11 @@ func TestCompletionBash(t *testing.T) { // Key subcommands "list add-recovery passwd", // Global flags - "-store", "-encryption-password", "-verbose", + "-store", "-password", "-verbose", // Command-specific flags - "-dry-run", "-recovery", "-source", "-output", + "-dry-run", "-add-recovery-key", "-source", "-output", // Value completions - "local b2 s3 sftp", + "local: s3: b2: sftp://", "gdrive", "onedrive", } { if !strings.Contains(out, marker) { @@ -62,15 +62,14 @@ func TestCompletionZsh(t *testing.T) { "passwd:Change the repository password", "-new-password[New repository password]", // Global flags with descriptions - "-store[Storage backend]", + "-store[Storage backend URI", "-verbose[Log detailed operations]", // Subcommand-specific flags - "-recovery[Generate a 24-word recovery key]", + "-add-recovery-key[Generate a 24-word recovery key]", "-dry-run[Scan without writing]", "-keep-last[Keep N most recent snapshots]", - // Value completions - "(local b2 s3 sftp)", - "(local sftp gdrive gdrive-changes onedrive onedrive-changes)", + // Value completions (source type list still present) + "(local: sftp:// gdrive gdrive-changes onedrive onedrive-changes)", "(bash zsh fish)", } { if !strings.Contains(out, marker) { @@ -109,10 +108,9 @@ func TestCompletionFish(t *testing.T) { "__fish_seen_subcommand_from forget", "-l dry-run", "-l keep-last", - "-l recovery", - // Value completions - "'local b2 s3 sftp'", - "'local sftp gdrive gdrive-changes onedrive onedrive-changes'", + "-l add-recovery-key", + // Value completions (source type list still present) + "'local: sftp:// gdrive gdrive-changes onedrive onedrive-changes'", "'bash zsh fish'", } { if !strings.Contains(out, marker) { diff --git a/cmd/cloudstic/flags.go b/cmd/cloudstic/flags.go index da1143e..9938730 100644 --- a/cmd/cloudstic/flags.go +++ b/cmd/cloudstic/flags.go @@ -16,58 +16,48 @@ func envDefault(key, fallback string) string { return fallback } +func envBool(key string) bool { + v := os.Getenv(key) + return v == "1" || v == "true" +} + type globalFlags struct { - storeType, storePath, storePrefix *string + store *string s3Endpoint, s3Region *string s3AccessKey, s3SecretKey *string - sftpHost, sftpPort *string - sftpUser, sftpPassword, sftpKey *string - sourceSFTPHost, sourceSFTPPort *string - sourceSFTPUser, sourceSFTPPassword, sourceSFTPKey *string - storeSFTPHost, storeSFTPPort *string - storeSFTPUser, storeSFTPPassword, storeSFTPKey *string - encryptionKey, encryptionPassword *string + sourceSFTPPassword, sourceSFTPKey *string + storeSFTPPassword, storeSFTPKey *string + encryptionKey *string + password *string recoveryKey *string kmsKeyARN, kmsRegion, kmsEndpoint *string - enablePackfile *bool - password, verbose, quiet, debug *bool + disablePackfile *bool + prompt, verbose, quiet, debug *bool debugLog *ui.SafeLogWriter } func addGlobalFlags(fs *flag.FlagSet) *globalFlags { g := &globalFlags{} - g.storeType = fs.String("store", envDefault("CLOUDSTIC_STORE", "local"), "store type (local, b2, s3, sftp)") - g.storePath = fs.String("store-path", envDefault("CLOUDSTIC_STORE_PATH", "./backup_store"), "Local/SFTP path or B2/S3 bucket name") - g.storePrefix = fs.String("store-prefix", envDefault("CLOUDSTIC_STORE_PREFIX", ""), "Key prefix for B2/S3 objects") - g.s3Endpoint = fs.String("s3-endpoint", envDefault("CLOUDSTIC_S3_ENDPOINT", ""), "S3 compatible endpoint URL") + g.store = fs.String("store", envDefault("CLOUDSTIC_STORE", "local:./backup_store"), "Storage backend URI: local:, s3:[/], b2:[/], sftp://[user@]host[:port]/") + g.s3Endpoint = fs.String("s3-endpoint", envDefault("CLOUDSTIC_S3_ENDPOINT", ""), "S3 compatible endpoint URL (for MinIO, R2, etc.)") g.s3Region = fs.String("s3-region", envDefault("CLOUDSTIC_S3_REGION", "us-east-1"), "S3 region") g.s3AccessKey = fs.String("s3-access-key", envDefault("AWS_ACCESS_KEY_ID", ""), "S3 access key ID") g.s3SecretKey = fs.String("s3-secret-key", envDefault("AWS_SECRET_ACCESS_KEY", ""), "S3 secret access key") - g.sftpHost = fs.String("sftp-host", envDefault("CLOUDSTIC_SFTP_HOST", ""), "SFTP server hostname") - g.sftpPort = fs.String("sftp-port", envDefault("CLOUDSTIC_SFTP_PORT", "22"), "SFTP server port") - g.sftpUser = fs.String("sftp-user", envDefault("CLOUDSTIC_SFTP_USER", ""), "SFTP username") - g.sftpPassword = fs.String("sftp-password", envDefault("CLOUDSTIC_SFTP_PASSWORD", ""), "SFTP password") - g.sftpKey = fs.String("sftp-key", envDefault("CLOUDSTIC_SFTP_KEY", ""), "Path to SSH private key") - g.sourceSFTPHost = fs.String("source-sftp-host", "", "Override: SFTP source hostname") - g.sourceSFTPPort = fs.String("source-sftp-port", "", "Override: SFTP source port") - g.sourceSFTPUser = fs.String("source-sftp-user", "", "Override: SFTP source username") - g.sourceSFTPPassword = fs.String("source-sftp-password", "", "Override: SFTP source password") - g.sourceSFTPKey = fs.String("source-sftp-key", "", "Override: SFTP source private key") + g.sourceSFTPPassword = fs.String("source-sftp-password", envDefault("CLOUDSTIC_SOURCE_SFTP_PASSWORD", ""), "SFTP source password") + g.sourceSFTPKey = fs.String("source-sftp-key", envDefault("CLOUDSTIC_SOURCE_SFTP_KEY", ""), "Path to SSH private key for SFTP source") + + g.storeSFTPPassword = fs.String("store-sftp-password", envDefault("CLOUDSTIC_STORE_SFTP_PASSWORD", ""), "SFTP store password") + g.storeSFTPKey = fs.String("store-sftp-key", envDefault("CLOUDSTIC_STORE_SFTP_KEY", ""), "Path to SSH private key for SFTP store") - g.storeSFTPHost = fs.String("store-sftp-host", "", "Override: SFTP store hostname") - g.storeSFTPPort = fs.String("store-sftp-port", "", "Override: SFTP store port") - g.storeSFTPUser = fs.String("store-sftp-user", "", "Override: SFTP store username") - g.storeSFTPPassword = fs.String("store-sftp-password", "", "Override: SFTP store password") - g.storeSFTPKey = fs.String("store-sftp-key", "", "Override: SFTP store private key") g.encryptionKey = fs.String("encryption-key", envDefault("CLOUDSTIC_ENCRYPTION_KEY", ""), "Platform key (hex-encoded, 32 bytes)") - g.encryptionPassword = fs.String("encryption-password", envDefault("CLOUDSTIC_ENCRYPTION_PASSWORD", ""), "Password for password-based encryption") + g.password = fs.String("password", envDefault("CLOUDSTIC_PASSWORD", ""), "Repository password") g.recoveryKey = fs.String("recovery-key", envDefault("CLOUDSTIC_RECOVERY_KEY", ""), "Recovery key (BIP39 24-word mnemonic)") g.kmsKeyARN = fs.String("kms-key-arn", envDefault("CLOUDSTIC_KMS_KEY_ARN", ""), "AWS KMS key ARN for kms-platform slots") g.kmsRegion = fs.String("kms-region", envDefault("CLOUDSTIC_KMS_REGION", ""), "AWS KMS region (defaults to us-east-1)") g.kmsEndpoint = fs.String("kms-endpoint", envDefault("CLOUDSTIC_KMS_ENDPOINT", ""), "Custom AWS KMS endpoint URL") - g.enablePackfile = fs.Bool("enable-packfile", true, "Bundle small objects into 8MB packs to save S3 PUTs") - g.password = fs.Bool("password", false, "Prompt for a password (used alongside --encryption-key or --kms-key-arn to add a password layer)") + g.disablePackfile = fs.Bool("disable-packfile", envBool("CLOUDSTIC_DISABLE_PACKFILE"), "Disable bundling small objects into 8MB packs") + g.prompt = fs.Bool("prompt", false, "Prompt for password interactively (use alongside --encryption-key or --kms-key-arn to add a password layer)") g.verbose = fs.Bool("verbose", false, "Log detailed file-level operations") g.quiet = fs.Bool("quiet", false, "Suppress progress bars (keeps final summary)") g.debug = fs.Bool("debug", false, "Log every store request (network calls, timing, sizes)") diff --git a/cmd/cloudstic/store.go b/cmd/cloudstic/store.go index 9204e47..3df5f05 100644 --- a/cmd/cloudstic/store.go +++ b/cmd/cloudstic/store.go @@ -4,7 +4,9 @@ import ( "context" "encoding/hex" "fmt" + "net/url" "os" + "strings" cloudstic "github.com/cloudstic/cli" "github.com/cloudstic/cli/internal/logger" @@ -46,7 +48,7 @@ func (g *globalFlags) openClient() (*cloudstic.Client, error) { } raw = g.applyDebug(raw) - packfileEnabled := g.enablePackfile != nil && *g.enablePackfile + packfileEnabled := g.disablePackfile == nil || !*g.disablePackfile var reporter cloudstic.Reporter if *g.quiet { @@ -111,13 +113,13 @@ func (g *globalFlags) buildKeychain(ctx context.Context) (keychain.Chain, error) if len(platformKey) > 0 { chain = append(chain, keychain.WithPlatformKey(platformKey)) } - if *g.encryptionPassword != "" { - chain = append(chain, keychain.WithPassword(*g.encryptionPassword)) + if *g.password != "" { + chain = append(chain, keychain.WithPassword(*g.password)) } if *g.recoveryKey != "" { chain = append(chain, keychain.WithRecoveryKey(*g.recoveryKey)) } - promptRequested := g.password != nil && *g.password + promptRequested := g.prompt != nil && *g.prompt if (len(chain) == 0 || promptRequested) && term.IsTerminal(os.Stdin.Fd()) { chain = append(chain, keychain.WithPrompt( func() (string, error) { return ui.PromptPassword("Repository password") }, @@ -143,40 +145,107 @@ func (g *globalFlags) parsePlatformKey() ([]byte, error) { return platformKey, nil } +// storeURIParts holds the parsed components of a --store URI. +type storeURIParts struct { + scheme string // "local", "s3", "b2", "sftp" + // S3/B2 fields + bucket string + prefix string + // local field + path string + // SFTP fields + host string + port string + user string +} + +// parseStoreURI parses a --store flag value into its components. +// +// Supported formats: +// +// local: e.g. local:./backup_store +// s3:[/] e.g. s3:my-bucket or s3:my-bucket/prod +// b2:[/] e.g. b2:my-bucket or b2:my-bucket/prod +// sftp://[user@]host[:port]/ e.g. sftp://backup@host.com/backups +func parseStoreURI(raw string) (*storeURIParts, error) { + if strings.HasPrefix(raw, "sftp://") { + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("invalid store URI %q: %w", raw, err) + } + if u.Hostname() == "" { + return nil, fmt.Errorf("invalid store URI %q: sftp URI must include a hostname", raw) + } + user := "" + if u.User != nil { + user = u.User.Username() + } + return &storeURIParts{ + scheme: "sftp", + host: u.Hostname(), + port: u.Port(), + user: user, + path: u.Path, + }, nil + } + + idx := strings.IndexByte(raw, ':') + if idx < 0 { + return nil, fmt.Errorf("invalid store URI %q: missing scheme (e.g. local:./path, s3:bucket, b2:bucket, sftp://host/path)", raw) + } + scheme := raw[:idx] + rest := raw[idx+1:] + + switch scheme { + case "local": + if rest == "" { + return nil, fmt.Errorf("invalid store URI %q: local path cannot be empty", raw) + } + return &storeURIParts{scheme: "local", path: rest}, nil + case "s3", "b2": + if rest == "" { + return nil, fmt.Errorf("invalid store URI %q: bucket name cannot be empty", raw) + } + bucket, prefix, _ := strings.Cut(rest, "/") + if bucket == "" { + return nil, fmt.Errorf("invalid store URI %q: bucket name cannot be empty", raw) + } + return &storeURIParts{scheme: scheme, bucket: bucket, prefix: prefix}, nil + default: + return nil, fmt.Errorf("unknown store scheme %q in %q: supported schemes are local, s3, b2, sftp", scheme, raw) + } +} + func (g *globalFlags) initObjectStore() (store.ObjectStore, error) { - var inner store.ObjectStore - var err error + uri, err := parseStoreURI(*g.store) + if err != nil { + return nil, err + } - switch *g.storeType { + var inner store.ObjectStore + switch uri.scheme { case "local": - inner, err = store.NewLocalStore(*g.storePath) + inner, err = store.NewLocalStore(uri.path) case "b2": keyID := os.Getenv("B2_KEY_ID") appKey := os.Getenv("B2_APP_KEY") if keyID == "" || appKey == "" { return nil, fmt.Errorf("B2_KEY_ID and B2_APP_KEY env vars required for b2 store") } - inner, err = store.NewB2Store(*g.storePath, store.WithCredentials(keyID, appKey), store.WithPrefix(*g.storePrefix)) + inner, err = store.NewB2Store(uri.bucket, store.WithCredentials(keyID, appKey), store.WithPrefix(uri.prefix)) case "s3": - if *g.storePath == "" { - return nil, fmt.Errorf("-store-path must be set to the S3 bucket name") - } inner, err = store.NewS3Store( context.Background(), - *g.storePath, + uri.bucket, store.WithS3Endpoint(*g.s3Endpoint), store.WithS3Region(*g.s3Region), store.WithS3Credentials(*g.s3AccessKey, *g.s3SecretKey), - store.WithS3Prefix(*g.storePrefix), + store.WithS3Prefix(uri.prefix), ) case "sftp": - sftpHost, sftpOpts := g.sftpStoreOpts(g.storeSFTPHost, g.storeSFTPPort, g.storeSFTPUser, g.storeSFTPPassword, g.storeSFTPKey, g.storePath) - if sftpHost == "" { - return nil, fmt.Errorf("--sftp-host is required for sftp store") - } - inner, err = store.NewSFTPStore(sftpHost, sftpOpts...) + inner, err = store.NewSFTPStore(uri.host, g.buildSFTPStoreOpts(uri)...) default: - return nil, fmt.Errorf("unsupported store type: %s", *g.storeType) + return nil, fmt.Errorf("unsupported store type: %s", uri.scheme) } if err != nil { @@ -186,92 +255,107 @@ func (g *globalFlags) initObjectStore() (store.ObjectStore, error) { return inner, nil } -func (g *globalFlags) sftpStoreOpts(host, port, user, pass, key, path *string) (string, []store.SFTPStoreOption) { - h := *host - if h == "" { - h = *g.sftpHost - } - p := *port - if p == "" { - p = *g.sftpPort - } - u := *user - if u == "" { - u = *g.sftpUser - } - pw := *pass - if pw == "" { - pw = *g.sftpPassword - } - k := *key - if k == "" { - k = *g.sftpKey - } - bp := *path - - if h == "" { - return "", nil - } - +func (g *globalFlags) buildSFTPStoreOpts(uri *storeURIParts) []store.SFTPStoreOption { opts := []store.SFTPStoreOption{ - store.WithSFTPBasePath(bp), + store.WithSFTPBasePath(uri.path), } - if p != "" { - opts = append(opts, store.WithSFTPPort(p)) + if uri.port != "" { + opts = append(opts, store.WithSFTPPort(uri.port)) } - if u != "" { - opts = append(opts, store.WithSFTPUser(u)) + if uri.user != "" { + opts = append(opts, store.WithSFTPUser(uri.user)) } - if pw != "" { + if pw := *g.storeSFTPPassword; pw != "" { opts = append(opts, store.WithSFTPPassword(pw)) } - if k != "" { + if k := *g.storeSFTPKey; k != "" { opts = append(opts, store.WithSFTPKey(k)) } - return h, opts + return opts } -func (g *globalFlags) sftpSourceOpts(host, port, user, pass, key, path *string) (string, []source.SFTPOption) { - h := *host - if h == "" { - h = *g.sftpHost - } - p := *port - if p == "" { - p = *g.sftpPort - } - u := *user - if u == "" { - u = *g.sftpUser - } - pw := *pass - if pw == "" { - pw = *g.sftpPassword +// sourceURIParts holds the parsed components of a --source URI or keyword. +type sourceURIParts struct { + scheme string // "local", "sftp", "gdrive", "gdrive-changes", "onedrive", "onedrive-changes" + // local/sftp fields + path string + // sftp-specific fields + host string + port string + user string +} + +// parseSourceURI parses a --source flag value into its components. +// +// Supported formats: +// +// local: e.g. local:./documents +// sftp://[user@]host[:port]/ e.g. sftp://backup@host.com/data +// gdrive +// gdrive-changes +// onedrive +// onedrive-changes +func parseSourceURI(raw string) (*sourceURIParts, error) { + if strings.HasPrefix(raw, "sftp://") { + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("invalid source URI %q: %w", raw, err) + } + if u.Hostname() == "" { + return nil, fmt.Errorf("invalid source URI %q: sftp URI must include a hostname", raw) + } + user := "" + if u.User != nil { + user = u.User.Username() + } + return &sourceURIParts{ + scheme: "sftp", + host: u.Hostname(), + port: u.Port(), + user: user, + path: u.Path, + }, nil } - k := *key - if k == "" { - k = *g.sftpKey + + idx := strings.IndexByte(raw, ':') + if idx >= 0 { + scheme := raw[:idx] + rest := raw[idx+1:] + switch scheme { + case "local": + if rest == "" { + return nil, fmt.Errorf("invalid source URI %q: local path cannot be empty", raw) + } + return &sourceURIParts{scheme: "local", path: rest}, nil + default: + return nil, fmt.Errorf("unknown source scheme %q in %q: supported URI formats are local: and sftp://[user@]host[:port]/", scheme, raw) + } } - bp := *path - if h == "" { - return "", nil + // Bare keyword (cloud sources) + switch raw { + case "gdrive", "gdrive-changes", "onedrive", "onedrive-changes": + return &sourceURIParts{scheme: raw}, nil + default: + return nil, fmt.Errorf("unknown source %q: supported values are local:, sftp://[user@]host[:port]/, gdrive, gdrive-changes, onedrive, onedrive-changes", raw) } +} +func (g *globalFlags) buildSFTPSourceOpts(uri *sourceURIParts) []source.SFTPOption { opts := []source.SFTPOption{ - source.WithSFTPSourceBasePath(bp), + source.WithSFTPSourceBasePath(uri.path), } - if p != "" { - opts = append(opts, source.WithSFTPSourcePort(p)) + if uri.port != "" { + opts = append(opts, source.WithSFTPSourcePort(uri.port)) } - if u != "" { - opts = append(opts, source.WithSFTPSourceUser(u)) + if uri.user != "" { + opts = append(opts, source.WithSFTPSourceUser(uri.user)) } - if pw != "" { + if pw := *g.sourceSFTPPassword; pw != "" { opts = append(opts, source.WithSFTPSourcePassword(pw)) } - if k != "" { + if k := *g.sourceSFTPKey; k != "" { opts = append(opts, source.WithSFTPSourceKey(k)) } - return h, opts + return opts } diff --git a/cmd/cloudstic/store_test.go b/cmd/cloudstic/store_test.go index fcaab39..ba1b033 100644 --- a/cmd/cloudstic/store_test.go +++ b/cmd/cloudstic/store_test.go @@ -17,6 +17,108 @@ func newTestLocalStore(t *testing.T) *store.LocalStore { return s } +func TestParseStoreURI(t *testing.T) { + tests := []struct { + raw string + want storeURIParts + wantErr bool + }{ + // local + {raw: "local:./backup_store", want: storeURIParts{scheme: "local", path: "./backup_store"}}, + {raw: "local:/abs/path", want: storeURIParts{scheme: "local", path: "/abs/path"}}, + {raw: "local:", wantErr: true}, + + // s3 + {raw: "s3:my-bucket", want: storeURIParts{scheme: "s3", bucket: "my-bucket"}}, + {raw: "s3:my-bucket/prod", want: storeURIParts{scheme: "s3", bucket: "my-bucket", prefix: "prod"}}, + {raw: "s3:my-bucket/nested/prefix", want: storeURIParts{scheme: "s3", bucket: "my-bucket", prefix: "nested/prefix"}}, + {raw: "s3:", wantErr: true}, + + // b2 + {raw: "b2:my-bucket", want: storeURIParts{scheme: "b2", bucket: "my-bucket"}}, + {raw: "b2:my-bucket/prod", want: storeURIParts{scheme: "b2", bucket: "my-bucket", prefix: "prod"}}, + {raw: "b2:", wantErr: true}, + + // sftp + {raw: "sftp://host.example.com/backups", want: storeURIParts{scheme: "sftp", host: "host.example.com", path: "/backups"}}, + {raw: "sftp://user@host.example.com/backups", want: storeURIParts{scheme: "sftp", host: "host.example.com", user: "user", path: "/backups"}}, + {raw: "sftp://user@host.example.com:2222/backups", want: storeURIParts{scheme: "sftp", host: "host.example.com", port: "2222", user: "user", path: "/backups"}}, + {raw: "sftp://host.example.com:22/backups", want: storeURIParts{scheme: "sftp", host: "host.example.com", port: "22", path: "/backups"}}, + {raw: "sftp:///no-host", wantErr: true}, + + // invalid + {raw: "no-colon", wantErr: true}, + {raw: "unknown:value", wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.raw, func(t *testing.T) { + got, err := parseStoreURI(tc.raw) + if tc.wantErr { + if err == nil { + t.Fatalf("parseStoreURI(%q): expected error, got %+v", tc.raw, got) + } + return + } + if err != nil { + t.Fatalf("parseStoreURI(%q): unexpected error: %v", tc.raw, err) + } + if *got != tc.want { + t.Errorf("parseStoreURI(%q):\n got %+v\n want %+v", tc.raw, *got, tc.want) + } + }) + } +} + +func TestParseSourceURI(t *testing.T) { + tests := []struct { + raw string + want sourceURIParts + wantErr bool + }{ + // local + {raw: "local:./documents", want: sourceURIParts{scheme: "local", path: "./documents"}}, + {raw: "local:/abs/path", want: sourceURIParts{scheme: "local", path: "/abs/path"}}, + {raw: "local:", wantErr: true}, + + // sftp + {raw: "sftp://host.example.com/data", want: sourceURIParts{scheme: "sftp", host: "host.example.com", path: "/data"}}, + {raw: "sftp://user@host.example.com/data", want: sourceURIParts{scheme: "sftp", host: "host.example.com", user: "user", path: "/data"}}, + {raw: "sftp://user@host.example.com:2222/data", want: sourceURIParts{scheme: "sftp", host: "host.example.com", port: "2222", user: "user", path: "/data"}}, + {raw: "sftp:///no-host", wantErr: true}, + + // cloud keywords + {raw: "gdrive", want: sourceURIParts{scheme: "gdrive"}}, + {raw: "gdrive-changes", want: sourceURIParts{scheme: "gdrive-changes"}}, + {raw: "onedrive", want: sourceURIParts{scheme: "onedrive"}}, + {raw: "onedrive-changes", want: sourceURIParts{scheme: "onedrive-changes"}}, + + // invalid + {raw: "sftp", wantErr: true}, + {raw: "local", wantErr: true}, + {raw: "unknown:value", wantErr: true}, + {raw: "unknown-keyword", wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.raw, func(t *testing.T) { + got, err := parseSourceURI(tc.raw) + if tc.wantErr { + if err == nil { + t.Fatalf("parseSourceURI(%q): expected error, got %+v", tc.raw, got) + } + return + } + if err != nil { + t.Fatalf("parseSourceURI(%q): unexpected error: %v", tc.raw, err) + } + if *got != tc.want { + t.Errorf("parseSourceURI(%q):\n got %+v\n want %+v", tc.raw, *got, tc.want) + } + }) + } +} + func TestApplyDebug_Disabled(t *testing.T) { logger.Writer = nil diff --git a/cmd/cloudstic/usage.go b/cmd/cloudstic/usage.go index 7951d90..33fb2ed 100644 --- a/cmd/cloudstic/usage.go +++ b/cmd/cloudstic/usage.go @@ -36,27 +36,33 @@ func printUsage() { t.HeadingSub("GLOBAL OPTIONS", "(also settable via env vars)") t.Flags([][2]string{ - {"-store ", ui.Env("Storage backend: local, b2, s3, sftp", "CLOUDSTIC_STORE")}, - {"-store-path ", ui.Env("Local/SFTP path or B2/S3 bucket name", "CLOUDSTIC_STORE_PATH")}, - {"-store-prefix ", ui.Env("Key prefix for B2/S3 objects", "CLOUDSTIC_STORE_PREFIX")}, + {"-store ", ui.Env("Storage backend URI (see formats below)", "CLOUDSTIC_STORE")}, {"-s3-endpoint ", ui.Env("S3 compatible endpoint (for MinIO, R2, etc.)", "CLOUDSTIC_S3_ENDPOINT")}, {"-s3-region ", ui.Env("S3 region", "CLOUDSTIC_S3_REGION")}, {"-s3-access-key ", ui.Env("S3 Access Key ID", "AWS_ACCESS_KEY_ID")}, {"-s3-secret-key ", ui.Env("S3 Secret Access Key", "AWS_SECRET_ACCESS_KEY")}, - {"-sftp-host ", ui.Env("SFTP server hostname", "CLOUDSTIC_SFTP_HOST")}, - {"-sftp-port ", ui.Env("SFTP server port (default 22)", "CLOUDSTIC_SFTP_PORT")}, - {"-sftp-user ", ui.Env("SFTP username", "CLOUDSTIC_SFTP_USER")}, - {"-sftp-password ", ui.Env("SFTP password", "CLOUDSTIC_SFTP_PASSWORD")}, - {"-sftp-key ", ui.Env("Path to SSH private key", "CLOUDSTIC_SFTP_KEY")}, + {"-source-sftp-password ", ui.Env("SFTP source password", "CLOUDSTIC_SOURCE_SFTP_PASSWORD")}, + {"-source-sftp-key ", ui.Env("Path to SSH private key for SFTP source", "CLOUDSTIC_SOURCE_SFTP_KEY")}, + {"-store-sftp-password ", ui.Env("SFTP store password", "CLOUDSTIC_STORE_SFTP_PASSWORD")}, + {"-store-sftp-key ", ui.Env("Path to SSH private key for SFTP store", "CLOUDSTIC_STORE_SFTP_KEY")}, {"-verbose", "Log detailed file-level operations"}, {"-quiet", "Suppress progress bars (keeps final summary)"}, {"-debug", "Log every store request (network calls, timing, sizes)"}, }) + t.Blank() + t.Note( + "Store URI formats:", + " local: e.g. local:./backup_store", + " s3:[/] e.g. s3:my-bucket or s3:my-bucket/prod", + " b2:[/] e.g. b2:my-bucket or b2:my-bucket/prod", + " sftp://[user@]host[:port]/ e.g. sftp://backup@host.com/backups", + ) t.Heading("ENCRYPTION OPTIONS") t.Flags([][2]string{ + {"-password ", ui.Env("Repository password", "CLOUDSTIC_PASSWORD")}, + {"-prompt", "Prompt for password interactively (use alongside --encryption-key or --kms-key-arn)"}, {"-encryption-key ", ui.Env("Platform key (64 hex chars = 32 bytes)", "CLOUDSTIC_ENCRYPTION_KEY")}, - {"-encryption-password", ui.Env("Password for password-based encryption", "CLOUDSTIC_ENCRYPTION_PASSWORD")}, {"-recovery-key ", ui.Env("Recovery key (24-word seed phrase)", "CLOUDSTIC_RECOVERY_KEY")}, {"-kms-key-arn ", ui.Env("AWS KMS key ARN for kms-platform slots", "CLOUDSTIC_KMS_KEY_ARN")}, {"-kms-region ", ui.Env("AWS KMS region", "CLOUDSTIC_KMS_REGION")}, @@ -64,7 +70,7 @@ func printUsage() { }) t.Blank() t.Note( - "Encryption is required by default (AES-256-GCM). Provide -encryption-password", + "Encryption is required by default (AES-256-GCM). Provide -password", "or -encryption-key when running 'cloudstic init'. Use -recovery-key to open a", "repository with a recovery seed phrase.", ) @@ -73,7 +79,7 @@ func printUsage() { t.Command("init", "") t.Flags([][2]string{ - {"-recovery", "Generate a 24-word recovery key during init"}, + {"-add-recovery-key", "Generate a 24-word recovery key during init"}, {"-no-encryption", "Create an unencrypted repository (not recommended)"}, {"-adopt-slots", "Initialize by adopting existing key slots if found"}, }) @@ -96,22 +102,35 @@ func printUsage() { }) t.Note( " Change the repository password. Provide current credentials via", - " -encryption-password, -encryption-key, or -kms-key-arn to unlock.", + " -password, -encryption-key, or -kms-key-arn to unlock.", ) t.Blank() t.Command("backup", "") t.Flags([][2]string{ - {"-source ", "local, sftp, gdrive, gdrive-changes, onedrive, onedrive-changes"}, - {"-source-path ", "Path to source directory (local or SFTP remote path)"}, + {"-source ", ui.Env("Source URI: local:, sftp://[user@]host[:port]/, gdrive, gdrive-changes, onedrive, onedrive-changes", "CLOUDSTIC_SOURCE")}, {"-drive-id ", "Shared drive ID for gdrive (omit for My Drive)"}, {"-root-folder ", "Root folder ID for gdrive (defaults to entire drive)"}, + {"-skip-native-files", "Exclude Google-native files (Docs, Sheets, Slides, etc.)"}, + {"-google-credentials ", ui.Env("Path to Google service account credentials JSON", "GOOGLE_APPLICATION_CREDENTIALS")}, + {"-google-token-file ", ui.Env("Path to Google OAuth token file", "GOOGLE_TOKEN_FILE")}, + {"-onedrive-client-id ", ui.Env("OneDrive OAuth client ID", "ONEDRIVE_CLIENT_ID")}, + {"-onedrive-token-file ", ui.Env("Path to OneDrive OAuth token file", "ONEDRIVE_TOKEN_FILE")}, {"-tag ", "Tag to apply to the snapshot (repeatable)"}, {"-exclude ", "Exclude pattern, gitignore syntax (repeatable)"}, {"-exclude-file ", "Load exclude patterns from file (one per line, gitignore syntax)"}, {"-dry-run", "Scan source and report changes without writing to the store"}, }) t.Blank() + t.Note( + "Source URI formats:", + " local: e.g. local:./documents", + " sftp://[user@]host[:port]/ e.g. sftp://backup@host.com/data", + " gdrive Google Drive (full scan)", + " gdrive-changes Google Drive (incremental via Changes API)", + " onedrive OneDrive (full scan)", + " onedrive-changes OneDrive (incremental via delta API)", + ) t.Command("restore", "[snapshot_id]") t.Flags([][2]string{ @@ -146,6 +165,10 @@ func printUsage() { {"-keep-weekly N", "Keep N weekly snapshots"}, {"-keep-monthly N", "Keep N monthly snapshots"}, {"-keep-yearly N", "Keep N yearly snapshots"}, + {"-source ", "Filter by source URI (e.g. local:./docs, gdrive, sftp://host/path)"}, + {"-account ", "Filter by account"}, + {"-tag ", "Filter by tag (repeatable)"}, + {"-group-by ", "Group snapshots by fields (default: source,account,path)"}, }) t.Blank() @@ -161,11 +184,10 @@ func printUsage() { t.Command("check", "[snapshot_id]") t.Flags([][2]string{ {"-read-data", "Re-hash all chunk data for full byte-level verification"}, - {"-snapshot ", "Check a specific snapshot (default: all)"}, }) t.Note(" Verify the integrity of the repository by walking the full reference", " chain: index/latest → snapshot → HAMT nodes → filemeta → content → chunks.", - " Reports missing, corrupt, or unreadable objects.") + " Defaults to the latest snapshot. Reports missing, corrupt, or unreadable objects.") t.Blank() t.Command("cat", " [object_key...]") @@ -183,16 +205,17 @@ func printUsage() { t.Heading("EXAMPLES") t.Examples( - `cloudstic init -encryption-password "my secret passphrase"`, - `cloudstic init -encryption-password "my secret passphrase" -recovery`, - "cloudstic backup -source local -source-path ./documents", - "cloudstic backup -source gdrive -store b2 -store-path my-bucket", + `cloudstic init -password "my secret passphrase"`, + `cloudstic init -password "my secret passphrase" -add-recovery-key`, + "cloudstic backup -source local:./documents", + "cloudstic backup -source gdrive -store b2:my-bucket", + "cloudstic backup -source sftp://backup@host.com/data -source-sftp-key ~/.ssh/id_ed25519", "cloudstic list", "cloudstic restore", "cloudstic restore abc123 -output ./my-backup.zip", "cloudstic restore abc123 -path Documents/report.pdf", "cloudstic restore abc123 -path Documents/", - "cloudstic backup -source local -source-path ./documents -dry-run", + "cloudstic backup -source local:./documents -dry-run", "cloudstic prune -dry-run -verbose", ) t.Blank() diff --git a/docs/user-guide.md b/docs/user-guide.md index 17cc4cb..faf6015 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -52,7 +52,7 @@ Cloudstic is a content-addressable backup tool that creates encrypted, deduplica cloudstic init # 2. Back up a local directory (prompts for password) -cloudstic backup -source local -source-path ~/Documents +cloudstic backup -source local:~/Documents # 3. List snapshots cloudstic list @@ -64,8 +64,8 @@ cloudstic restore When running in a terminal, Cloudstic prompts for the repository password if no credential is provided via flags or environment variables. For non-interactive use (scripts, cron), pass the password explicitly: ```bash -cloudstic init -encryption-password "my secret passphrase" -cloudstic backup -source local -source-path ~/Documents -encryption-password "my secret passphrase" +cloudstic init -password "my secret passphrase" +cloudstic backup -source local:~/Documents -password "my secret passphrase" ``` ## Installation @@ -160,14 +160,13 @@ Override with the `CLOUDSTIC_CONFIG_DIR` environment variable. Most flags can be set via environment variables to avoid repeating them. For example: ```bash -export CLOUDSTIC_STORE=s3 -export CLOUDSTIC_STORE_PATH=my-backup-bucket -export CLOUDSTIC_ENCRYPTION_PASSWORD="my secret passphrase" +export CLOUDSTIC_STORE=s3:my-backup-bucket +export CLOUDSTIC_PASSWORD="my secret passphrase" export AWS_ACCESS_KEY_ID=your-access-key export AWS_SECRET_ACCESS_KEY=your-secret-key # Now commands are much shorter: -cloudstic backup -source local -source-path ~/Documents +cloudstic backup -source local:~/Documents cloudstic list cloudstic restore ``` @@ -187,7 +186,7 @@ These flags apply to all commands: | `-verbose` | Log detailed file-level operations (files scanned, written, deleted) | | `-quiet` | Suppress progress bars (keeps final summary output) | | `-debug` | Log every store request (network calls, timing, sizes) | -| `-enable-packfile` | Bundle small objects into 8MB packs to save S3 PUTs (default: true) | +| `-disable-packfile` | Disable bundling small objects into 8MB packs (packfile is on by default) — env: `CLOUDSTIC_DISABLE_PACKFILE=1` | `-verbose` and `-quiet` are mutually exclusive. If both are set, `-quiet` takes precedence. @@ -200,25 +199,25 @@ Initialize a new repository. Encryption is **required by default**. cloudstic init # Interactive with a recovery key (strongly recommended) -cloudstic init -recovery +cloudstic init -add-recovery-key # Non-interactive — password provided via flag -cloudstic init -encryption-password "my secret passphrase" +cloudstic init -password "my secret passphrase" # Non-interactive with a recovery key -cloudstic init -encryption-password "my secret passphrase" -recovery +cloudstic init -password "my secret passphrase" -add-recovery-key # Platform key encryption (for automation) cloudstic init -encryption-key <64-hex-chars> # Both password and platform key (dual access) -cloudstic init -encryption-password "passphrase" -encryption-key +cloudstic init -password "passphrase" -encryption-key # Unencrypted (must be explicit — not recommended) cloudstic init -no-encryption ``` -When no encryption credential is provided and stdin is a terminal, `init` prompts for a new password with confirmation. In non-interactive environments (piped input, cron jobs), you must pass `-encryption-password`, `-encryption-key`, or `-no-encryption` explicitly. +When no encryption credential is provided and stdin is a terminal, `init` prompts for a new password with confirmation. In non-interactive environments (piped input, cron jobs), you must pass `-password`, `-encryption-key`, or `-no-encryption` explicitly. If you are using a platform key or KMS but also want to protect the repository with a password, pass `-password` to explicitly trigger the prompt: @@ -230,14 +229,13 @@ cloudstic init -encryption-key -password | Flag | Description | |------|-------------| -| `-encryption-password` | Password for password-based encryption | +| `-password` | Password for password-based encryption. Omit the value to force an interactive prompt even when other credentials are provided | | `-encryption-key` | Platform key (64 hex chars = 32 bytes) | -| `-password` | Force interactive password prompt (even when other credentials are provided) | -| `-recovery` | Generate a 24-word recovery key during init | +| `-add-recovery-key` | Generate a 24-word recovery key during init | | `-no-encryption` | Create an unencrypted repository (not recommended) | | `-adopt-slots` | Adopt existing key slots (and add new credentials to them) | -When `-recovery` is used, a 24-word seed phrase is displayed **once**. Write it down and store it safely — it's your last resort if you lose your password. +When `-add-recovery-key` is used, a 24-word seed phrase is displayed **once**. Write it down and store it safely — it's your last resort if you lose your password. --- @@ -247,7 +245,7 @@ Create a new snapshot from a source. ```bash # Back up a local directory -cloudstic backup -source local -source-path ~/Documents +cloudstic backup -source local:~/Documents # Back up Google Drive (My Drive) cloudstic backup -source gdrive @@ -256,21 +254,20 @@ cloudstic backup -source gdrive cloudstic backup -source gdrive -drive-id -root-folder # Back up with tags -cloudstic backup -source local -source-path ~/Documents -tag daily -tag important +cloudstic backup -source local:~/Documents -tag daily -tag important # Verbose output (shows individual files) -cloudstic backup -source local -source-path ~/Documents -verbose +cloudstic backup -source local:~/Documents -verbose # Dry run — see what would change without writing to the store -cloudstic backup -source local -source-path ~/Documents -dry-run +cloudstic backup -source local:~/Documents -dry-run ``` **Flags:** | Flag | Default | Description | |------|---------|-------------| -| `-source` | `gdrive` | Source type: `local`, `sftp`, `gdrive`, `gdrive-changes`, `onedrive`, `onedrive-changes` | -| `-source-path` | `.` | Path to source directory (local filesystem or SFTP remote path) | +| `-source` | `gdrive` | Source type: `local:`, `sftp://[user@]host[:port]/`, `gdrive`, `gdrive-changes`, `onedrive`, `onedrive-changes` | | `-drive-id` | | Shared drive ID for Google Drive (omit for My Drive) | | `-root-folder` | | Root folder ID for Google Drive (defaults to entire drive) | | `-tag` | | Tag to apply to the snapshot (repeatable) | @@ -289,17 +286,17 @@ You can exclude files and directories from the backup using gitignore-style patt ```bash # Exclude specific directories and file types -cloudstic backup -source local -source-path ~/project \ +cloudstic backup -source local:~/project \ -exclude ".git/" -exclude "node_modules/" -exclude "*.tmp" -exclude "*.log" # Works with cloud sources too cloudstic backup -source gdrive-changes -exclude "node_modules/" -exclude "*.tmp" # Load patterns from a file -cloudstic backup -source local -source-path ~/project -exclude-file ~/project/.backupignore +cloudstic backup -source local:~/project -exclude-file ~/project/.backupignore # Combine both -cloudstic backup -source local -source-path ~/project \ +cloudstic backup -source local:~/project \ -exclude "build/" -exclude-file .backupignore ``` @@ -469,9 +466,8 @@ cloudstic forget -keep-last 5 -source gdrive -account user@gmail.com | Flag | Description | |------|-------------| | `-tag` | Filter by tag (repeatable) | -| `-source` | Filter by source type | +| `-source` | Filter by source URI (e.g., `local:/path`, `gdrive`, `sftp://host/path`) | | `-account` | Filter by account | -| `-path` | Filter by path | | `-group-by` | Group snapshots by fields (default: `source,account,path`) | **Other flags:** @@ -549,7 +545,7 @@ cloudstic check -read-data # Check a specific snapshot cloudstic check -cloudstic check -snapshot latest +cloudstic check latest # Verbose output — log each verified object cloudstic check -verbose @@ -634,7 +630,7 @@ Generate a 24-word recovery key for an existing encrypted repository. Requires y cloudstic key add-recovery # Non-interactive -cloudstic key add-recovery -encryption-password "my secret passphrase" +cloudstic key add-recovery -password "my secret passphrase" # For KMS-managed repositories cloudstic key add-recovery -kms-key-arn arn:aws:kms:us-east-1:123:key/abc @@ -655,7 +651,7 @@ Change (or add) the repository password. You must provide your current credentia cloudstic key passwd # Non-interactive -cloudstic key passwd -encryption-password "old passphrase" -new-password "new passphrase" +cloudstic key passwd -password "old passphrase" -new-password "new passphrase" # Unlock with platform key, set a password cloudstic key passwd -encryption-key -new-password "my passphrase" @@ -761,7 +757,7 @@ See [Shell Completions](#shell-completions) below for setup instructions. ## Shell Completions -Cloudstic can generate tab-completion scripts for popular shells. Once set up, pressing `Tab` will complete commands, flags, and flag values (like `-store local|b2|s3|sftp` and `-source local|sftp|gdrive|...`). +Cloudstic can generate tab-completion scripts for popular shells. Once set up, pressing `Tab` will complete commands, flags, and flag values (like `-store local:|s3:|b2:|sftp://...` and `-source local:|sftp://[user@]host/|gdrive|...`). ### Bash @@ -824,22 +820,21 @@ All sources produce the same snapshot format. You can back up different sources ### Local Directory -Back up files from a local filesystem path. No authentication or environment variables required. +Back up files from a local filesystem path. No authentication or environment variables required. Specify the path as part of the source URI: `-source local:`. ```bash -cloudstic backup -source local -source-path /path/to/directory +cloudstic backup -source local:/path/to/directory # Skip common development directories -cloudstic backup -source local -source-path ~/project \ +cloudstic backup -source local:~/project \ -exclude ".git/" -exclude "node_modules/" -exclude "*.tmp" # Use an exclude file -cloudstic backup -source local -source-path ~/project -exclude-file .backupignore +cloudstic backup -source local:~/project -exclude-file .backupignore ``` | Flag | Default | Description | |------|---------|-------------| -| `-source-path` | `.` | Root directory to back up | | `-exclude` | | Exclude pattern, gitignore syntax (repeatable) | | `-exclude-file` | | File containing exclude patterns (one per line) | | `-volume-uuid` | | Override volume UUID (see [Portable drives](#portable-drives)) | @@ -854,10 +849,10 @@ When backing up a portable or external drive from multiple machines, Cloudstic a ```bash # Back up a portable drive — UUID is auto-detected -cloudstic backup -source local -source-path /Volumes/MyDrive +cloudstic backup -source local:/Volumes/MyDrive # Override UUID when auto-detection fails or for custom lineage -cloudstic backup -source local -source-path /mnt/backup -volume-uuid "A1B2C3D4-1234-5678-ABCD-EF0123456789" +cloudstic backup -source local:/mnt/backup -volume-uuid "A1B2C3D4-1234-5678-ABCD-EF0123456789" ``` The volume UUID can also be set via the `CLOUDSTIC_VOLUME_UUID` environment variable. When provided, the explicit UUID takes precedence over auto-detection. @@ -884,44 +879,35 @@ The volume UUID is determined from the backup root path only. If your backup dir ```bash # Good: back up each volume independently -cloudstic backup -source local -source-path /Volumes/MyDrive -cloudstic backup -source local -source-path /Volumes/OtherDrive +cloudstic backup -source local:/Volumes/MyDrive +cloudstic backup -source local:/Volumes/OtherDrive # Avoid: backing up a parent that contains mount points from different volumes -cloudstic backup -source local -source-path /Volumes +cloudstic backup -source local:/Volumes ``` Note that symlinks to other volumes are **not followed** — only direct mount points within the tree are traversed. ### SFTP Source -Back up files from a remote SFTP server. Supports password authentication, SSH private key, and ssh-agent. +Back up files from a remote SFTP server. Supports password authentication, SSH private key, and ssh-agent. Specify the server and path using a URI: `sftp://[user@]host[:port]/`. ```bash # Back up a remote directory via SFTP -cloudstic backup -source sftp -source-path /data/documents \ - -sftp-host myserver.com -sftp-user backup -sftp-key ~/.ssh/id_ed25519 +cloudstic backup -source sftp://backup@myserver.com/data/documents \ + -source-sftp-key ~/.ssh/id_ed25519 # Using password auth -cloudstic backup -source sftp -source-path /home/user/files \ - -sftp-host myserver.com -sftp-user backup -sftp-password "secret" +cloudstic backup -source sftp://backup@myserver.com/home/user/files \ + -source-sftp-password "secret" ``` | Flag | Description | |------|-------------| -| `-source-path` | Remote directory path to back up | -| `-sftp-host` | SFTP server hostname | -| `-sftp-port` | SFTP server port (default: `22`) | -| `-sftp-user` | SFTP username | -| `-sftp-password` | SFTP password (optional if using key auth) | -| `-sftp-key` | Path to SSH private key (optional if using password auth) | +| `-source-sftp-password` | SFTP password (optional if using key auth) | +| `-source-sftp-key` | Path to SSH private key (optional if using password auth) | -> [!TIP] -> **Advanced: Source-Specific Overrides** -> If you are using SFTP as *both* a source and a store (e.g. backing up one SFTP server to another), you can use the `-source-sftp-*` flags to override the global SFTP settings for the source: -> `-source-sftp-host`, `-source-sftp-port`, `-source-sftp-user`, `-source-sftp-password`, `-source-sftp-key`. - -If neither `-sftp-password` nor `-sftp-key` is provided, Cloudstic will fall back to your `SSH_AUTH_SOCK` agent. +If neither `-source-sftp-password` nor `-source-sftp-key` is provided, Cloudstic will fall back to your `SSH_AUTH_SOCK` agent. Cloudstic walks the remote directory recursively. File permissions are not preserved — only name, size, modification time, and content are captured. @@ -1044,7 +1030,7 @@ Each snapshot records which source produced it. This metadata is used by retenti ```bash # Keep 30 daily snapshots for Google Drive, 7 for local cloudstic forget -keep-daily 30 -source gdrive -prune -cloudstic forget -keep-daily 7 -source local -prune +cloudstic forget -keep-daily 7 -source local:~/Documents -prune ``` --- @@ -1057,10 +1043,10 @@ Store backups on the local filesystem. This is the default. ```bash # Uses default path ./backup_store -cloudstic init -encryption-password "passphrase" +cloudstic init -password "passphrase" # Custom path -cloudstic init -store local -store-path /mnt/external/backups -encryption-password "passphrase" +cloudstic init -store local:/mnt/external/backups -password "passphrase" ``` ### Backblaze B2 @@ -1071,14 +1057,14 @@ Store backups in a Backblaze B2 bucket. Requires B2 application keys. export B2_KEY_ID=your-key-id export B2_APP_KEY=your-app-key -cloudstic init -store b2 -store-path my-bucket-name -encryption-password "passphrase" -cloudstic backup -store b2 -store-path my-bucket-name -source local -source-path ~/Documents +cloudstic init -store b2:my-bucket-name -password "passphrase" +cloudstic backup -store b2:my-bucket-name -source local:~/Documents ``` -Use `-store-prefix` to namespace objects within a bucket: +Use a prefix to namespace objects within a bucket: ```bash -cloudstic init -store b2 -store-path my-bucket -store-prefix "laptop/" -encryption-password "passphrase" +cloudstic init -store b2:my-bucket/laptop/ -password "passphrase" ``` **Environment variables:** @@ -1098,23 +1084,23 @@ Cloudstic uses the standard AWS SDK for Go, meaning it automatically loads crede # Using explicit environment variables export AWS_ACCESS_KEY_ID=your-access-key export AWS_SECRET_ACCESS_KEY=your-secret-key -cloudstic init -store s3 -store-path my-bucket-name -encryption-password "passphrase" +cloudstic init -store s3:my-bucket-name -password "passphrase" # Using an existing AWS CLI profile (e.g., from ~/.aws/credentials) export AWS_PROFILE=my-profile -cloudstic backup -store s3 -store-path my-bucket-name -source local -source-path ~/Documents +cloudstic backup -store s3:my-bucket-name -source local:~/Documents ``` If using an alternative S3 provider, you must specific the custom endpoint URL. Keep in mind you may also need to modify the `-s3-region` (defaults to `us-east-1`): ```bash -cloudstic init -store s3 -s3-endpoint https://.r2.cloudflarestorage.com -store-path my-bucket -s3-region auto -encryption-password "passphrase" +cloudstic init -store s3:my-bucket -s3-endpoint https://.r2.cloudflarestorage.com -s3-region auto -password "passphrase" ``` -Use `-store-prefix` to namespace objects within a bucket: +Use a prefix to namespace objects within a bucket: ```bash -cloudstic init -store s3 -store-path my-bucket -store-prefix "laptop/" -encryption-password "passphrase" +cloudstic init -store s3:my-bucket/laptop/ -password "passphrase" ``` **Environment variables:** @@ -1132,27 +1118,31 @@ Store backups on a remote SFTP server. Supports password authentication, SSH pri ```bash # Initialize a repository on an SFTP server -cloudstic init -store sftp -store-path /backups/cloudstic \ - -sftp-host myserver.com -sftp-user backup -sftp-key ~/.ssh/id_ed25519 \ - -encryption-password "passphrase" +cloudstic init -store sftp://backup@myserver.com/backups/cloudstic \ + -store-sftp-key ~/.ssh/id_ed25519 \ + -password "passphrase" # Back up to the SFTP store -cloudstic backup -store sftp -store-path /backups/cloudstic \ - -sftp-host myserver.com -sftp-user backup -sftp-key ~/.ssh/id_ed25519 \ - -source local -source-path ~/Documents +cloudstic backup -store sftp://backup@myserver.com/backups/cloudstic \ + -store-sftp-key ~/.ssh/id_ed25519 \ + -source local:~/Documents ``` -The `-store-path` is the remote directory path on the SFTP server where backup objects will be stored. It will be created if it doesn't exist. +The path component of the URI (`/backups/cloudstic` in the example above) is the remote directory where backup objects will be stored. It will be created if it doesn't exist. + +**Flags:** + +| Flag | Description | +|------|-------------| +| `-store-sftp-password` | SFTP password for the store (optional if using key auth) | +| `-store-sftp-key` | Path to SSH private key for the store (optional if using password auth) | **Environment variables:** | Variable | Description | | :--- | :--- | -| `CLOUDSTIC_SFTP_HOST` | SFTP server hostname | -| `CLOUDSTIC_SFTP_PORT` | SFTP server port (default: `22`) | -| `CLOUDSTIC_SFTP_USER` | SFTP username | -| `CLOUDSTIC_SFTP_PASSWORD` | SFTP password | -| `CLOUDSTIC_SFTP_KEY` | Path to SSH private key | +| `CLOUDSTIC_STORE_SFTP_PASSWORD` | SFTP password for the store | +| `CLOUDSTIC_STORE_SFTP_KEY` | Path to SSH private key for the store | --- @@ -1162,7 +1152,7 @@ Encryption is **required by default**. All backup data is encrypted with AES-256 ### Interactive password prompt -When running in a terminal, Cloudstic prompts for the repository password **only if no other credential is provided** via flags (`-encryption-password`, `-encryption-key`, `-recovery-key`, `-kms-key-arn`) or environment variables (`CLOUDSTIC_ENCRYPTION_PASSWORD`, etc.). +When running in a terminal, Cloudstic prompts for the repository password **only if no other credential is provided** via flags (`-password`, `-encryption-key`, `-recovery-key`, `-kms-key-arn`) or environment variables (`CLOUDSTIC_PASSWORD`, etc.). To explicitly request an interactive password prompt alongside a platform key or KMS key, use the `-password` flag: @@ -1187,7 +1177,7 @@ In non-interactive environments (piped input, cron, CI), you must provide creden | Slot type | Credential | Use case | | :--- | :--- | :--- | -| `password` | `--encryption-password` | Day-to-day personal use | +| `password` | `--password` | Day-to-day personal use | | `platform` | `--encryption-key` | Automation, CI/CD, platform integration (legacy) | | `kms-platform` | `--kms-key-arn` | HSM-backed platform integration via AWS KMS (also supports `--kms-region` and `--kms-endpoint`) | | `recovery` | `--recovery-key` | Emergency access when password is lost | @@ -1196,7 +1186,7 @@ In non-interactive environments (piped input, cron, CI), you must provide creden ```bash # Initialize with password + recovery key -cloudstic init -encryption-password "strong passphrase" -recovery +cloudstic init -password "strong passphrase" -add-recovery-key # Write down the 24-word recovery phrase and store it safely! ``` @@ -1217,7 +1207,7 @@ cloudstic restore -recovery-key "word1 word2 ... word24" cloudstic key add-recovery # Non-interactive -cloudstic key add-recovery -encryption-password "my passphrase" +cloudstic key add-recovery -password "my passphrase" # For KMS-managed repositories cloudstic key add-recovery -kms-key-arn arn:aws:kms:us-east-1:123:key/abc @@ -1230,7 +1220,7 @@ cloudstic key add-recovery -kms-key-arn arn:aws:kms:us-east-1:123:key/abc cloudstic key passwd # Non-interactive -cloudstic key passwd -encryption-password "old passphrase" -new-password "new passphrase" +cloudstic key passwd -password "old passphrase" -new-password "new passphrase" ``` --- @@ -1267,19 +1257,20 @@ cloudstic forget -keep-daily 7 -keep-monthly 12 -dry-run | Variable | Flag equivalent | Description | | :--- | :--- | :--- | -| `CLOUDSTIC_STORE` | `-store` | Storage backend: `local`, `s3`, `b2`, `sftp` | -| `CLOUDSTIC_STORE_PATH` | `-store-path` | Local/SFTP path or S3/B2 bucket name | -| `CLOUDSTIC_STORE_PREFIX` | `-store-prefix` | Key prefix for S3/B2 objects | +| `CLOUDSTIC_STORE` | `-store` | Storage backend URI: `local:`, `s3:[/]`, `b2:[/]`, `sftp://[user@]host[:port]/` | | `CLOUDSTIC_S3_ENDPOINT` | `-s3-endpoint` | S3 compatible endpoint (for MinIO, R2, etc.) | | `CLOUDSTIC_S3_REGION` | `-s3-region` | S3 Region | | `AWS_ACCESS_KEY_ID` | `-s3-access-key` | S3 Access Key ID | | `AWS_SECRET_ACCESS_KEY` | `-s3-secret-key` | S3 Secret Access Key | -| `CLOUDSTIC_SOURCE` | `-source` | Source type: `local`, `sftp`, `gdrive`, `gdrive-changes`, `onedrive`, `onedrive-changes` | -| `CLOUDSTIC_SOURCE_PATH` | `-source-path` | Source directory path (local or SFTP remote) | +| `CLOUDSTIC_STORE_SFTP_PASSWORD` | `-store-sftp-password` | SFTP password for the store | +| `CLOUDSTIC_STORE_SFTP_KEY` | `-store-sftp-key` | Path to SSH private key for the store | +| `CLOUDSTIC_SOURCE` | `-source` | Source URI: `local:`, `sftp://[user@]host[:port]/`, `gdrive`, `gdrive-changes`, `onedrive`, `onedrive-changes` | +| `CLOUDSTIC_SOURCE_SFTP_PASSWORD` | `-source-sftp-password` | SFTP password for the source | +| `CLOUDSTIC_SOURCE_SFTP_KEY` | `-source-sftp-key` | Path to SSH private key for the source | | `CLOUDSTIC_DRIVE_ID` | `-drive-id` | Shared drive ID for Google Drive | | `CLOUDSTIC_ROOT_FOLDER` | `-root-folder` | Root folder ID for Google Drive | | `CLOUDSTIC_ENCRYPTION_KEY` | `-encryption-key` | Platform key (hex) | -| `CLOUDSTIC_ENCRYPTION_PASSWORD` | `-encryption-password` | Encryption password | +| `CLOUDSTIC_PASSWORD` | `-password` | Encryption password | | `CLOUDSTIC_RECOVERY_KEY` | `-recovery-key` | Recovery seed phrase | | `CLOUDSTIC_KMS_KEY_ARN` | `-kms-key-arn` | AWS KMS key ARN for kms-platform slots | | `CLOUDSTIC_KMS_REGION` | `-kms-region` | AWS KMS region | @@ -1291,10 +1282,3 @@ cloudstic forget -keep-daily 7 -keep-monthly 12 -dry-run | `ONEDRIVE_TOKEN_FILE` | — | Override OneDrive token path | | `B2_KEY_ID` | — | Backblaze B2 key ID | | `B2_APP_KEY` | — | Backblaze B2 application key | -| `CLOUDSTIC_SFTP_HOST` | `-sftp-host` | SFTP server hostname | -| `CLOUDSTIC_SFTP_PORT` | `-sftp-port` | SFTP server port (default: `22`) | -| `CLOUDSTIC_SFTP_USER` | `-sftp-user` | SFTP username | -| `CLOUDSTIC_SFTP_PASSWORD` | `-sftp-password` | SFTP password | -| `CLOUDSTIC_SFTP_KEY` | `-sftp-key` | Path to SSH private key | -| — | `-source-sftp-*` | Advanced: Overrides global `-sftp-*` for the source | -| — | `-store-sftp-*` | Advanced: Overrides global `-sftp-*` for the store | diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index dae6d46..279bac7 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -164,7 +164,7 @@ func TestCLI_EndToEnd_Matrix(t *testing.T) { storeArgs := store.Setup(t) password := "test-matrix-passphrase" - baseEncArgs := append(storeArgs, "-encryption-password", password) + baseEncArgs := append(storeArgs, "-password", password) // 1. Initial State src.WriteFile(t, "file1.txt", "hello world") @@ -276,7 +276,7 @@ func TestCLI_EndToEnd_Matrix(t *testing.T) { run(t, bin, append([]string{"list"}, baseEncArgs...)...) // 10. Test Key Validation (Wrong Password) - out = runExpectFail(t, bin, append([]string{"list", "--encryption-password", "wrong-password"}, storeArgs...)...) + out = runExpectFail(t, bin, append([]string{"list", "--password", "wrong-password"}, storeArgs...)...) if !strings.Contains(out, "no provided credential matches") { t.Errorf("Expected credential mismatch error, got: %s", out) } @@ -284,7 +284,7 @@ func TestCLI_EndToEnd_Matrix(t *testing.T) { // 11. Test Init Requires Encryption // (We use a fresh initialized dummy temp dir for this test to not ruin the matrix store) dummyStoreDir := t.TempDir() - dummyStoreArgs := []string{"--store", "local", "--store-path", dummyStoreDir} + dummyStoreArgs := []string{"--store", "local:" + dummyStoreDir} out = runExpectFail(t, bin, append([]string{"init"}, dummyStoreArgs...)...) if !strings.Contains(out, "encryption is required") { t.Errorf("Expected encryption-required error, got: %s", out) @@ -298,7 +298,7 @@ func TestCLI_EndToEnd_Matrix(t *testing.T) { // 13. Test Backup Storage Backend With Recovery Key Generation & Restore // Re-init the isolated dummy store with recovery key enabled - out = run(t, bin, append([]string{"init", "--adopt-slots", "--encryption-password", password, "--recovery"}, dummyStoreArgs...)...) + out = run(t, bin, append([]string{"init", "--adopt-slots", "--password", password, "--add-recovery-key"}, dummyStoreArgs...)...) if !strings.Contains(out, "RECOVERY KEY") { t.Fatalf("Expected recovery key output on init, got: %s", out) } @@ -307,7 +307,7 @@ func TestCLI_EndToEnd_Matrix(t *testing.T) { t.Fatal("Could not extract mnemonic from recovery key output") } - run(t, bin, append([]string{"backup", "--encryption-password", password}, append(srcArgs, dummyStoreArgs...)...)...) + run(t, bin, append([]string{"backup", "--password", password}, append(srcArgs, dummyStoreArgs...)...)...) zipRecoveryPath := filepath.Join(restoreDir, "recovery_restore.zip") restoreRecoveryArgs := append([]string{"restore", "--output", zipRecoveryPath, "--recovery-key", mnemonic}, dummyStoreArgs...) @@ -320,21 +320,21 @@ func TestCLI_EndToEnd_Matrix(t *testing.T) { // 14. Test Forget Policy & Dry Run // Re-init another dummy store to test forget logic dummyPolicyDir := t.TempDir() - dummyPolicyStoreArgs := []string{"--store", "local", "--store-path", dummyPolicyDir} - run(t, bin, append([]string{"init", "--encryption-password", password}, dummyPolicyStoreArgs...)...) + dummyPolicyStoreArgs := []string{"--store", "local:" + dummyPolicyDir} + run(t, bin, append([]string{"init", "--password", password}, dummyPolicyStoreArgs...)...) for i := range 3 { src.WriteFile(t, "policy-file.txt", strings.Repeat("x", i+1)) - run(t, bin, append([]string{"backup", "--encryption-password", password}, append(srcArgs, dummyPolicyStoreArgs...)...)...) + run(t, bin, append([]string{"backup", "--password", password}, append(srcArgs, dummyPolicyStoreArgs...)...)...) } - out = run(t, bin, append([]string{"forget", "--keep-last", "1", "--dry-run", "--encryption-password", password}, dummyPolicyStoreArgs...)...) + out = run(t, bin, append([]string{"forget", "--keep-last", "1", "--dry-run", "--password", password}, dummyPolicyStoreArgs...)...) if !strings.Contains(out, "would remove") { t.Errorf("Expected dry-run output, got: %s", out) } - run(t, bin, append([]string{"forget", "--keep-last", "1", "--prune", "--encryption-password", password}, dummyPolicyStoreArgs...)...) - out = run(t, bin, append([]string{"list", "--encryption-password", password}, dummyPolicyStoreArgs...)...) + run(t, bin, append([]string{"forget", "--keep-last", "1", "--prune", "--password", password}, dummyPolicyStoreArgs...)...) + out = run(t, bin, append([]string{"list", "--password", password}, dummyPolicyStoreArgs...)...) if !strings.Contains(out, "1 snapshot") { t.Errorf("Expected 1 snapshot after policy, got: %s", out) } @@ -342,7 +342,7 @@ func TestCLI_EndToEnd_Matrix(t *testing.T) { // 15. Test Unencrypted Backup Lifecycle // Verify that the full backup/restore flow works without encryption. unencDir := t.TempDir() - unencStoreArgs := []string{"--store", "local", "--store-path", unencDir} + unencStoreArgs := []string{"--store", "local:" + unencDir} // 15a. Init with --no-encryption out = run(t, bin, append([]string{"init", "--no-encryption"}, unencStoreArgs...)...) @@ -441,16 +441,15 @@ func TestCLI_EndToEnd_BackupExcludePatterns(t *testing.T) { } password := "test-exclude-pass" - storeArgs := []string{"--store", "local", "--store-path", storeDir} - baseArgs := append(storeArgs, "--encryption-password", password) + storeArgs := []string{"--store", "local:" + storeDir} + baseArgs := append(storeArgs, "--password", password) // Init repo. run(t, bin, append([]string{"init"}, baseArgs...)...) // Backup with -exclude flags and -exclude-file. backupArgs := append([]string{"backup", - "-source", "local", - "-source-path", srcDir, + "-source", "local:" + srcDir, "-exclude", ".git/", "-exclude", "node_modules/", "-exclude", "*.tmp", diff --git a/e2e/local.go b/e2e/local.go index 5b6b197..3640d84 100644 --- a/e2e/local.go +++ b/e2e/local.go @@ -17,7 +17,7 @@ func newLocalSource(t *testing.T) *localSource { func (s *localSource) Name() string { return "local" } func (s *localSource) Env() TestEnv { return Hermetic } func (s *localSource) Setup(t *testing.T) []string { - return []string{"-source", "local", "-source-path", s.dir} + return []string{"-source", "local:" + s.dir} } func (s *localSource) WriteFile(t *testing.T, relPath, content string) { t.Helper() @@ -41,5 +41,5 @@ func newLocalStore(t *testing.T) *localStore { func (s *localStore) Name() string { return "local" } func (s *localStore) Env() TestEnv { return Hermetic } func (s *localStore) Setup(t *testing.T) []string { - return []string{"-store", "local", "-store-path", s.dir} + return []string{"-store", "local:" + s.dir} } diff --git a/e2e/minio.go b/e2e/minio.go index 53f3988..7877e9b 100644 --- a/e2e/minio.go +++ b/e2e/minio.go @@ -90,14 +90,9 @@ func newMinIOTestStore(t *testing.T) *minIOTestStore { func (s *minIOTestStore) Name() string { return "minio" } func (s *minIOTestStore) Env() TestEnv { return Hermetic } func (s *minIOTestStore) Setup(t *testing.T) []string { - // Let's inject a random store prefix down in the bucket just to be safe, - // though the bucket itself is randomly named. prefix := "e2e-root/" - return []string{ - "-store", "s3", - "-store-path", s.bucket, - "-store-prefix", prefix, + "-store", "s3:" + s.bucket + "/" + prefix, "-s3-endpoint", s.endpoint, "-s3-region", "us-east-1", "-s3-access-key", s.accessKey, diff --git a/e2e/portable_darwin.go b/e2e/portable_darwin.go index a0654ed..174db73 100644 --- a/e2e/portable_darwin.go +++ b/e2e/portable_darwin.go @@ -58,7 +58,7 @@ if _, err := os.Stat(s.mountPoint); os.IsNotExist(err) { t.Fatalf("expected mount point %s after partitioning", s.mountPoint) } -return []string{"-source", "local", "-source-path", s.mountPoint} +return []string{"-source", "local:" + s.mountPoint} } func (s *portableDriveSource) WriteFile(t *testing.T, relPath, content string) { diff --git a/e2e/portable_linux.go b/e2e/portable_linux.go index 89af82a..a633134 100644 --- a/e2e/portable_linux.go +++ b/e2e/portable_linux.go @@ -129,7 +129,7 @@ func (s *portableDriveSource) Setup(t *testing.T) []string { t.Logf("warning: GPT partition UUID symlink not found for %s; udev triggered", partDev) } - return []string{"-source", "local", "-source-path", s.mountPoint} + return []string{"-source", "local:" + s.mountPoint} } func (s *portableDriveSource) WriteFile(t *testing.T, relPath, content string) { diff --git a/e2e/sftp.go b/e2e/sftp.go index 857e921..c3460e5 100644 --- a/e2e/sftp.go +++ b/e2e/sftp.go @@ -46,11 +46,7 @@ func (s *sftpTestStore) Name() string { return "sftp" } func (s *sftpTestStore) Env() TestEnv { return Hermetic } func (s *sftpTestStore) Setup(t *testing.T) []string { return []string{ - "-store", "sftp", - "-store-path", s.basePath, - "-store-sftp-host", s.host, - "-store-sftp-port", s.port, - "-store-sftp-user", s.user, + "-store", "sftp://" + s.user + "@" + s.host + ":" + s.port + s.basePath, "-store-sftp-password", s.password, } } @@ -108,11 +104,7 @@ func (s *sftpTestSource) Name() string { return "sftp" } func (s *sftpTestSource) Env() TestEnv { return Hermetic } func (s *sftpTestSource) Setup(t *testing.T) []string { return []string{ - "-source", "sftp", - "-source-path", s.rootPath, - "-source-sftp-host", s.host, - "-source-sftp-port", s.port, - "-source-sftp-user", s.user, + "-source", "sftp://" + s.user + "@" + s.host + ":" + s.port + s.rootPath, "-source-sftp-password", s.password, } } diff --git a/internal/core/models.go b/internal/core/models.go index 57b7ed2..98c35d2 100644 --- a/internal/core/models.go +++ b/internal/core/models.go @@ -71,11 +71,11 @@ type LeafEntry struct { // first-class field on the snapshot so that forget policies can group by // source identity (Type + Account + Path). type SourceInfo struct { - Type string `json:"type"` // e.g. "gdrive", "local" - Account string `json:"account,omitempty"` // Google account email, hostname, etc. - Path string `json:"path,omitempty"` // root folder ID, filesystem path, etc. - VolumeUUID string `json:"volume_uuid,omitempty"` // stable volume identity across mounts/machines - VolumeLabel string `json:"volume_label,omitempty"` // human-readable volume name (e.g. "MyDrive") + Type string `json:"type"` // e.g. "gdrive", "local" + Account string `json:"account,omitempty"` // Google account email, hostname, etc. + Path string `json:"path,omitempty"` // root folder ID, filesystem path, etc. + VolumeUUID string `json:"volume_uuid,omitempty"` // stable volume identity across mounts/machines + VolumeLabel string `json:"volume_label,omitempty"` // human-readable volume name (e.g. "MyDrive") } // Snapshot represents a backup checkpoint @@ -90,7 +90,6 @@ type Snapshot struct { Tags []string `json:"tags,omitempty"` ChangeToken string `json:"change_token,omitempty"` ExcludeHash string `json:"exclude_hash,omitempty"` - HAMTVersion int `json:"hamt_version,omitempty"` // 1 = legacy, 2 = affinity keys } // Index represents a pointer to the latest snapshot diff --git a/internal/engine/backup.go b/internal/engine/backup.go index 479195f..9e8abd9 100644 --- a/internal/engine/backup.go +++ b/internal/engine/backup.go @@ -342,7 +342,6 @@ func (bm *BackupManager) saveSnapshot(ctx context.Context, root string, seq int, Meta: meta, ChangeToken: changeToken, ExcludeHash: bm.cfg.excludeHash, - HAMTVersion: 2, } hash, snapData, err := core.ComputeJSONHash(&snap) diff --git a/internal/engine/policy.go b/internal/engine/policy.go index 195780a..42e5b6f 100644 --- a/internal/engine/policy.go +++ b/internal/engine/policy.go @@ -187,8 +187,15 @@ func matchesFilter(snap *core.Snapshot, f snapshotFilter) bool { if f.source != "" && (snap.Source == nil || snap.Source.Type != f.source) { return false } - if f.account != "" && (snap.Source == nil || snap.Source.Account != f.account) { - return false + if f.account != "" { + if snap.Source == nil { + return false + } + // Accept either the human-readable account (hostname/email) or the + // VolumeUUID so that portable-drive snapshots can be targeted by UUID. + if snap.Source.Account != f.account && snap.Source.VolumeUUID != f.account { + return false + } } if f.path != "" && (snap.Source == nil || snap.Source.Path != f.path) { return false diff --git a/internal/engine/policy_test.go b/internal/engine/policy_test.go index 6f59a85..b63af0f 100644 --- a/internal/engine/policy_test.go +++ b/internal/engine/policy_test.go @@ -159,6 +159,34 @@ func TestMatchesFilter(t *testing.T) { } } +// TestMatchesFilter_VolumeUUID verifies that filtering by -account accepts +// VolumeUUID as an alternative to the hostname, so that portable-drive +// snapshots can be targeted by their stable UUID. +func TestMatchesFilter_VolumeUUID(t *testing.T) { + const uuid = "A1B2C3D4-1234-5678-ABCD-EF0123456789" + + portable := &core.SourceInfo{ + Type: "local", + Account: "macbook-pro", // hostname on machine A + Path: "Documents", + VolumeUUID: uuid, + } + snap := core.Snapshot{Source: portable} + + // Filtering by VolumeUUID should match. + if !matchesFilter(&snap, snapshotFilter{account: uuid}) { + t.Error("should match when account filter equals VolumeUUID") + } + // Filtering by hostname still works. + if !matchesFilter(&snap, snapshotFilter{account: "macbook-pro"}) { + t.Error("should still match when account filter equals hostname") + } + // A different UUID or hostname should not match. + if matchesFilter(&snap, snapshotFilter{account: "OTHER-UUID"}) { + t.Error("should not match a different UUID") + } +} + // TestGroupSnapshots_VolumeUUID verifies that snapshots from different // machines but the same VolumeUUID are grouped together. func TestGroupSnapshots_VolumeUUID(t *testing.T) { diff --git a/pkg/source/gdrive.go b/pkg/source/gdrive.go index 5e92756..2d3c528 100644 --- a/pkg/source/gdrive.go +++ b/pkg/source/gdrive.go @@ -117,7 +117,6 @@ func NewGDriveSource(ctx context.Context, opts ...GDriveOption) (*GDriveSource, var srv *drive.Service var err error - if cfg.httpClient != nil { srv, err = drive.NewService(ctx, option.WithHTTPClient(cfg.httpClient)) if err != nil { diff --git a/scripts/benchmark/affinity.sh b/scripts/benchmark/affinity.sh index d2969e9..e8b8201 100755 --- a/scripts/benchmark/affinity.sh +++ b/scripts/benchmark/affinity.sh @@ -118,10 +118,10 @@ echo "=== Scenario A: Clustered (all $CHANGED changes in dir_01) ===" REPO_A="$TMP_DIR/repo_a" mkdir -p "$REPO_A" -$CLI init -store local -store-path "$REPO_A" --no-encryption 2>&1 | tail -1 +$CLI init -store "local:$REPO_A" --no-encryption 2>&1 | tail -1 # Backup 1: full initial backup. -run_backup "-store local -store-path $REPO_A" "-source local -source-path $DATA" > /dev/null +run_backup "-store local:$REPO_A" "-source local:$DATA" > /dev/null NODES_BEFORE_A=$(count_nodes "$REPO_A") echo " After backup 1: $NODES_BEFORE_A node objects in store" @@ -131,7 +131,7 @@ for f in $(seq 1 $CHANGED); do done # Backup 2: incremental (full scan, but only changed nodes reach persistent store). -run_backup "-store local -store-path $REPO_A" "-source local -source-path $DATA" > /dev/null +run_backup "-store local:$REPO_A" "-source local:$DATA" > /dev/null NODES_AFTER_A=$(count_nodes "$REPO_A") NEW_NODES_A=$((NODES_AFTER_A - NODES_BEFORE_A)) @@ -151,10 +151,10 @@ echo "=== Scenario B: Scattered (1 change in each of $CHANGED dirs) ===" REPO_B="$TMP_DIR/repo_b" mkdir -p "$REPO_B" -$CLI init -store local -store-path "$REPO_B" --no-encryption 2>&1 | tail -1 +$CLI init -store "local:$REPO_B" --no-encryption 2>&1 | tail -1 # Backup 1: full initial backup. -run_backup "-store local -store-path $REPO_B" "-source local -source-path $DATA" > /dev/null +run_backup "-store local:$REPO_B" "-source local:$DATA" > /dev/null NODES_BEFORE_B=$(count_nodes "$REPO_B") echo " After backup 1: $NODES_BEFORE_B node objects in store" @@ -165,7 +165,7 @@ for d in $(seq 1 $CHANGED); do done # Backup 2. -run_backup "-store local -store-path $REPO_B" "-source local -source-path $DATA" > /dev/null +run_backup "-store local:$REPO_B" "-source local:$DATA" > /dev/null NODES_AFTER_B=$(count_nodes "$REPO_B") NEW_NODES_B=$((NODES_AFTER_B - NODES_BEFORE_B)) diff --git a/scripts/benchmark/run.sh b/scripts/benchmark/run.sh index 082a129..87cc694 100755 --- a/scripts/benchmark/run.sh +++ b/scripts/benchmark/run.sh @@ -310,12 +310,12 @@ benchmark_cloudstic() { if [ "$STORE" == "s3" ]; then BENCH_REPO_DIR="" BENCH_S3_PREFIX="s3://$S3_BUCKET/cloudstic/" - store_flags="-store s3 -encryption-password $PASSWORD -store-path $S3_BUCKET -store-prefix cloudstic/" + store_flags="-store s3:$S3_BUCKET/cloudstic/ -encryption-password $PASSWORD" $CLOUDSTIC_BIN init $store_flags >/dev/null || true else BENCH_REPO_DIR="$repo" BENCH_S3_PREFIX="" - store_flags="-store local -store-path $repo" + store_flags="-store local:$repo" $CLOUDSTIC_BIN init $store_flags >/dev/null fi @@ -325,7 +325,7 @@ benchmark_cloudstic() { source_flags="-source gdrive-changes" else reset_data_dir - source_flags="-source local -source-path $DATA_DIR" + source_flags="-source local:$DATA_DIR" fi run_bench "Initial Backup" $CLOUDSTIC_BIN backup $store_flags $source_flags -quiet $DEBUG_FLAG diff --git a/scripts/test_hamt.sh b/scripts/test_hamt.sh index 868b52a..f2a7f4a 100755 --- a/scripts/test_hamt.sh +++ b/scripts/test_hamt.sh @@ -25,8 +25,8 @@ cleanup() { trap cleanup EXIT CLI="go run cmd/cloudstic/main.go" -STORE_FLAGS="-store local -store-path $TMP_DIR/repo" -SOURCE_FLAGS="-source local -source-path $TMP_DIR/data" +STORE_FLAGS="-store local:$TMP_DIR/repo" +SOURCE_FLAGS="-source local:$TMP_DIR/data" # Strip ANSI color codes, then extract the 64-char hex hash from "Snapshot saved" strip_ansi() { sed 's/\x1b\[[0-9;]*m//g'; } diff --git a/scripts/test_repack.sh b/scripts/test_repack.sh index 4576fd8..ce8444c 100755 --- a/scripts/test_repack.sh +++ b/scripts/test_repack.sh @@ -36,8 +36,8 @@ echo "Testing Repack Strategy in $TMP_DIR" echo "" echo "=== Test 1: Full Orphan Repack ===" -STORE1="-store local -store-path $TMP_DIR/repo" -SOURCE1="-source local -source-path $TMP_DIR/data" +STORE1="-store local:$TMP_DIR/repo" +SOURCE1="-source local:$TMP_DIR/data" mkdir -p "$TMP_DIR/data" "$TMP_DIR/repo" @@ -84,8 +84,8 @@ check "ls shows all 20 files after prune (got $FILE_COUNT)" "[ '$FILE_COUNT' -eq echo "" echo "=== Test 2: Partial Fragment Repack ===" -STORE2="-store local -store-path $TMP_DIR/repo2" -SOURCE2="-source local -source-path $TMP_DIR/data2" +STORE2="-store local:$TMP_DIR/repo2" +SOURCE2="-source local:$TMP_DIR/data2" mkdir -p "$TMP_DIR/data2" "$TMP_DIR/repo2" diff --git a/scripts/test_selective_restore.sh b/scripts/test_selective_restore.sh index 7b5d51f..666319e 100755 --- a/scripts/test_selective_restore.sh +++ b/scripts/test_selective_restore.sh @@ -29,7 +29,7 @@ strip_ansi() { sed 's/\x1b\[[0-9;]*m//g'; } echo "Testing Selective Restore (Google Drive source) in $TMP_DIR" -STORE="-store local -store-path $TMP_DIR/repo" +STORE="-store local:$TMP_DIR/repo" # --------------------------------------------------------- # Step 1: Backup from Google Drive