Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,6 @@ test-output.json
*.test
*.test.exe
*.prof
.DS_Store
.DS_Store
# opencode
.opencode/
37 changes: 19 additions & 18 deletions cmd/cloudstic/cmd_backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,28 @@ import (
)

type backupArgs struct {
g *globalFlags
sourceURI string
driveID string
rootFolder string
dryRun bool
excludeFile string
skipNativeFiles bool
volumeUUID string
googleCreds string
googleTokenFile string
onedriveClientID string
g *globalFlags
sourceURI string
driveID string
rootFolder string
dryRun bool
excludeFile string
skipNativeFiles bool
volumeUUID string
googleCreds string
googleTokenFile string
onedriveClientID string
onedriveTokenFile string
tags stringArrayFlags
excludes stringArrayFlags
tags stringArrayFlags
excludes stringArrayFlags
}

func parseBackupArgs() *backupArgs {
fs := flag.NewFlagSet("backup", flag.ExitOnError)
a := &backupArgs{}
a.g = addGlobalFlags(fs)
sourceURI := fs.String("source", envDefault("CLOUDSTIC_SOURCE", "gdrive"), "Source URI: local:<path>, sftp://[user@]host[:port]/<path>, gdrive, gdrive-changes, onedrive, onedrive-changes")
sourceURI := fs.String("source", envDefault("CLOUDSTIC_SOURCE", "gdrive"), "Source URI: local:<path>, sftp://[user@]host[:port]/<path>, gdrive[:<path>], gdrive-changes[:<path>], onedrive[:<path>], onedrive-changes[:<path>]")
driveID := fs.String("drive-id", envDefault("CLOUDSTIC_DRIVE_ID", ""), "Shared drive ID for gdrive source (omit for My Drive)")
rootFolder := fs.String("root-folder", envDefault("CLOUDSTIC_ROOT_FOLDER", ""), "Root folder ID for gdrive source (defaults to entire drive)")
dryRun := fs.Bool("dry-run", false, "Scan source and report changes without writing to the store")
skipNativeFiles := fs.Bool("skip-native-files", false, "Exclude Google-native files (Docs, Sheets, Slides, etc.) from the backup")
excludeFile := fs.String("exclude-file", "", "Path to file with exclude patterns (one per line, gitignore syntax)")
Expand All @@ -52,7 +51,7 @@ func parseBackupArgs() *backupArgs {
mustParse(fs)
a.sourceURI = *sourceURI
a.driveID = *driveID
a.rootFolder = *rootFolder
a.rootFolder = ""
a.dryRun = *dryRun
Comment thread
rmanibus marked this conversation as resolved.
a.skipNativeFiles = *skipNativeFiles
a.excludeFile = *excludeFile
Expand Down Expand Up @@ -172,7 +171,7 @@ func initSource(ctx context.Context, sourceURI, driveID, rootFolder string, skip
source.WithCredsPath(googleCreds),
source.WithTokenPath(tokenPath),
source.WithDriveID(driveID),
source.WithRootFolderID(rootFolder),
source.WithRootPath(uri.path),
source.WithGDriveExcludePatterns(excludePatterns),
}
if skipNativeFiles {
Expand All @@ -188,7 +187,7 @@ func initSource(ctx context.Context, sourceURI, driveID, rootFolder string, skip
source.WithCredsPath(googleCreds),
source.WithTokenPath(tokenPath),
source.WithDriveID(driveID),
source.WithRootFolderID(rootFolder),
source.WithRootPath(uri.path),
source.WithGDriveExcludePatterns(excludePatterns),
}
if skipNativeFiles {
Expand All @@ -203,6 +202,7 @@ func initSource(ctx context.Context, sourceURI, driveID, rootFolder string, skip
return source.NewOneDriveSource(ctx,
source.WithOneDriveClientID(onedriveClientID),
source.WithOneDriveTokenPath(tokenPath),
source.WithOneDriveRootPath(uri.path),
source.WithOneDriveExcludePatterns(excludePatterns),
)
case "onedrive-changes":
Expand All @@ -213,6 +213,7 @@ func initSource(ctx context.Context, sourceURI, driveID, rootFolder string, skip
return source.NewOneDriveChangeSource(ctx,
source.WithOneDriveClientID(onedriveClientID),
source.WithOneDriveTokenPath(tokenPath),
source.WithOneDriveRootPath(uri.path),
source.WithOneDriveExcludePatterns(excludePatterns),
)
default:
Expand Down
2 changes: 1 addition & 1 deletion cmd/cloudstic/cmd_forget_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,4 +117,4 @@ func TestRunForget_Policy_DryRun(t *testing.T) {
if !strings.Contains(got, "dry run") {
t.Errorf("expected 'dry run' in summary, got:\n%s", got)
}
}
}
8 changes: 3 additions & 5 deletions cmd/cloudstic/completion.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ _cloudstic() {
-*)
# skip flags and their values
case "${words[i]}" in
-store|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-drive-id|-root-folder|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account|-json)
-store|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-drive-id|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account|-json)
((i++)) ;;
esac
;;
Expand All @@ -76,7 +76,7 @@ _cloudstic() {
init)
cmd_flags="-add-recovery-key -no-encryption -adopt-slots" ;;
backup)
cmd_flags="-source -drive-id -root-folder -skip-native-files -google-credentials -google-token-file -onedrive-client-id -onedrive-token-file -tag -dry-run" ;;
cmd_flags="-source -drive-id -skip-native-files -google-credentials -google-token-file -onedrive-client-id -onedrive-token-file -tag -dry-run" ;;
restore)
cmd_flags="-output -dry-run" ;;
prune)
Expand Down Expand Up @@ -199,7 +199,7 @@ _cloudstic() {
-*)
# Skip flags with values
case "${words[i]}" in
-store|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-drive-id|-root-folder|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account)
-store|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-drive-id|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account)
(( i++ )) ;;
esac
;;
Expand Down Expand Up @@ -228,7 +228,6 @@ _cloudstic() {
_arguments $global_flags \
'-source[Source URI]:uri:(local: sftp:// gdrive gdrive-changes onedrive onedrive-changes)' \
'-drive-id[Shared drive ID]:id:' \
'-root-folder[Root folder ID]:id:' \
'-skip-native-files[Exclude Google-native files]' \
'-google-credentials[Google service account credentials JSON]:path:_files' \
'-google-token-file[Google OAuth token file]:path:_files' \
Expand Down Expand Up @@ -379,7 +378,6 @@ complete -c cloudstic -n '__fish_seen_subcommand_from init' -l adopt-slots -d 'A
# backup
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l source -x -a 'local: sftp:// gdrive gdrive-changes onedrive onedrive-changes' -d 'Source URI'
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l drive-id -x -d 'Shared drive ID'
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l root-folder -x -d 'Root folder ID'
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l skip-native-files -d 'Exclude Google-native files'
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l google-credentials -r -F -d 'Google service account credentials JSON'
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l google-token-file -r -F -d 'Google OAuth token file'
Expand Down
22 changes: 11 additions & 11 deletions cmd/cloudstic/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,18 @@ func envBool(key string) bool {
}

type globalFlags struct {
store *string
s3Endpoint, s3Region *string
s3AccessKey, s3SecretKey *string
store *string
s3Endpoint, s3Region *string
s3AccessKey, s3SecretKey *string
sourceSFTPPassword, sourceSFTPKey *string
storeSFTPPassword, storeSFTPKey *string
encryptionKey *string
password *string
recoveryKey *string
kmsKeyARN, kmsRegion, kmsEndpoint *string
disablePackfile *bool
prompt, verbose, quiet, debug *bool
debugLog *ui.SafeLogWriter
storeSFTPPassword, storeSFTPKey *string
encryptionKey *string
password *string
recoveryKey *string
kmsKeyARN, kmsRegion, kmsEndpoint *string
disablePackfile *bool
prompt, verbose, quiet, debug *bool
debugLog *ui.SafeLogWriter
}

func addGlobalFlags(fs *flag.FlagSet) *globalFlags {
Expand Down
13 changes: 11 additions & 2 deletions cmd/cloudstic/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,8 @@ func parseSourceURI(raw string) (*sourceURIParts, error) {
return nil, fmt.Errorf("invalid source URI %q: local path cannot be empty", raw)
}
return &sourceURIParts{scheme: "local", path: rest}, nil
case "gdrive", "gdrive-changes", "onedrive", "onedrive-changes":
return &sourceURIParts{scheme: scheme, path: ensureLeadingSlash(rest)}, nil
default:
return nil, fmt.Errorf("unknown source scheme %q in %q: supported URI formats are local:<path> and sftp://[user@]host[:port]/<path>", scheme, raw)
}
Expand All @@ -335,12 +337,19 @@ func parseSourceURI(raw string) (*sourceURIParts, error) {
// Bare keyword (cloud sources)
switch raw {
case "gdrive", "gdrive-changes", "onedrive", "onedrive-changes":
return &sourceURIParts{scheme: raw}, nil
return &sourceURIParts{scheme: raw, path: "/"}, nil
default:
return nil, fmt.Errorf("unknown source %q: supported values are local:<path>, sftp://[user@]host[:port]/<path>, gdrive, gdrive-changes, onedrive, onedrive-changes", raw)
return nil, fmt.Errorf("unknown source %q: supported values are local:<path>, sftp://[user@]host[:port]/<path>, gdrive[:<path>], gdrive-changes[:<path>], onedrive[:<path>], onedrive-changes[:<path>]", raw)
}
}

func ensureLeadingSlash(s string) string {
if s == "" || !strings.HasPrefix(s, "/") {
return "/" + s
}
return s
}

func (g *globalFlags) buildSFTPSourceOpts(uri *sourceURIParts) []source.SFTPOption {
opts := []source.SFTPOption{
source.WithSFTPSourceBasePath(uri.path),
Expand Down
11 changes: 7 additions & 4 deletions cmd/cloudstic/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,13 @@ func TestParseSourceURI(t *testing.T) {
{raw: "sftp:///no-host", wantErr: true},

// cloud keywords
{raw: "gdrive", want: sourceURIParts{scheme: "gdrive"}},
{raw: "gdrive-changes", want: sourceURIParts{scheme: "gdrive-changes"}},
{raw: "onedrive", want: sourceURIParts{scheme: "onedrive"}},
{raw: "onedrive-changes", want: sourceURIParts{scheme: "onedrive-changes"}},
{raw: "gdrive", want: sourceURIParts{scheme: "gdrive", path: "/"}},
{raw: "gdrive-changes", want: sourceURIParts{scheme: "gdrive-changes", path: "/"}},
{raw: "onedrive", want: sourceURIParts{scheme: "onedrive", path: "/"}},
{raw: "onedrive-changes", want: sourceURIParts{scheme: "onedrive-changes", path: "/"}},
{raw: "gdrive:/some/path", want: sourceURIParts{scheme: "gdrive", path: "/some/path"}},
{raw: "gdrive:some/path", want: sourceURIParts{scheme: "gdrive", path: "/some/path"}},
{raw: "onedrive:/documents", want: sourceURIParts{scheme: "onedrive", path: "/documents"}},

// invalid
{raw: "sftp", wantErr: true},
Expand Down
1 change: 0 additions & 1 deletion cmd/cloudstic/usage.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@ func printUsage() {
t.Flags([][2]string{
{"-source <uri>", ui.Env("Source URI: local:<path>, sftp://[user@]host[:port]/<path>, gdrive, gdrive-changes, onedrive, onedrive-changes", "CLOUDSTIC_SOURCE")},
{"-drive-id <id>", "Shared drive ID for gdrive (omit for My Drive)"},
{"-root-folder <id>", "Root folder ID for gdrive (defaults to entire drive)"},
{"-skip-native-files", "Exclude Google-native files (Docs, Sheets, Slides, etc.)"},
{"-google-credentials <path>", ui.Env("Path to Google service account credentials JSON", "GOOGLE_APPLICATION_CREDENTIALS")},
{"-google-token-file <path>", ui.Env("Path to Google OAuth token file", "GOOGLE_TOKEN_FILE")},
Expand Down
2 changes: 1 addition & 1 deletion docs/sources.md
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ Walks the remote directory tree via SFTP. Supports password, SSH private key, an
| **SourceInfo.Account** | Google account email |
| **SourceInfo.Path** | `my-drive://` or `<driveID>://<rootFolderID>` |
Comment thread
rmanibus marked this conversation as resolved.

Lists all files and folders via `files.list`, then topologically sorts folders so parents are emitted before children. Supports My Drive and Shared Drives (via `-drive-id`), with optional folder scoping (via `-root-folder`).
Lists all files and folders via `files.list`, then topologically sorts folders so parents are emitted before children. Supports My Drive and Shared Drives (via `-drive-id`), with optional folder scoping (via `gdrive:/path/to/folder`).

### `gdrive-changes` — Google Drive (Changes API)

Expand Down
19 changes: 8 additions & 11 deletions docs/user-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ cloudstic backup -source local:~/Documents
cloudstic backup -source gdrive

# Back up a specific Google Drive shared drive and folder
cloudstic backup -source gdrive -drive-id <shared-drive-id> -root-folder <folder-id>
cloudstic backup -source gdrive:/path/to/folder -drive-id <shared-drive-id>

# Back up with tags
cloudstic backup -source local:~/Documents -tag daily -tag important
Expand All @@ -268,9 +268,8 @@ cloudstic backup -source local:~/Documents -dry-run

| Flag | Default | Description |
|------|---------|-------------|
| `-source` | `gdrive` | Source type: `local:<path>`, `sftp://[user@]host[:port]/<path>`, `gdrive`, `gdrive-changes`, `onedrive`, `onedrive-changes` |
| `-source` | `gdrive` | Source type: `local:<path>`, `sftp://[user@]host[:port]/<path>`, `gdrive[:<path>]`, `gdrive-changes[:<path>]`, `onedrive[:<path>]`, `onedrive-changes[:<path>]` |
| `-drive-id` | | Shared drive ID for Google Drive (omit for My Drive) |
| `-root-folder` | | Root folder ID for Google Drive (defaults to entire drive) |
| `-tag` | | Tag to apply to the snapshot (repeatable) |
| `-exclude` | | Exclude pattern using gitignore syntax (repeatable) |
| `-exclude-file` | | Path to file containing exclude patterns, one per line |
Expand Down Expand Up @@ -928,19 +927,18 @@ No configuration is required — Cloudstic ships with built-in OAuth credentials

```bash
# Back up entire My Drive
cloudstic backup -source gdrive
cloudstic backup -source gdrive:/

# Back up a shared drive
cloudstic backup -source gdrive -drive-id <shared-drive-id>
cloudstic backup -source gdrive:/ -drive-id <shared-drive-id>

# Back up only a specific folder (by Google Drive folder ID)
cloudstic backup -source gdrive -root-folder <folder-id>
# Back up only a specific folder
cloudstic backup -source gdrive:/path/to/folder
```

| Flag | Description |
|------|-------------|
| `-drive-id` | Shared Drive ID (omit for personal My Drive) |
| `-root-folder` | Restrict backup to a specific folder by ID |

**Environment variables (optional overrides):**

Expand All @@ -965,7 +963,7 @@ cloudstic backup -source gdrive-changes
cloudstic backup -source gdrive-changes
```

Uses the same authentication and flags as [Google Drive](#google-drive) (`-drive-id`, `-root-folder`). No setup required — just run the command and authorize in the browser.
Uses the same authentication and flags as [Google Drive](#google-drive) (`-drive-id`). No setup required — just run the command and authorize in the browser.

> **Tip:** You can use `-source gdrive-changes` from day one — the first run performs a full scan just like `gdrive`. Only fall back to `-source gdrive` if you need to force a complete rescan.

Expand Down Expand Up @@ -1265,11 +1263,10 @@ cloudstic forget -keep-daily 7 -keep-monthly 12 -dry-run
| `AWS_SECRET_ACCESS_KEY` | `-s3-secret-key` | S3 Secret Access Key |
| `CLOUDSTIC_STORE_SFTP_PASSWORD` | `-store-sftp-password` | SFTP password for the store |
| `CLOUDSTIC_STORE_SFTP_KEY` | `-store-sftp-key` | Path to SSH private key for the store |
| `CLOUDSTIC_SOURCE` | `-source` | Source URI: `local:<path>`, `sftp://[user@]host[:port]/<path>`, `gdrive`, `gdrive-changes`, `onedrive`, `onedrive-changes` |
| `CLOUDSTIC_SOURCE` | `-source` | Source URI: `local:<path>`, `sftp://[user@]host[:port]/<path>`, `gdrive[:<path>]`, `gdrive-changes[:<path>]`, `onedrive[:<path>]`, `onedrive-changes[:<path>]` |
| `CLOUDSTIC_SOURCE_SFTP_PASSWORD` | `-source-sftp-password` | SFTP password for the source |
| `CLOUDSTIC_SOURCE_SFTP_KEY` | `-source-sftp-key` | Path to SSH private key for the source |
| `CLOUDSTIC_DRIVE_ID` | `-drive-id` | Shared drive ID for Google Drive |
| `CLOUDSTIC_ROOT_FOLDER` | `-root-folder` | Root folder ID for Google Drive |
| `CLOUDSTIC_ENCRYPTION_KEY` | `-encryption-key` | Platform key (hex) |
| `CLOUDSTIC_PASSWORD` | `-password` | Encryption password |
| `CLOUDSTIC_RECOVERY_KEY` | `-recovery-key` | Recovery seed phrase |
Expand Down
Loading
Loading