Skip to content

feat: drive identity by name (RFC 0008) #84

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rmanibus merged 1 commit into from
Mar 14, 2026
Merged

feat: drive identity by name (RFC 0008) #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 7 additions & 10 deletions cmd/cloudstic/cmd_backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ import (
type backupArgs struct {
g *globalFlags
sourceURI string
driveID string
rootFolder string
dryRun bool
excludeFile string
skipNativeFiles bool
Expand All @@ -36,8 +34,7 @@ func parseBackupArgs() *backupArgs {
fs := flag.NewFlagSet("backup", flag.ExitOnError)
a := &backupArgs{}
a.g = addGlobalFlags(fs)
sourceURI := fs.String("source", envDefault("CLOUDSTIC_SOURCE", "gdrive"), "Source URI: local:<path>, sftp://[user@]host[:port]/<path>, gdrive[:<path>], gdrive-changes[:<path>], onedrive[:<path>], onedrive-changes[:<path>]")
driveID := fs.String("drive-id", envDefault("CLOUDSTIC_DRIVE_ID", ""), "Shared drive ID for gdrive source (omit for My Drive)")
sourceURI := fs.String("source", envDefault("CLOUDSTIC_SOURCE", "gdrive"), "Source URI: local:<path>, sftp://[user@]host[:port]/<path>, gdrive[://<Drive Name>][/<path>], gdrive-changes[://<Drive Name>][/<path>], onedrive[://<Drive Name>][/<path>], onedrive-changes[://<Drive Name>][/<path>]")
dryRun := fs.Bool("dry-run", false, "Scan source and report changes without writing to the store")
skipNativeFiles := fs.Bool("skip-native-files", false, "Exclude Google-native files (Docs, Sheets, Slides, etc.) from the backup")
excludeFile := fs.String("exclude-file", "", "Path to file with exclude patterns (one per line, gitignore syntax)")
Expand All @@ -50,8 +47,6 @@ func parseBackupArgs() *backupArgs {
fs.Var(&a.excludes, "exclude", "Exclude pattern (gitignore syntax, repeatable)")
mustParse(fs)
a.sourceURI = *sourceURI
a.driveID = *driveID
a.rootFolder = ""
a.dryRun = *dryRun
a.skipNativeFiles = *skipNativeFiles
a.excludeFile = *excludeFile
Expand All @@ -73,7 +68,7 @@ func (r *runner) runBackup() int {

ctx := context.Background()

src, err := initSource(ctx, a.sourceURI, a.driveID, a.rootFolder, a.skipNativeFiles, a.volumeUUID, a.googleCreds, a.googleTokenFile, a.onedriveClientID, a.onedriveTokenFile, a.g, excludePatterns)
src, err := initSource(ctx, a.sourceURI, a.skipNativeFiles, a.volumeUUID, a.googleCreds, a.googleTokenFile, a.onedriveClientID, a.onedriveTokenFile, a.g, excludePatterns)
if err != nil {
return r.fail("Failed to init source: %v", err)
}
Expand Down Expand Up @@ -145,7 +140,7 @@ func (r *runner) printBackupSummary(res *engine.RunResult) {
}
}

func initSource(ctx context.Context, sourceURI, driveID, rootFolder string, skipNativeFiles bool, volumeUUID, googleCreds, googleTokenFile, onedriveClientID, onedriveTokenFile string, g *globalFlags, excludePatterns []string) (source.Source, error) {
func initSource(ctx context.Context, sourceURI string, skipNativeFiles bool, volumeUUID, googleCreds, googleTokenFile, onedriveClientID, onedriveTokenFile string, g *globalFlags, excludePatterns []string) (source.Source, error) {
uri, err := parseSourceURI(sourceURI)
if err != nil {
return nil, err
Expand All @@ -170,7 +165,7 @@ func initSource(ctx context.Context, sourceURI, driveID, rootFolder string, skip
gdriveOpts := []source.GDriveOption{
source.WithCredsPath(googleCreds),
source.WithTokenPath(tokenPath),
source.WithDriveID(driveID),
source.WithDriveName(uri.host),
source.WithRootPath(uri.path),
source.WithGDriveExcludePatterns(excludePatterns),
}
Expand All @@ -186,7 +181,7 @@ func initSource(ctx context.Context, sourceURI, driveID, rootFolder string, skip
gdriveOpts := []source.GDriveOption{
source.WithCredsPath(googleCreds),
source.WithTokenPath(tokenPath),
source.WithDriveID(driveID),
source.WithDriveName(uri.host),
source.WithRootPath(uri.path),
source.WithGDriveExcludePatterns(excludePatterns),
}
Expand All @@ -202,6 +197,7 @@ func initSource(ctx context.Context, sourceURI, driveID, rootFolder string, skip
return source.NewOneDriveSource(ctx,
source.WithOneDriveClientID(onedriveClientID),
source.WithOneDriveTokenPath(tokenPath),
source.WithOneDriveDriveName(uri.host),
source.WithOneDriveRootPath(uri.path),
source.WithOneDriveExcludePatterns(excludePatterns),
)
Expand All @@ -213,6 +209,7 @@ func initSource(ctx context.Context, sourceURI, driveID, rootFolder string, skip
return source.NewOneDriveChangeSource(ctx,
source.WithOneDriveClientID(onedriveClientID),
source.WithOneDriveTokenPath(tokenPath),
source.WithOneDriveDriveName(uri.host),
source.WithOneDriveRootPath(uri.path),
source.WithOneDriveExcludePatterns(excludePatterns),
)
Expand Down
8 changes: 3 additions & 5 deletions cmd/cloudstic/completion.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ _cloudstic() {
-*)
# skip flags and their values
case "${words[i]}" in
-store|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-drive-id|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account|-json)
-store|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account|-json)
((i++)) ;;
esac
;;
Expand All @@ -76,7 +76,7 @@ _cloudstic() {
init)
cmd_flags="-add-recovery-key -no-encryption -adopt-slots" ;;
backup)
cmd_flags="-source -drive-id -skip-native-files -google-credentials -google-token-file -onedrive-client-id -onedrive-token-file -tag -dry-run" ;;
cmd_flags="-source -skip-native-files -google-credentials -google-token-file -onedrive-client-id -onedrive-token-file -tag -dry-run" ;;
restore)
cmd_flags="-output -dry-run" ;;
prune)
Expand Down Expand Up @@ -199,7 +199,7 @@ _cloudstic() {
-*)
# Skip flags with values
case "${words[i]}" in
-store|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-drive-id|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account)
-store|-s3-endpoint|-s3-region|-s3-access-key|-s3-secret-key|-source-sftp-password|-source-sftp-key|-store-sftp-password|-store-sftp-key|-encryption-key|-password|-recovery-key|-kms-key-arn|-kms-region|-kms-endpoint|-source|-google-credentials|-google-token-file|-onedrive-client-id|-onedrive-token-file|-tag|-output|-keep-last|-keep-hourly|-keep-daily|-keep-weekly|-keep-monthly|-keep-yearly|-group-by|-account)
(( i++ )) ;;
esac
;;
Expand Down Expand Up @@ -227,7 +227,6 @@ _cloudstic() {
backup)
_arguments $global_flags \
'-source[Source URI]:uri:(local: sftp:// gdrive gdrive-changes onedrive onedrive-changes)' \
'-drive-id[Shared drive ID]:id:' \
'-skip-native-files[Exclude Google-native files]' \
'-google-credentials[Google service account credentials JSON]:path:_files' \
'-google-token-file[Google OAuth token file]:path:_files' \
Expand Down Expand Up @@ -377,7 +376,6 @@ complete -c cloudstic -n '__fish_seen_subcommand_from init' -l adopt-slots -d 'A

# backup
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l source -x -a 'local: sftp:// gdrive gdrive-changes onedrive onedrive-changes' -d 'Source URI'
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l drive-id -x -d 'Shared drive ID'
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l skip-native-files -d 'Exclude Google-native files'
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l google-credentials -r -F -d 'Google service account credentials JSON'
complete -c cloudstic -n '__fish_seen_subcommand_from backup' -l google-token-file -r -F -d 'Google OAuth token file'
Expand Down
14 changes: 14 additions & 0 deletions cmd/cloudstic/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,20 @@ func parseSourceURI(raw string) (*sourceURIParts, error) {
}
return &sourceURIParts{scheme: "local", path: rest}, nil
case "gdrive", "gdrive-changes", "onedrive", "onedrive-changes":
if strings.HasPrefix(rest, "//") {
// Format: scheme://Drive Name/path
rest = rest[2:]
idx := strings.IndexByte(rest, '/')
driveName := ""
path := "/"
if idx >= 0 {
driveName = rest[:idx]
path = ensureLeadingSlash(rest[idx:])
} else {
driveName = rest
}
return &sourceURIParts{scheme: scheme, host: driveName, path: path}, nil
}
return &sourceURIParts{scheme: scheme, path: ensureLeadingSlash(rest)}, nil
default:
return nil, fmt.Errorf("unknown source scheme %q in %q: supported URI formats are local:<path> and sftp://[user@]host[:port]/<path>", scheme, raw)
Expand Down
4 changes: 4 additions & 0 deletions cmd/cloudstic/store_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ func TestParseSourceURI(t *testing.T) {
{raw: "gdrive:/some/path", want: sourceURIParts{scheme: "gdrive", path: "/some/path"}},
{raw: "gdrive:some/path", want: sourceURIParts{scheme: "gdrive", path: "/some/path"}},
{raw: "onedrive:/documents", want: sourceURIParts{scheme: "onedrive", path: "/documents"}},
{raw: "gdrive://My Shared Drive/some/path", want: sourceURIParts{scheme: "gdrive", host: "My Shared Drive", path: "/some/path"}},
{raw: "gdrive-changes://Company Data/finance", want: sourceURIParts{scheme: "gdrive-changes", host: "Company Data", path: "/finance"}},
{raw: "onedrive://Personal/documents", want: sourceURIParts{scheme: "onedrive", host: "Personal", path: "/documents"}},
{raw: "onedrive-changes://Shared/photos", want: sourceURIParts{scheme: "onedrive-changes", host: "Shared", path: "/photos"}},

// invalid
{raw: "sftp", wantErr: true},
Expand Down
3 changes: 1 addition & 2 deletions cmd/cloudstic/usage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,7 @@ func printUsage() {

t.Command("backup", "")
t.Flags([][2]string{
{"-source <uri>", ui.Env("Source URI: local:<path>, sftp://[user@]host[:port]/<path>, gdrive, gdrive-changes, onedrive, onedrive-changes", "CLOUDSTIC_SOURCE")},
{"-drive-id <id>", "Shared drive ID for gdrive (omit for My Drive)"},
{"-source <uri>", ui.Env("Source URI: local:<path>, sftp://[user@]host[:port]/<path>, gdrive[://<Drive Name>][/<path>], gdrive-changes[://<Drive Name>][/<path>], onedrive[://<Drive Name>][/<path>], onedrive-changes[://<Drive Name>][/<path>]", "CLOUDSTIC_SOURCE")},
{"-skip-native-files", "Exclude Google-native files (Docs, Sheets, Slides, etc.)"},
{"-google-credentials <path>", ui.Env("Path to Google service account credentials JSON", "GOOGLE_APPLICATION_CREDENTIALS")},
{"-google-token-file <path>", ui.Env("Path to Google OAuth token file", "GOOGLE_TOKEN_FILE")},
Expand Down
2 changes: 1 addition & 1 deletion docs/sources.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ Walks the remote directory tree via SFTP. Supports password, SSH private key, an
| **SourceInfo.Account** | Google account email |
| **SourceInfo.Path** | `my-drive://` or `<driveID>://<rootFolderID>` |

Lists all files and folders via `files.list`, then topologically sorts folders so parents are emitted before children. Supports My Drive and Shared Drives (via `-drive-id`), with optional folder scoping (via `gdrive:/path/to/folder`).
Lists all files and folders via `files.list`, then topologically sorts folders so parents are emitted before children. Supports My Drive and Shared Drives (via `gdrive://<Drive Name>`), with optional folder scoping (via `gdrive://<Drive Name>/path/to/folder`).

### `gdrive-changes` — Google Drive (Changes API)

Expand Down
32 changes: 13 additions & 19 deletions docs/user-guide.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ Cloudstic is a content-addressable backup tool that creates encrypted, deduplica
- [Local Directory](#local-directory)
- [SFTP](#sftp-source)
- [Google Drive](#google-drive)
- [Google Drive (Changes API)](#google-drive-changes-api)
- [Google Drive (Incremental)](#google-drive-incremental)
- [OneDrive](#onedrive)
- [OneDrive (Changes API)](#onedrive-changes-api)
- [OneDrive (Incremental)](#onedrive-incremental)
- [Storage Backends](#storage-backends)
- [Local](#local-storage)
- [Amazon S3](#amazon-s3)
Expand Down Expand Up @@ -252,7 +252,7 @@ cloudstic backup -source local:~/Documents
cloudstic backup -source gdrive

# Back up a specific Google Drive shared drive and folder
cloudstic backup -source gdrive:/path/to/folder -drive-id <shared-drive-id>
cloudstic backup -source "gdrive://Company Data/path/to/folder"

# Back up with tags
cloudstic backup -source local:~/Documents -tag daily -tag important
Expand All @@ -268,8 +268,7 @@ cloudstic backup -source local:~/Documents -dry-run

| Flag | Default | Description |
|------|---------|-------------|
| `-source` | `gdrive` | Source type: `local:<path>`, `sftp://[user@]host[:port]/<path>`, `gdrive[:<path>]`, `gdrive-changes[:<path>]`, `onedrive[:<path>]`, `onedrive-changes[:<path>]` |
| `-drive-id` | | Shared drive ID for Google Drive (omit for My Drive) |
| `-source` | `gdrive` | Source type: `local:<path>`, `sftp://[user@]host[:port]/<path>`, `gdrive[://<Drive Name>][/<path>]`, `gdrive-changes[://<Drive Name>][/<path>]`, `onedrive[://<Drive Name>][/<path>]`, `onedrive-changes[://<Drive Name>][/<path>]` |
| `-tag` | | Tag to apply to the snapshot (repeatable) |
| `-exclude` | | Exclude pattern using gitignore syntax (repeatable) |
| `-exclude-file` | | Path to file containing exclude patterns, one per line |
Expand Down Expand Up @@ -812,9 +811,9 @@ A **source** is where Cloudstic reads files from during a backup. Each source ty
| [Local directory](#local-directory) | `local` | Files on your local filesystem | None |
| [SFTP](#sftp-source) | `sftp` | Files on a remote SFTP server | Password, SSH key, or ssh-agent |
| [Google Drive](#google-drive) | `gdrive` | Full scan of Google Drive (My Drive or Shared Drive) | Automatic (browser) |
| [Google Drive (Changes API)](#google-drive-changes-api) | `gdrive-changes` | Incremental changes since last backup (recommended for Google Drive) | Automatic (browser) |
| [Google Drive (Incremental)](#google-drive-incremental) | `gdrive-changes` | Incremental changes since last backup (recommended for Google Drive) | Automatic (browser) |
| [OneDrive](#onedrive) | `onedrive` | Full scan of Microsoft OneDrive | Automatic (browser) |
| [OneDrive (Changes API)](#onedrive-changes-api) | `onedrive-changes` | Incremental changes since last backup (recommended for OneDrive) | Automatic (browser) |
| [OneDrive (Incremental)](#onedrive-incremental) | `onedrive-changes` | Incremental changes since last backup (recommended for OneDrive) | Automatic (browser) |

All sources produce the same snapshot format. You can back up different sources into the same repository, and snapshots are tagged with source metadata so retention policies can be applied per-source.

Expand Down Expand Up @@ -917,7 +916,7 @@ The `-exclude` and `-exclude-file` flags work with SFTP sources. See [Exclude pa

Full scan of a Google Drive account. On each backup, Cloudstic lists every file and folder, compares metadata against the previous snapshot, and uploads anything new or changed.

> **Note:** For routine backups, prefer [`gdrive-changes`](#google-drive-changes-api) instead — it is significantly faster and makes far fewer API requests.
> **Note:** For routine backups, prefer [`gdrive-changes`](#google-drive-incremental) instead — it is significantly faster and makes far fewer API requests.

**When to use:** First backup of a Google Drive, or when you want a guaranteed complete rescan (e.g. after recovering from an error).

Expand All @@ -930,24 +929,20 @@ No configuration is required — Cloudstic ships with built-in OAuth credentials
cloudstic backup -source gdrive:/

# Back up a shared drive
cloudstic backup -source gdrive:/ -drive-id <shared-drive-id>
cloudstic backup -source "gdrive://Company Data"

# Back up only a specific folder
cloudstic backup -source gdrive:/path/to/folder
cloudstic backup -source "gdrive://Company Data/path/to/folder"
```

| Flag | Description |
|------|-------------|
| `-drive-id` | Shared Drive ID (omit for personal My Drive) |

**Environment variables (optional overrides):**

| Variable | Description |
|----------|-------------|
| `GOOGLE_APPLICATION_CREDENTIALS` | Path to your own Google OAuth credentials JSON file (overrides built-in credentials) |
| `GOOGLE_TOKEN_FILE` | Override token cache path (default: `<config-dir>/google_token.json`) |

### Google Drive (Changes API)
### Google Drive (Incremental)

**This is the recommended way to back up Google Drive.** Uses the Google Drive Changes API to fetch only files that changed since the last backup, rather than listing every file on the drive. This dramatically reduces both backup duration and the number of API requests — a drive with 100,000 files but 50 daily changes only needs to process those 50 files instead of listing all 100,000.

Expand All @@ -963,7 +958,7 @@ cloudstic backup -source gdrive-changes
cloudstic backup -source gdrive-changes
```

Uses the same authentication and flags as [Google Drive](#google-drive) (`-drive-id`). No setup required — just run the command and authorize in the browser.
Uses the same authentication and flags as [Google Drive](#google-drive). No setup required — just run the command and authorize in the browser.

> **Tip:** You can use `-source gdrive-changes` from day one — the first run performs a full scan just like `gdrive`. Only fall back to `-source gdrive` if you need to force a complete rescan.

Expand Down Expand Up @@ -1002,7 +997,7 @@ No client secret is needed — Cloudstic uses the public client flow with PKCE.
| `ONEDRIVE_CLIENT_ID` | Azure app client ID (overrides built-in credentials) |
| `ONEDRIVE_TOKEN_FILE` | Override token cache path (default: `<config-dir>/onedrive_token.json`) |

### OneDrive (Changes API)
### OneDrive (Incremental)

**This is the recommended way to back up OneDrive.** Uses the Microsoft Graph delta API to fetch only files that changed since the last backup, rather than listing every file on the drive. This dramatically reduces both backup duration and the number of API requests.

Expand Down Expand Up @@ -1263,10 +1258,9 @@ cloudstic forget -keep-daily 7 -keep-monthly 12 -dry-run
| `AWS_SECRET_ACCESS_KEY` | `-s3-secret-key` | S3 Secret Access Key |
| `CLOUDSTIC_STORE_SFTP_PASSWORD` | `-store-sftp-password` | SFTP password for the store |
| `CLOUDSTIC_STORE_SFTP_KEY` | `-store-sftp-key` | Path to SSH private key for the store |
| `CLOUDSTIC_SOURCE` | `-source` | Source URI: `local:<path>`, `sftp://[user@]host[:port]/<path>`, `gdrive[:<path>]`, `gdrive-changes[:<path>]`, `onedrive[:<path>]`, `onedrive-changes[:<path>]` |
| `CLOUDSTIC_SOURCE` | `-source` | Source URI: `local:<path>`, `sftp://[user@]host[:port]/<path>`, `gdrive[://<Drive Name>][/<path>]`, `gdrive-changes[://<Drive Name>][/<path>]`, `onedrive[://<Drive Name>][/<path>]`, `onedrive-changes[://<Drive Name>][/<path>]` |
| `CLOUDSTIC_SOURCE_SFTP_PASSWORD` | `-source-sftp-password` | SFTP password for the source |
| `CLOUDSTIC_SOURCE_SFTP_KEY` | `-source-sftp-key` | Path to SSH private key for the source |
| `CLOUDSTIC_DRIVE_ID` | `-drive-id` | Shared drive ID for Google Drive |
| `CLOUDSTIC_ENCRYPTION_KEY` | `-encryption-key` | Platform key (hex) |
| `CLOUDSTIC_PASSWORD` | `-password` | Encryption password |
| `CLOUDSTIC_RECOVERY_KEY` | `-recovery-key` | Recovery seed phrase |
Expand Down
Loading
Loading