/
push.go
149 lines (124 loc) · 4.29 KB
/
push.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
package cmd
import (
"context"
"fmt"
"time"
gphotos "github.com/gphotosuploader/google-photos-api-client-go/v2"
"github.com/spf13/cobra"
"golang.org/x/oauth2"
"github.com/gphotosuploader/gphotos-uploader-cli/internal/app"
"github.com/gphotosuploader/gphotos-uploader-cli/internal/cmd/flags"
"github.com/gphotosuploader/gphotos-uploader-cli/internal/config"
"github.com/gphotosuploader/gphotos-uploader-cli/internal/photos"
"github.com/gphotosuploader/gphotos-uploader-cli/internal/task"
"github.com/gphotosuploader/gphotos-uploader-cli/internal/upload"
"github.com/gphotosuploader/gphotos-uploader-cli/internal/worker"
)
// PushCmd holds the required data for the push cmd
type PushCmd struct {
*flags.GlobalFlags
// command flags
NumberOfWorkers int
DryRunMode bool
}
func NewPushCmd(globalFlags *flags.GlobalFlags) *cobra.Command {
cmd := &PushCmd{GlobalFlags: globalFlags}
pushCmd := &cobra.Command{
Use: "push",
Short: "Push local files to Google Photos service",
Long: `Scan configured folders in the configuration and push all new object to Google Photos service.`,
Args: cobra.NoArgs,
RunE: cmd.Run,
}
pushCmd.Flags().IntVar(&cmd.NumberOfWorkers, "workers", 5, "Number of workers")
pushCmd.Flags().BoolVar(&cmd.DryRunMode, "dry-run", false, "Dry run mode")
return pushCmd
}
func (cmd *PushCmd) Run(cobraCmd *cobra.Command, args []string) error {
cfg, err := config.LoadConfigAndValidate(cmd.CfgDir)
if err != nil {
return fmt.Errorf("please review your configuration or run 'gphotos-uploader-cli init': file=%s, err=%s", cmd.CfgDir, err)
}
cli, err := app.Start(cfg)
if err != nil {
return err
}
defer func() {
_ = cli.Stop()
}()
if cmd.DryRunMode {
cli.Logger.Info("Running in dry run mode. No changes will be made.")
}
// get OAuth2 Configuration with our App credentials
oauth2Config := oauth2.Config{
ClientID: cfg.APIAppCredentials.ClientID,
ClientSecret: cfg.APIAppCredentials.ClientSecret,
Endpoint: photos.Endpoint,
Scopes: photos.Scopes,
}
uploadQueue := worker.NewJobQueue(cmd.NumberOfWorkers, cli.Logger)
uploadQueue.Start()
defer uploadQueue.Stop()
time.Sleep(1 * time.Second) // sleeps to avoid log messages colliding with output.
// launch all folder upload jobs
var totalItems int
for _, config := range cfg.Jobs {
folder := upload.UploadFolderJob{
FileTracker: cli.FileTracker,
SourceFolder: config.SourceFolder,
CreateAlbum: config.MakeAlbums.Enabled,
CreateAlbumBasedOn: config.MakeAlbums.Use,
Filter: upload.NewFilter(config.IncludePatterns, config.ExcludePatterns, config.UploadVideos),
}
// get UploadItem{} to be uploaded to Google Photos.
itemsToUpload, err := folder.ScanFolder(cli.Logger)
if err != nil {
cli.Logger.Fatalf("Failed to scan folder %s: %v", config.SourceFolder, err)
}
cli.Logger.Infof("%d files pending to be uploaded in folder '%s'.", len(itemsToUpload), config.SourceFolder)
// get a Google Photos client for the specified account.
ctx := context.Background()
c, err := cli.NewOAuth2Client(ctx, oauth2Config, config.Account)
if err != nil {
return err
}
photosService, err := gphotos.NewClient(c, gphotos.WithSessionStorer(cli.UploadTracker))
if err != nil {
return err
}
// enqueue files to be uploaded. The workers will receive it via channel.
totalItems += len(itemsToUpload)
for _, i := range itemsToUpload {
if cmd.DryRunMode {
uploadQueue.Submit(&task.NoOpJob{})
} else {
uploadQueue.Submit(&task.EnqueuedUpload{
Context: ctx,
PhotosClient: photosService,
FileTracker: cli.FileTracker,
Logger: cli.Logger,
Path: i.Path,
AlbumName: i.AlbumName,
DeleteOnSuccess: config.DeleteAfterUpload,
})
}
}
}
// get responses from the enqueued jobs
var uploadedItems int
for i := 0; i < totalItems; i++ {
r := <-uploadQueue.ChanJobResults()
if r.Err != nil {
cli.Logger.Failf("Error processing %s", r.ID)
} else {
uploadedItems++
cli.Logger.Debugf("Successfully processing %s", r.ID)
}
}
if cmd.DryRunMode {
cli.Logger.Info("Running in dry run mode. No changes has been made.")
} else {
cli.Logger.Infof("%d processed files: %d successfully, %d with errors", totalItems, uploadedItems, totalItems-uploadedItems)
}
return nil
}