-
Notifications
You must be signed in to change notification settings - Fork 1.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
GDrive: Fix fieldNotWritable, Add SupportSharedDrive, Add service_account method #512
base: main
Are you sure you want to change the base?
Changes from 14 commits
106609d
9cd6e4f
f4f04b9
f318207
2675350
cd9c56e
73ac766
fb87a5b
f096300
71d8858
f73922d
f517bdc
0327122
be00bb8
91492e1
478341c
c96cf41
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -24,13 +24,12 @@ import ( | |
type GDrive struct { | ||
service *drive.Service | ||
rootID string | ||
basedir string | ||
localConfigPath string | ||
authType string | ||
chunkSize int | ||
logger *log.Logger | ||
} | ||
|
||
const gDriveRootConfigFile = "root_id.conf" | ||
const gDriveTokenJSONFile = "token.json" | ||
const gDriveDirectoryMimeType = "application/vnd.google-apps.folder" | ||
|
||
|
@@ -45,65 +44,66 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir | |
} | ||
|
||
// If modifying these scopes, delete your previously saved client_secret.json. | ||
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) | ||
if err != nil { | ||
return nil, err | ||
} | ||
var httpClient *http.Client | ||
var AuthType string | ||
|
||
if strings.Contains(string(b), `"type": "service_account"`) { | ||
AuthType = "service" | ||
|
||
httpClient := getGDriveClient(ctx, config, localConfigPath, logger) | ||
logger.Println("GDrive: using Service Account credentials") | ||
config, err := google.JWTConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) | ||
if err != nil { | ||
return nil, err | ||
} | ||
httpClient = config.Client(ctx) | ||
} else { | ||
AuthType = "oauth" | ||
|
||
if localConfigPath == "" { | ||
return nil, fmt.Errorf("gdrive-local-config-path not set") | ||
} | ||
|
||
logger.Println("GDrive: using OAuth2 credentials") | ||
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope) | ||
if err != nil { | ||
return nil, err | ||
} | ||
httpClient = getGDriveClientFromToken(ctx, config, localConfigPath, logger) | ||
} | ||
|
||
srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient)) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger} | ||
err = storage.setupRoot() | ||
storage := &GDrive{service: srv, rootID: basedir, localConfigPath: localConfigPath, authType: AuthType, chunkSize: chunkSize, logger: logger} | ||
err = storage.checkRoot() | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
return storage, nil | ||
} | ||
|
||
func (s *GDrive) setupRoot() error { | ||
rootFileConfig := filepath.Join(s.localConfigPath, gDriveRootConfigFile) | ||
|
||
rootID, err := ioutil.ReadFile(rootFileConfig) | ||
if err != nil && !os.IsNotExist(err) { | ||
return err | ||
} | ||
|
||
if string(rootID) != "" { | ||
s.rootID = string(rootID) | ||
return nil | ||
} | ||
|
||
dir := &drive.File{ | ||
Name: s.basedir, | ||
MimeType: gDriveDirectoryMimeType, | ||
} | ||
|
||
di, err := s.service.Files.Create(dir).Fields("id").Do() | ||
if err != nil { | ||
return err | ||
} | ||
|
||
s.rootID = di.Id | ||
err = ioutil.WriteFile(rootFileConfig, []byte(s.rootID), os.FileMode(0600)) | ||
if err != nil { | ||
return err | ||
func (s *GDrive) checkRoot() error { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is a breaking change. basedir is the name of the folder to be created in the gdrive, rootID is the id of the folder once created and that we save. the two carry two different information. if we replce rootID value with basedir value:
I guess you meant this change to be the way to select a shared drive, is it? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep! I changed basedir to insert folder id, to select rootID. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I cannot accept a breaking change, sorry |
||
if s.rootID == "root" { | ||
switch s.authType { | ||
case "service": | ||
return fmt.Errorf("GDrive: Folder \"root\" is not available when using Service Account credentials") | ||
case "oauth": | ||
s.logger.Println("GDrive: Warning: Folder \"root\" is not recommended.") | ||
} | ||
} | ||
|
||
return nil | ||
_, err := s.service.Files.Get(s.rootID).SupportsAllDrives(true).Do() | ||
return err | ||
} | ||
|
||
func (s *GDrive) hasChecksum(f *drive.File) bool { | ||
return f.Md5Checksum != "" | ||
} | ||
|
||
func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) { | ||
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do() | ||
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Do() | ||
DoyunShin marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
|
||
func (s *GDrive) findID(filename string, token string) (string, error) { | ||
|
@@ -184,7 +184,7 @@ func (s *GDrive) Head(ctx context.Context, token string, filename string) (conte | |
} | ||
|
||
var fi *drive.File | ||
if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").Do(); err != nil { | ||
if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").SupportsAllDrives(true).Do(); err != nil { | ||
return | ||
} | ||
|
||
|
@@ -202,7 +202,7 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader | |
} | ||
|
||
var fi *drive.File | ||
fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").Do() | ||
fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").SupportsAllDrives(true).Do() | ||
if err != nil { | ||
return | ||
} | ||
|
@@ -214,7 +214,7 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader | |
contentLength = uint64(fi.Size) | ||
|
||
var res *http.Response | ||
res, err = s.service.Files.Get(fileID).Context(ctx).Download() | ||
res, err = s.service.Files.Get(fileID).Context(ctx).SupportsAllDrives(true).Download() | ||
if err != nil { | ||
return | ||
} | ||
|
@@ -227,15 +227,15 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader | |
// Delete removes a file from storage | ||
func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err error) { | ||
metadata, _ := s.findID(fmt.Sprintf("%s.metadata", filename), token) | ||
_ = s.service.Files.Delete(metadata).Do() | ||
_ = s.service.Files.Delete(metadata).SupportsAllDrives(true).Do() | ||
|
||
var fileID string | ||
fileID, err = s.findID(filename, token) | ||
if err != nil { | ||
return | ||
} | ||
|
||
err = s.service.Files.Delete(fileID).Context(ctx).Do() | ||
err = s.service.Files.Delete(fileID).Context(ctx).SupportsAllDrives(true).Do() | ||
return | ||
} | ||
|
||
|
@@ -252,7 +252,7 @@ func (s *GDrive) Purge(ctx context.Context, days time.Duration) (err error) { | |
|
||
for 0 < len(l.Files) { | ||
for _, fi := range l.Files { | ||
err = s.service.Files.Delete(fi.Id).Context(ctx).Do() | ||
err = s.service.Files.Delete(fi.Id).Context(ctx).SupportsAllDrives(true).Do() | ||
if err != nil { | ||
return | ||
} | ||
|
@@ -296,10 +296,9 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader | |
Name: token, | ||
Parents: []string{s.rootID}, | ||
MimeType: gDriveDirectoryMimeType, | ||
Size: int64(contentLength), | ||
} | ||
|
||
di, err := s.service.Files.Create(dir).Fields("id").Do() | ||
di, err := s.service.Files.Create(dir).Fields("id").SupportsAllDrives(true).Do() | ||
if err != nil { | ||
return err | ||
} | ||
|
@@ -314,7 +313,7 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader | |
MimeType: contentType, | ||
} | ||
|
||
_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do() | ||
_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).SupportsAllDrives(true).Do() | ||
|
||
if err != nil { | ||
return err | ||
|
@@ -324,7 +323,7 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader | |
} | ||
|
||
// Retrieve a token, saves the token, then returns the generated client. | ||
func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client { | ||
func getGDriveClientFromToken(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client { | ||
tokenFile := filepath.Join(localConfigPath, gDriveTokenJSONFile) | ||
tok, err := gDriveTokenFromFile(tokenFile) | ||
if err != nil { | ||
|
@@ -337,10 +336,13 @@ func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath | |
|
||
// Request a token from the web, then returns the retrieved token. | ||
func getGDriveTokenFromWeb(ctx context.Context, config *oauth2.Config, logger *log.Logger) *oauth2.Token { | ||
config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob" | ||
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline) | ||
|
||
fmt.Printf("Go to the following link in your browser then type the "+ | ||
"authorization code: \n%v\n", authURL) | ||
"authorization code.\n%v\n", authURL) | ||
|
||
fmt.Printf("Authorization code: ") | ||
DoyunShin marked this conversation as resolved.
Show resolved
Hide resolved
|
||
var authCode string | ||
if _, err := fmt.Scan(&authCode); err != nil { | ||
logger.Fatalf("Unable to read authorization code %v", err) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
let's have a separate cli param for service account file
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
param like --gdrive-type=service / oauth2?
or we can actually check in json.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Oauth2
{ "installed": { "client_id": "clientid", "project_id": "projectname", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_secret": "clientsecret" } }
Service Account
{ "type": "service_account", "project_id": "projectname", "private_key_id": "keyid", "private_key": "-----BEGIN PRIVATE KEY-----\nKEYSECRET\n-----END PRIVATE KEY-----\n", "client_email": "transfersh@projectname.iam.gserviceaccount.com", "client_id": "clientid", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/transfersh%40projectname.iam.gserviceaccount.com" }
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
param like
gdrive-client-json-filepath
:gdrive-service-account-json-filepath
only one of the two can be set
in the factory you can translate this to a single param with the path and a new param for the type (service account/oauth2): it is up to you