From 291e7d60ffa4aa84dc28bcfb4eb0ca41df086b56 Mon Sep 17 00:00:00 2001 From: Aleksandar Nikolikj Date: Tue, 24 Mar 2020 20:28:15 +1100 Subject: [PATCH 1/7] Adding endpoint config to enable minio backend --- pkg/sync/pull.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/sync/pull.go b/pkg/sync/pull.go index 2cb002e..7e0234c 100644 --- a/pkg/sync/pull.go +++ b/pkg/sync/pull.go @@ -288,8 +288,16 @@ func (self *Puller) Pull(remoteUri string, localDir string) string { return fmt.Sprintf("Failed to detect AWS region: %v", err) } } + endpointURL := os.Getenv("AWS_ENDPOINT_URL") + + s3Config := &aws.Config{ + Endpoint: aws.String(endpointURL), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + svc := s3.New(sess, s3Config) - svc := s3.New(sess, aws.NewConfig().WithRegion(region)) downloader := s3manager.NewDownloaderWithClient(svc) // spawn worker goroutines From a8c87bebfe228df62218e90b8e9714705589f902 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Tue, 24 Mar 2020 17:04:21 -0700 Subject: [PATCH 2/7] write out objects in 0666 mode This is that we can run objinsync in a sidecar container under a different uid than the one used in the main application container --- pkg/sync/pull.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/sync/pull.go b/pkg/sync/pull.go index 771251b..04a314f 100644 --- a/pkg/sync/pull.go +++ b/pkg/sync/pull.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "fmt" "io" - "io/ioutil" "os" "path/filepath" "strings" @@ -151,9 +150,11 @@ func (self *Puller) downloadHandler(task DownloadTask, downloader GenericDownloa } // create file - tmpfile, err := ioutil.TempFile(self.workingDir, filepath.Base(task.LocalPath)) + tmpfileName := fmt.Sprintf("%x", md5.Sum([]byte(task.LocalPath))) + tmpfilePath := filepath.Join(self.workingDir, tmpfileName) + tmpfile, err := os.OpenFile(tmpfilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { - self.errMsgQueue <- fmt.Sprintf("Failed to create file for download: %v", err) + self.errMsgQueue <- fmt.Sprintf("Failed to create temp file for download: %v", err) return } defer tmpfile.Close() @@ -164,7 +165,7 @@ func (self *Puller) downloadHandler(task DownloadTask, downloader GenericDownloa }) // use rename to make file update atomic - err = os.Rename(tmpfile.Name(), task.LocalPath) + err = os.Rename(tmpfilePath, task.LocalPath) if err != nil { self.errMsgQueue <- fmt.Sprintf("Failed to replace file %s for download: %v", task.LocalPath, err) return From f02382ceba66ef707b0c485447f775e7aa7b7228 Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Tue, 24 Mar 2020 17:18:28 -0700 Subject: [PATCH 3/7] feat: override default file mode from command line --- main.go | 21 ++++++++++++++++----- pkg/sync/pull.go | 20 +++++++++++++------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index 9c76db0..10402a6 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "runtime/debug" + "strconv" "time" "github.com/getsentry/sentry-go" @@ -19,11 +20,12 @@ import ( ) var ( - InitialRunFinished atomic.Bool - FlagRunOnce bool - FlagStatusAddr = ":8087" - FlagExclude []string - FlagScratch bool + InitialRunFinished atomic.Bool + FlagRunOnce bool + FlagStatusAddr = ":8087" + FlagExclude []string + FlagScratch bool + FlagDefaultFileMode = "0666" metricsSyncTime = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "objinsync", @@ -106,6 +108,13 @@ func main() { if !FlagScratch { puller.PopulateChecksum() } + if FlagDefaultFileMode != "" { + mode, err := strconv.ParseInt(FlagDefaultFileMode, 8, 64) + if err != nil { + log.Fatal("invalid default file mode", err) + } + puller.SetDefaultFileMode(os.FileMode(mode)) + } pull := func() { start := time.Now() @@ -158,6 +167,8 @@ func main() { false, "skip checksums calculation and override all files during the initial sync", ) + pullCmd.PersistentFlags().StringVarP( + &FlagDefaultFileMode, "default-file-mode", "m", "0666", "default mode to use for creating local file") rootCmd.AddCommand(pullCmd) rootCmd.Execute() diff --git a/pkg/sync/pull.go b/pkg/sync/pull.go index 04a314f..c04be8b 100644 --- a/pkg/sync/pull.go +++ b/pkg/sync/pull.go @@ -103,6 +103,7 @@ type Puller struct { LocalDir string workingDir string + defaultMode os.FileMode exclude []string workerCnt int uidCache map[string]string @@ -152,7 +153,7 @@ func (self *Puller) downloadHandler(task DownloadTask, downloader GenericDownloa // create file tmpfileName := fmt.Sprintf("%x", md5.Sum([]byte(task.LocalPath))) tmpfilePath := filepath.Join(self.workingDir, tmpfileName) - tmpfile, err := os.OpenFile(tmpfilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + tmpfile, err := os.OpenFile(tmpfilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, self.defaultMode) if err != nil { self.errMsgQueue <- fmt.Sprintf("Failed to create temp file for download: %v", err) return @@ -447,17 +448,22 @@ func (self *Puller) PopulateChecksum() { } } +func (self *Puller) SetDefaultFileMode(mode os.FileMode) { + self.defaultMode = mode +} + func NewPuller(remoteUri string, localDir string) (*Puller, error) { if _, err := os.Stat(localDir); os.IsNotExist(err) { return nil, fmt.Errorf("local directory `%s` does not exist: %v", localDir, err) } return &Puller{ - RemoteUri: remoteUri, - LocalDir: localDir, - workingDir: filepath.Join(localDir, ".objinsync"), - workerCnt: 5, - uidCache: map[string]string{}, - uidLock: &sync.Mutex{}, + RemoteUri: remoteUri, + LocalDir: localDir, + workingDir: filepath.Join(localDir, ".objinsync"), + defaultMode: 0666, + workerCnt: 5, + uidCache: map[string]string{}, + uidLock: &sync.Mutex{}, }, nil } From 54bad26d8ef993aeb9a7a672ed691027a9d27aae Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Tue, 24 Mar 2020 17:27:04 -0700 Subject: [PATCH 4/7] change default file mode to 0664 to be more sensible --- main.go | 4 ++-- pkg/sync/pull.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 10402a6..1d5c5fd 100644 --- a/main.go +++ b/main.go @@ -25,7 +25,7 @@ var ( FlagStatusAddr = ":8087" FlagExclude []string FlagScratch bool - FlagDefaultFileMode = "0666" + FlagDefaultFileMode = "0664" metricsSyncTime = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "objinsync", @@ -168,7 +168,7 @@ func main() { "skip checksums calculation and override all files during the initial sync", ) pullCmd.PersistentFlags().StringVarP( - &FlagDefaultFileMode, "default-file-mode", "m", "0666", "default mode to use for creating local file") + &FlagDefaultFileMode, "default-file-mode", "m", "0664", "default mode to use for creating local file") rootCmd.AddCommand(pullCmd) rootCmd.Execute() diff --git a/pkg/sync/pull.go b/pkg/sync/pull.go index c04be8b..59c83e1 100644 --- a/pkg/sync/pull.go +++ b/pkg/sync/pull.go @@ -461,7 +461,7 @@ func NewPuller(remoteUri string, localDir string) (*Puller, error) { RemoteUri: remoteUri, LocalDir: localDir, workingDir: filepath.Join(localDir, ".objinsync"), - defaultMode: 0666, + defaultMode: 0664, workerCnt: 5, uidCache: map[string]string{}, uidLock: &sync.Mutex{}, From 8b1852cb82503c5d2a0472d90d0e40d70b37d823 Mon Sep 17 00:00:00 2001 From: Aleksandar Nikolikj Date: Tue, 24 Mar 2020 20:28:15 +1100 Subject: [PATCH 5/7] Adding endpoint config to enable minio backend --- pkg/sync/pull.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/sync/pull.go b/pkg/sync/pull.go index 59c83e1..d39ea6b 100644 --- a/pkg/sync/pull.go +++ b/pkg/sync/pull.go @@ -307,8 +307,16 @@ func (self *Puller) Pull() string { return fmt.Sprintf("Failed to detect AWS region: %v", err) } } + endpointURL := os.Getenv("AWS_ENDPOINT_URL") + + s3Config := &aws.Config{ + Endpoint: aws.String(endpointURL), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + svc := s3.New(sess, s3Config) - svc := s3.New(sess, aws.NewConfig().WithRegion(region)) downloader := s3manager.NewDownloaderWithClient(svc) if err := self.SetupWorkingDir(); err != nil { From 5127fe40ef3ff0e471434c184d3fe6943907971f Mon Sep 17 00:00:00 2001 From: Aleksandar Nikolikj Date: Tue, 24 Mar 2020 20:28:15 +1100 Subject: [PATCH 6/7] Adding endpoint config to enable minio backend --- pkg/sync/pull.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/sync/pull.go b/pkg/sync/pull.go index 59c83e1..d39ea6b 100644 --- a/pkg/sync/pull.go +++ b/pkg/sync/pull.go @@ -307,8 +307,16 @@ func (self *Puller) Pull() string { return fmt.Sprintf("Failed to detect AWS region: %v", err) } } + endpointURL := os.Getenv("AWS_ENDPOINT_URL") + + s3Config := &aws.Config{ + Endpoint: aws.String(endpointURL), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + svc := s3.New(sess, s3Config) - svc := s3.New(sess, aws.NewConfig().WithRegion(region)) downloader := s3manager.NewDownloaderWithClient(svc) if err := self.SetupWorkingDir(); err != nil { From 95912375497f90cd08f4b968c1d8717c5733f1ab Mon Sep 17 00:00:00 2001 From: Qingping Hou Date: Wed, 25 Mar 2020 00:14:01 -0700 Subject: [PATCH 7/7] feat: make endpoint and disable ssl configurable --- main.go | 8 ++++++++ pkg/sync/pull.go | 20 ++++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index 1d5c5fd..0e97a9a 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,8 @@ var ( FlagExclude []string FlagScratch bool FlagDefaultFileMode = "0664" + FlagS3Endpoint = "" + FlagDisableSSL = false metricsSyncTime = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "objinsync", @@ -102,6 +104,8 @@ func main() { if err != nil { log.Fatal(err) } + puller.DisableSSL = FlagDisableSSL + puller.S3Endpoint = FlagS3Endpoint if FlagExclude != nil { puller.AddExcludePatterns(FlagExclude) } @@ -156,6 +160,8 @@ func main() { pullCmd.PersistentFlags().BoolVarP( &FlagRunOnce, "once", "o", false, "run action once and then exit") + pullCmd.PersistentFlags().BoolVarP( + &FlagDisableSSL, "disable-ssl", "", false, "disable SSL for object storage connection") pullCmd.PersistentFlags().StringVarP( &FlagStatusAddr, "status-addr", "s", ":8087", "binding address for status endpoint") pullCmd.PersistentFlags().StringSliceVarP( @@ -169,6 +175,8 @@ func main() { ) pullCmd.PersistentFlags().StringVarP( &FlagDefaultFileMode, "default-file-mode", "m", "0664", "default mode to use for creating local file") + pullCmd.PersistentFlags().StringVarP( + &FlagS3Endpoint, "s3-endpoint", "", "", "override endpoint to use for remote object store (e.g. minio)") rootCmd.AddCommand(pullCmd) rootCmd.Execute() diff --git a/pkg/sync/pull.go b/pkg/sync/pull.go index d39ea6b..c58b439 100644 --- a/pkg/sync/pull.go +++ b/pkg/sync/pull.go @@ -99,8 +99,10 @@ func uidFromLocalPath(localPath string) (string, error) { } type Puller struct { - RemoteUri string - LocalDir string + RemoteUri string + LocalDir string + DisableSSL bool + S3Endpoint string workingDir string defaultMode os.FileMode @@ -307,13 +309,14 @@ func (self *Puller) Pull() string { return fmt.Sprintf("Failed to detect AWS region: %v", err) } } - endpointURL := os.Getenv("AWS_ENDPOINT_URL") - s3Config := &aws.Config{ - Endpoint: aws.String(endpointURL), - Region: aws.String(region), - DisableSSL: aws.Bool(true), - S3ForcePathStyle: aws.Bool(true), + s3Config := &aws.Config{Region: aws.String(region)} + if self.DisableSSL { + s3Config.DisableSSL = aws.Bool(true) + } + if self.S3Endpoint != "" { + s3Config.Endpoint = aws.String(self.S3Endpoint) + s3Config.S3ForcePathStyle = aws.Bool(true) } svc := s3.New(sess, s3Config) @@ -468,6 +471,7 @@ func NewPuller(remoteUri string, localDir string) (*Puller, error) { return &Puller{ RemoteUri: remoteUri, LocalDir: localDir, + DisableSSL: false, workingDir: filepath.Join(localDir, ".objinsync"), defaultMode: 0664, workerCnt: 5,