From 5a29fc9346b56b6ab5c6377f58d49675ace49838 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 6 Jun 2018 15:04:06 +0800 Subject: [PATCH] Refactor object check logic - remove the quick md5 sum check - remove the useless disable and size check - add the last modified check Signed-off-by: Xuanwo --- constants/config.go | 5 ---- constants/model.go | 6 ++--- migrate/migrate.go | 10 -------- migrate/object.go | 57 ++++++++++----------------------------------- model/task.go | 6 ++--- 5 files changed, 16 insertions(+), 68 deletions(-) diff --git a/constants/config.go b/constants/config.go index 49d51ee..33a661c 100644 --- a/constants/config.go +++ b/constants/config.go @@ -13,11 +13,6 @@ database_file: ~/.qscamel/db var ( // DefaultConcurrency is default num of objects being migrated concurrently. DefaultConcurrency = runtime.NumCPU() * 10 - // GoldenRatio is the most beautiful number on earth. - // ref: https://en.wikipedia.org/wiki/Golden_ratio - GoldenRatio = 0.618 - // MB represent 1024*1024 Byte. - MB int64 = 1024 * 1024 ) // Path store all path related constants. diff --git a/constants/model.go b/constants/model.go index 81a36fc..26d3da9 100644 --- a/constants/model.go +++ b/constants/model.go @@ -16,10 +16,8 @@ const ( // Constants for task ignore existing config. const ( - TaskIgnoreExistingDisable = "disable" - TaskIgnoreExistingSize = "size" - TaskIgnoreExistingQuickMD5Sum = "quick_md5sum" - TaskIgnoreExistingFullMD5Sum = "full_md5sum" + TaskIgnoreExistingLastModified = "last_modified" + TaskIgnoreExistingMD5Sum = "md5sum" ) // Constants for database key. diff --git a/migrate/migrate.go b/migrate/migrate.go index 956de44..c61e872 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -48,8 +48,6 @@ var ( src endpoint.Source dst endpoint.Destination - - md5sum func(ctx context.Context, e endpoint.Base, o *model.Object) (md5 string, err error) ) // Execute will execute migrate task. @@ -146,14 +144,6 @@ func run(ctx context.Context) (err error) { return } - // Set md5sum function. - if t.IgnoreExisting == constants.TaskIgnoreExistingQuickMD5Sum { - md5sum = quickSumObject - } - if t.IgnoreExisting == constants.TaskIgnoreExistingFullMD5Sum { - md5sum = fullSumObject - } - switch t.Type { case constants.TaskTypeCopy: t.Handle = copyObject diff --git a/migrate/object.go b/migrate/object.go index 73416f9..8fd8e65 100644 --- a/migrate/object.go +++ b/migrate/object.go @@ -15,8 +15,7 @@ import ( // checkObject will tell whether an object is ok. func checkObject(ctx context.Context, o *model.Object) (ok bool, err error) { - if t.IgnoreExisting == "" || - t.IgnoreExisting == constants.TaskIgnoreExistingDisable { + if t.IgnoreExisting == "" { return false, nil } @@ -38,16 +37,20 @@ func checkObject(ctx context.Context, o *model.Object) (ok bool, err error) { if do == nil { return } - // Check size. if so.Size != do.Size { logrus.Infof("Object %s size is not match, execute an operation on it.", o.Key) return } - if t.IgnoreExisting == constants.TaskIgnoreExistingSize { + // Check last modified + if t.IgnoreExisting == constants.TaskIgnoreExistingLastModified { + if so.LastModified > do.LastModified { + logrus.Infof("Object %s was modified, execute an operation on it.", o.Key) + return + } logrus.Infof("Object %s check passed, ignore.", o.Key) - return + return true, nil } // Check md5. @@ -114,59 +117,23 @@ func statObject( return } - if t.IgnoreExisting != constants.TaskIgnoreExistingQuickMD5Sum && - t.IgnoreExisting != constants.TaskIgnoreExistingFullMD5Sum { + if t.IgnoreExisting != constants.TaskIgnoreExistingMD5Sum { return } if len(ro.MD5) != 32 { - ro.MD5, err = md5sum(ctx, e, o) + ro.MD5, err = md5SumObject(ctx, e, o) if err != nil { logrus.Errorf( "%s calculate object %s md5 failed for %v.", e.Name(ctx), o.Key, err) return } - // If it's the full md5, we can update the object md5. - if t.IgnoreExisting == constants.TaskIgnoreExistingFullMD5Sum { - o.MD5 = ro.MD5 - } } return } -// quickSumObject will get the object's quick md5 -func quickSumObject( - ctx context.Context, e endpoint.Base, o *model.Object, -) (m string, err error) { - // If object size <= 3MB, use full sum instead. - if o.Size <= 3*constants.MB { - return fullSumObject(ctx, e, o) - } - - goldenPoint := int64(float64(o.Size) * constants.GoldenRatio) - - pos := [][]int64{ - {0, constants.MB - 1}, - {goldenPoint, goldenPoint + constants.MB - 1}, - {o.Size - constants.MB - 1, o.Size - 1}, - } - content := make([]byte, 3*constants.MB) - - for _, v := range pos { - c, err := e.ReadAt(ctx, o.Key, v[0], v[1]) - if err != nil { - logrus.Errorf("%s read object %s failed for %v.", e.Name(ctx), o.Key, err) - return "", err - } - content = append(content, c...) - } - - sum := md5.Sum(content) - return hex.EncodeToString(sum[:]), nil -} - -// fullSumObject will get the object's full md5 -func fullSumObject( +// md5SumObject will get the object's md5 +func md5SumObject( ctx context.Context, e endpoint.Base, o *model.Object, ) (m string, err error) { r, err := e.Read(ctx, o.Key) diff --git a/model/task.go b/model/task.go index df4b8e4..51a53db 100644 --- a/model/task.go +++ b/model/task.go @@ -103,10 +103,8 @@ func LoadTaskFromContent(content []byte) (t *Task, err error) { func (t *Task) Check() error { switch t.IgnoreExisting { case "": - case constants.TaskIgnoreExistingDisable: - case constants.TaskIgnoreExistingSize: - case constants.TaskIgnoreExistingQuickMD5Sum: - case constants.TaskIgnoreExistingFullMD5Sum: + case constants.TaskIgnoreExistingLastModified: + case constants.TaskIgnoreExistingMD5Sum: default: logrus.Errorf("%s is not a valid value for task ignore existing", t.IgnoreExisting) return constants.ErrTaskInvalid