Permalink
Browse files

Refactor object check logic

- remove the quick md5 sum check
- remove the useless disable and size check
- add the last modified check

Signed-off-by: Xuanwo <xuanwo@yunify.com>
  • Loading branch information...
Xuanwo committed Jun 6, 2018
1 parent c328e07 commit 5a29fc9346b56b6ab5c6377f58d49675ace49838
Showing with 16 additions and 68 deletions.
  1. +0 −5 constants/config.go
  2. +2 −4 constants/model.go
  3. +0 −10 migrate/migrate.go
  4. +12 −45 migrate/object.go
  5. +2 −4 model/task.go
@@ -13,11 +13,6 @@ database_file: ~/.qscamel/db
var (
// DefaultConcurrency is default num of objects being migrated concurrently.
DefaultConcurrency = runtime.NumCPU() * 10
// GoldenRatio is the most beautiful number on earth.
// ref: https://en.wikipedia.org/wiki/Golden_ratio
GoldenRatio = 0.618
// MB represent 1024*1024 Byte.
MB int64 = 1024 * 1024
)
// Path store all path related constants.
@@ -16,10 +16,8 @@ const (
// Constants for task ignore existing config.
const (
TaskIgnoreExistingDisable = "disable"
TaskIgnoreExistingSize = "size"
TaskIgnoreExistingQuickMD5Sum = "quick_md5sum"
TaskIgnoreExistingFullMD5Sum = "full_md5sum"
TaskIgnoreExistingLastModified = "last_modified"
TaskIgnoreExistingMD5Sum = "md5sum"
)
// Constants for database key.
@@ -48,8 +48,6 @@ var (
src endpoint.Source
dst endpoint.Destination
md5sum func(ctx context.Context, e endpoint.Base, o *model.Object) (md5 string, err error)
)
// Execute will execute migrate task.
@@ -146,14 +144,6 @@ func run(ctx context.Context) (err error) {
return
}
// Set md5sum function.
if t.IgnoreExisting == constants.TaskIgnoreExistingQuickMD5Sum {
md5sum = quickSumObject
}
if t.IgnoreExisting == constants.TaskIgnoreExistingFullMD5Sum {
md5sum = fullSumObject
}
switch t.Type {
case constants.TaskTypeCopy:
t.Handle = copyObject
@@ -15,8 +15,7 @@ import (
// checkObject will tell whether an object is ok.
func checkObject(ctx context.Context, o *model.Object) (ok bool, err error) {
if t.IgnoreExisting == "" ||
t.IgnoreExisting == constants.TaskIgnoreExistingDisable {
if t.IgnoreExisting == "" {
return false, nil
}
@@ -38,16 +37,20 @@ func checkObject(ctx context.Context, o *model.Object) (ok bool, err error) {
if do == nil {
return
}
// Check size.
if so.Size != do.Size {
logrus.Infof("Object %s size is not match, execute an operation on it.", o.Key)
return
}
if t.IgnoreExisting == constants.TaskIgnoreExistingSize {
// Check last modified
if t.IgnoreExisting == constants.TaskIgnoreExistingLastModified {
if so.LastModified > do.LastModified {
logrus.Infof("Object %s was modified, execute an operation on it.", o.Key)
return
}
logrus.Infof("Object %s check passed, ignore.", o.Key)
return
return true, nil
}
// Check md5.
@@ -114,59 +117,23 @@ func statObject(
return
}
if t.IgnoreExisting != constants.TaskIgnoreExistingQuickMD5Sum &&
t.IgnoreExisting != constants.TaskIgnoreExistingFullMD5Sum {
if t.IgnoreExisting != constants.TaskIgnoreExistingMD5Sum {
return
}
if len(ro.MD5) != 32 {
ro.MD5, err = md5sum(ctx, e, o)
ro.MD5, err = md5SumObject(ctx, e, o)
if err != nil {
logrus.Errorf(
"%s calculate object %s md5 failed for %v.", e.Name(ctx), o.Key, err)
return
}
// If it's the full md5, we can update the object md5.
if t.IgnoreExisting == constants.TaskIgnoreExistingFullMD5Sum {
o.MD5 = ro.MD5
}
}
return
}
// quickSumObject will get the object's quick md5
func quickSumObject(
ctx context.Context, e endpoint.Base, o *model.Object,
) (m string, err error) {
// If object size <= 3MB, use full sum instead.
if o.Size <= 3*constants.MB {
return fullSumObject(ctx, e, o)
}
goldenPoint := int64(float64(o.Size) * constants.GoldenRatio)
pos := [][]int64{
{0, constants.MB - 1},
{goldenPoint, goldenPoint + constants.MB - 1},
{o.Size - constants.MB - 1, o.Size - 1},
}
content := make([]byte, 3*constants.MB)
for _, v := range pos {
c, err := e.ReadAt(ctx, o.Key, v[0], v[1])
if err != nil {
logrus.Errorf("%s read object %s failed for %v.", e.Name(ctx), o.Key, err)
return "", err
}
content = append(content, c...)
}
sum := md5.Sum(content)
return hex.EncodeToString(sum[:]), nil
}
// fullSumObject will get the object's full md5
func fullSumObject(
// md5SumObject will get the object's md5
func md5SumObject(
ctx context.Context, e endpoint.Base, o *model.Object,
) (m string, err error) {
r, err := e.Read(ctx, o.Key)
@@ -103,10 +103,8 @@ func LoadTaskFromContent(content []byte) (t *Task, err error) {
func (t *Task) Check() error {
switch t.IgnoreExisting {
case "":
case constants.TaskIgnoreExistingDisable:
case constants.TaskIgnoreExistingSize:
case constants.TaskIgnoreExistingQuickMD5Sum:
case constants.TaskIgnoreExistingFullMD5Sum:
case constants.TaskIgnoreExistingLastModified:
case constants.TaskIgnoreExistingMD5Sum:
default:
logrus.Errorf("%s is not a valid value for task ignore existing", t.IgnoreExisting)
return constants.ErrTaskInvalid

0 comments on commit 5a29fc9

Please sign in to comment.