From 02bf0531c6bf825ab714e8672e3f2ab34396780c Mon Sep 17 00:00:00 2001 From: Emmanuel Date: Fri, 10 Dec 2021 12:27:47 +0100 Subject: [PATCH] Fixes applied to migration process (#862) * support to copy large files (use CopyBuffer to copy the file from src to dest) * rollback migration process if an error is found while executing it --- core/migration/migration.go | 39 +++++++++++++++++++++++------- fs/fs.go | 48 ++++++++++++++++++++++++++++++++----- 2 files changed, 73 insertions(+), 14 deletions(-) diff --git a/core/migration/migration.go b/core/migration/migration.go index 1e6b3bb0a..09faeea0c 100644 --- a/core/migration/migration.go +++ b/core/migration/migration.go @@ -2,6 +2,7 @@ package migration import ( "fmt" + "os" "path" "github.com/drand/drand/common" @@ -29,8 +30,30 @@ func CheckSBFolderStructure(baseFolder string) bool { // MigrateSBFolderStructure will migrate the file store structure from single-beacon drand version // to the new structure created to support multi-beacon feature. This should be called on any function -// which reads file store from disk, so we are sure the structure is the correct one. +// which reads file store from disk, so we are sure the structure is the correct one. It will check if +// the process ends successfully or not. If something went wrong, a rollback process will be executed. func MigrateSBFolderStructure(baseFolder string) error { + if err := runSBMigration(baseFolder); err != nil { + multiBeaconFolderPath := path.Join(baseFolder, common.MultiBeaconFolder) + isMigrationDone := fs.FolderExists(baseFolder, multiBeaconFolderPath) + + if isMigrationDone { + if localErr := os.RemoveAll(multiBeaconFolderPath); localErr != nil { + return fmt.Errorf("we could not rollback the migration, please remove %s before run daemon again. Err: %s", + multiBeaconFolderPath, err.Error()) + } + } + + return err + } + + return nil +} + +// runSBMigration will migrate the file store structure from single-beacon drand version +// to the new structure created to support multi-beacon feature. It has the logic of the +// migration process. +func runSBMigration(baseFolder string) error { groupFolderPath := path.Join(baseFolder, key.GroupFolderName) keyFolderPath := path.Join(baseFolder, key.KeyFolderName) dbFolderPath := path.Join(baseFolder, core.DefaultDBFolder) @@ -46,21 +69,21 @@ func MigrateSBFolderStructure(baseFolder string) error { } if fs.CreateSecureFolder(path.Join(baseFolder, common.MultiBeaconFolder)) == "" { - return fmt.Errorf("something went wrong with the multi beacon folder. Make sure that you have the appropriate rights") + return fmt.Errorf("something went wrong with the multi beacon folder, make sure that you have the appropriate rights") } // Create new folders to move actual files found. If one of them exists, we will be sure the all new structures have been created if isGroupFound || isKeyFound || isDBFound { if fs.CreateSecureFolder(path.Join(baseFolder, common.MultiBeaconFolder, common.DefaultBeaconID, key.GroupFolderName)) == "" { - return fmt.Errorf("something went wrong with the group folder. Make sure that you have the appropriate rights") + return fmt.Errorf("something went wrong with the group folder, make sure that you have the appropriate rights") } if fs.CreateSecureFolder(path.Join(baseFolder, common.MultiBeaconFolder, common.DefaultBeaconID, key.KeyFolderName)) == "" { - return fmt.Errorf("something went wrong with the key folder. Make sure that you have the appropriate rights") + return fmt.Errorf("something went wrong with the key folder, make sure that you have the appropriate rights") } if fs.CreateSecureFolder(path.Join(baseFolder, common.MultiBeaconFolder, common.DefaultBeaconID, core.DefaultDBFolder)) == "" { - return fmt.Errorf("something went wrong with the db folder. Make sure that you have the appropriate rights") + return fmt.Errorf("something went wrong with the db folder, make sure that you have the appropriate rights") } } @@ -70,7 +93,7 @@ func MigrateSBFolderStructure(baseFolder string) error { // Copy files to new destinations (only if the folder is found) if err := fs.CopyFolder(oldPath, newPath); err != nil { - return fmt.Errorf("something went wrong with the new group folder. Make sure that you have the appropriate rights") + return fmt.Errorf("something went wrong with the new group folder. make sure that you have the appropriate rights") } } @@ -80,7 +103,7 @@ func MigrateSBFolderStructure(baseFolder string) error { // Copy files to new destinations (only if the folder is found) if err := fs.CopyFolder(oldPath, newPath); err != nil { - return fmt.Errorf("something went wrong with the new key folder. Make sure that you have the appropriate rights") + return fmt.Errorf("something went wrong with the new key folder. make sure that you have the appropriate rights") } } @@ -90,7 +113,7 @@ func MigrateSBFolderStructure(baseFolder string) error { // Copy files to new destinations (only if the folder is found) if err := fs.CopyFolder(oldPath, newPath); err != nil { - return fmt.Errorf("something went wrong with the new db folder. Make sure that you have the appropriate rights") + return fmt.Errorf("something went wrong with the new db folder. make sure that you have the appropriate rights") } } diff --git a/fs/fs.go b/fs/fs.go index 6a74e6f8f..fab636c32 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -2,7 +2,9 @@ package fs import ( + "bufio" "fmt" + "io" "os" "os/user" "path" @@ -10,6 +12,7 @@ import ( const defaultDirectoryPermission = 0740 const rwFilePermission = 0600 +const copyChunkSize = 128 * 1024 // HomeFolder returns the home folder of the current user. func HomeFolder() string { @@ -136,18 +139,51 @@ func FolderExists(folderPath, name string) bool { } // CopyFile copy a file or folder from one path to another -func CopyFile(origFilePath, destFilePath string) error { - input, err := os.ReadFile(origFilePath) - if err != nil { +func CopyFile(origFilePath, destFilePath string) (err error) { + var src, dest *os.File + + if src, err = os.Open(origFilePath); err != nil { return err } - err = os.WriteFile(destFilePath, input, rwFilePermission) - if err != nil { + // close fi on exit and check for its returned error + defer func() { + if closeErr := src.Close(); closeErr != nil && err == nil { + err = closeErr + } + }() + + // make a reader buffer + srcReader := bufio.NewReader(src) + + if dest, err = os.Create(destFilePath); err != nil { return err } + // close fo on exit and check for its returned error + defer func() { + if closeErr := dest.Close(); closeErr != nil && err == nil { + err = closeErr + } + }() - return nil + // make a writer buffer + destWriter := bufio.NewWriter(dest) + + // make a buffer to keep chunks that are read + buf := make([]byte, copyChunkSize) + if _, err := io.CopyBuffer(destWriter, srcReader, buf); err != nil { + return err + } + + if err := destWriter.Flush(); err != nil { + return err + } + + if err := os.Chmod(destFilePath, rwFilePermission); err != nil { + return err + } + + return err } // CopyFolder copy files inside a folder to another folder recursively