Fixes applied to migration process (#862)

* support to copy large files (use CopyBuffer to copy the file from src to dest)
* rollback migration process if an error is found while executing it

core/migration/migration.go

@@ -2,6 +2,7 @@ package migration
 
 import (
 	"fmt"
+	"os"
 	"path"
 
 	"github.com/drand/drand/common"
@@ -29,8 +30,30 @@ func CheckSBFolderStructure(baseFolder string) bool {
 
 // MigrateSBFolderStructure will migrate the file store structure from single-beacon drand version
 // to the new structure created to support multi-beacon feature. This should be called on any function
-// which reads file store from disk, so we are sure the structure is the correct one.
+// which reads file store from disk, so we are sure the structure is the correct one. It will check if
+// the process ends successfully or not. If something went wrong, a rollback process will be executed.
 func MigrateSBFolderStructure(baseFolder string) error {
+	if err := runSBMigration(baseFolder); err != nil {
+		multiBeaconFolderPath := path.Join(baseFolder, common.MultiBeaconFolder)
+		isMigrationDone := fs.FolderExists(baseFolder, multiBeaconFolderPath)
+
+		if isMigrationDone {
+			if localErr := os.RemoveAll(multiBeaconFolderPath); localErr != nil {
+				return fmt.Errorf("we could not rollback the migration, please remove %s before run daemon again. Err: %s",
+					multiBeaconFolderPath, err.Error())
+			}
+		}
+
+		return err
+	}
+
+	return nil
+}
+
+// runSBMigration will migrate the file store structure from single-beacon drand version
+// to the new structure created to support multi-beacon feature. It has the logic of the
+// migration process.
+func runSBMigration(baseFolder string) error {
 	groupFolderPath := path.Join(baseFolder, key.GroupFolderName)
 	keyFolderPath := path.Join(baseFolder, key.KeyFolderName)
 	dbFolderPath := path.Join(baseFolder, core.DefaultDBFolder)
@@ -46,21 +69,21 @@ func MigrateSBFolderStructure(baseFolder string) error {
 	}
 
 	if fs.CreateSecureFolder(path.Join(baseFolder, common.MultiBeaconFolder)) == "" {
-		return fmt.Errorf("something went wrong with the multi beacon folder. Make sure that you have the appropriate rights")
+		return fmt.Errorf("something went wrong with the multi beacon folder, make sure that you have the appropriate rights")
 	}
 
 	// Create new folders to move actual files found. If one of them exists, we will be sure the all new structures have been created
 	if isGroupFound || isKeyFound || isDBFound {
 		if fs.CreateSecureFolder(path.Join(baseFolder, common.MultiBeaconFolder, common.DefaultBeaconID, key.GroupFolderName)) == "" {
-			return fmt.Errorf("something went wrong with the group folder. Make sure that you have the appropriate rights")
+			return fmt.Errorf("something went wrong with the group folder, make sure that you have the appropriate rights")
 		}
 
 		if fs.CreateSecureFolder(path.Join(baseFolder, common.MultiBeaconFolder, common.DefaultBeaconID, key.KeyFolderName)) == "" {
-			return fmt.Errorf("something went wrong with the key folder. Make sure that you have the appropriate rights")
+			return fmt.Errorf("something went wrong with the key folder, make sure that you have the appropriate rights")
 		}
 
 		if fs.CreateSecureFolder(path.Join(baseFolder, common.MultiBeaconFolder, common.DefaultBeaconID, core.DefaultDBFolder)) == "" {
-			return fmt.Errorf("something went wrong with the db folder. Make sure that you have the appropriate rights")
+			return fmt.Errorf("something went wrong with the db folder, make sure that you have the appropriate rights")
 		}
 	}
 
@@ -70,7 +93,7 @@ func MigrateSBFolderStructure(baseFolder string) error {
 
 		// Copy files to new destinations (only if the folder is found)
 		if err := fs.CopyFolder(oldPath, newPath); err != nil {
-			return fmt.Errorf("something went wrong with the new group folder. Make sure that you have the appropriate rights")
+			return fmt.Errorf("something went wrong with the new group folder. make sure that you have the appropriate rights")
 		}
 	}
 
@@ -80,7 +103,7 @@ func MigrateSBFolderStructure(baseFolder string) error {
 
 		// Copy files to new destinations (only if the folder is found)
 		if err := fs.CopyFolder(oldPath, newPath); err != nil {
-			return fmt.Errorf("something went wrong with the new key folder. Make sure that you have the appropriate rights")
+			return fmt.Errorf("something went wrong with the new key folder. make sure that you have the appropriate rights")
 		}
 	}
 
@@ -90,7 +113,7 @@ func MigrateSBFolderStructure(baseFolder string) error {
 
 		// Copy files to new destinations (only if the folder is found)
 		if err := fs.CopyFolder(oldPath, newPath); err != nil {
-			return fmt.Errorf("something went wrong with the new db folder. Make sure that you have the appropriate rights")
+			return fmt.Errorf("something went wrong with the new db folder. make sure that you have the appropriate rights")
 		}
 	}
 

fs/fs.go

@@ -2,14 +2,17 @@
 package fs
 
 import (
+	"bufio"
 	"fmt"
+	"io"
 	"os"
 	"os/user"
 	"path"
 )
 
 const defaultDirectoryPermission = 0740
 const rwFilePermission = 0600
+const copyChunkSize = 128 * 1024
 
 // HomeFolder returns the home folder of the current user.
 func HomeFolder() string {
@@ -136,18 +139,51 @@ func FolderExists(folderPath, name string) bool {
 }
 
 // CopyFile copy a file or folder from one path to another
-func CopyFile(origFilePath, destFilePath string) error {
-	input, err := os.ReadFile(origFilePath)
-	if err != nil {
+func CopyFile(origFilePath, destFilePath string) (err error) {
+	var src, dest *os.File
+
+	if src, err = os.Open(origFilePath); err != nil {
 		return err
 	}
 
-	err = os.WriteFile(destFilePath, input, rwFilePermission)
-	if err != nil {
+	// close fi on exit and check for its returned error
+	defer func() {
+		if closeErr := src.Close(); closeErr != nil && err == nil {
+			err = closeErr
+		}
+	}()
+
+	// make a reader buffer
+	srcReader := bufio.NewReader(src)
+
+	if dest, err = os.Create(destFilePath); err != nil {
 		return err
 	}
+	// close fo on exit and check for its returned error
+	defer func() {
+		if closeErr := dest.Close(); closeErr != nil && err == nil {
+			err = closeErr
+		}
+	}()
 
-	return nil
+	// make a writer buffer
+	destWriter := bufio.NewWriter(dest)
+
+	// make a buffer to keep chunks that are read
+	buf := make([]byte, copyChunkSize)
+	if _, err := io.CopyBuffer(destWriter, srcReader, buf); err != nil {
+		return err
+	}
+
+	if err := destWriter.Flush(); err != nil {
+		return err
+	}
+
+	if err := os.Chmod(destFilePath, rwFilePermission); err != nil {
+		return err
+	}
+
+	return err
 }
 
 // CopyFolder copy files inside a folder to another folder recursively