Storage object improvements.

* Add storageCopy(), storageMove(), and storagePathSync().
* Separate StorageFile object into separate read and write objects.
* Abstract out Posix file read/write objects.

build/error.yaml

@@ -80,6 +80,9 @@ memory: 94
 cipher: 95
 param-invalid: 96
 
+# Unable to close a path
+path-close: 97
+
 # This error should not be thrown directly -- it serves as a parent for the C errors
 runtime: 122
 

doc/xml/release.xml

@@ -62,7 +62,7 @@
                     </release-item>
 
                     <release-item>
-                        <p>Storage object improvements.  Convert all functions to variadic functions.  Enforce read-only storage. Add <code>storageLocalWrite()</code> helper function.  Add <code>storageExists()</code>, <code>storagePathCreate()</code>, and <code>storageRemove()</code>.  Add <code>StorageFile</code> object and <code>storageOpenRead()</code>/<code>storageOpenWrite()</code>.  Abstract Posix driver code into a separate module.  Add <code>storagePathRemove()</code> and use it in the Perl Posix driver.</p>
+                        <p>Storage object improvements.  Convert all functions to variadic functions.  Enforce read-only storage. Add <code>storageLocalWrite()</code> helper function.  Add <code>storageCopy()</code>, <code>storageExists()</code>, <code>storageMove()</code>, <code>storageNewRead()</code>/<code>storageNewWrite()</code>, <code>storagePathCreate()</code>, <code>storagePathRemove()</code>, <code>storagePathSync()</code>, and <code>storageRemove()</code>.  Add <code>StorageFileRead</code> and <code>StorageFileWrite</code> objects.  Abstract Posix driver code into a separate module. Call <code>storagePathCreate()</code> from the Perl Posix driver.</p>
                     </release-item>
 
                     <release-item>

lib/pgBackRest/Common/ExceptionAuto.pm

@@ -161,6 +161,8 @@ use constant ERROR_CIPHER                                           => 95;
 push @EXPORT, qw(ERROR_CIPHER);
 use constant ERROR_PARAM_INVALID                                    => 96;
 push @EXPORT, qw(ERROR_PARAM_INVALID);
+use constant ERROR_PATH_CLOSE                                       => 97;
+push @EXPORT, qw(ERROR_PATH_CLOSE);
 use constant ERROR_RUNTIME                                          => 122;
 push @EXPORT, qw(ERROR_RUNTIME);
 use constant ERROR_INVALID                                          => 123;

libc/LibC.xs

@@ -50,7 +50,7 @@ These includes are from the src directory.  There is no Perl-specific code in th
 #include "config/parse.h"
 #include "perl/config.h"
 #include "postgres/pageChecksum.h"
-#include "storage/driver/posix.h"
+#include "storage/driver/posix/driver.h"
 
 /***********************************************************************************************************************************
 Helper macros

libc/Makefile.PL

@@ -102,8 +102,12 @@ my @stryCFile =
     'config/parse.c',
     'perl/config.c',
     'postgres/pageChecksum.c',
-    'storage/driver/posix.c',
-    'storage/file.c',
+    'storage/driver/posix/driver.c',
+    'storage/driver/posix/driverFile.c',
+    'storage/driver/posix/driverRead.c',
+    'storage/driver/posix/driverWrite.c',
+    'storage/fileRead.c',
+    'storage/fileWrite.c',
     'storage/helper.c',
     'storage/storage.c',
 );

src/Makefile

@@ -81,8 +81,12 @@ SRCS = \
 	config/parse.c \
 	perl/config.c \
 	perl/exec.c \
-	storage/driver/posix.c \
-	storage/file.c \
+	storage/driver/posix/driver.c \
+	storage/driver/posix/driverFile.c \
+	storage/driver/posix/driverRead.c \
+	storage/driver/posix/driverWrite.c \
+	storage/fileRead.c \
+	storage/fileWrite.c \
 	storage/helper.c \
 	storage/storage.c \
 	main.c

src/command/archive/push/push.c

@@ -49,7 +49,7 @@ walStatus(const String *walSegment, bool confessOnError)
             const String *statusFile = strLstGet(fileList, 0);
 
             String *content = strNewBuf(
-                storageGetNP(storageOpenReadNP(storageSpool(), strNewFmt("%s/%s", STORAGE_SPOOL_ARCHIVE_OUT, strPtr(statusFile)))));
+                storageGetNP(storageNewReadNP(storageSpool(), strNewFmt("%s/%s", STORAGE_SPOOL_ARCHIVE_OUT, strPtr(statusFile)))));
 
             // Get the code and message if the file has content
             int code = 0;

src/common/error.auto.c

@@ -78,6 +78,7 @@ ERROR_DEFINE( 93, FileExistsError, RuntimeError);
 ERROR_DEFINE( 94, MemoryError, RuntimeError);
 ERROR_DEFINE( 95, CipherError, RuntimeError);
 ERROR_DEFINE( 96, ParamInvalidError, RuntimeError);
+ERROR_DEFINE( 97, PathCloseError, RuntimeError);
 ERROR_DEFINE(122, RuntimeError, RuntimeError);
 ERROR_DEFINE(123, InvalidError, RuntimeError);
 ERROR_DEFINE(124, UnhandledError, RuntimeError);
@@ -159,6 +160,7 @@ static const ErrorType *errorTypeList[] =
     &MemoryError,
     &CipherError,
     &ParamInvalidError,
+    &PathCloseError,
     &RuntimeError,
     &InvalidError,
     &UnhandledError,

src/common/error.auto.h

@@ -80,6 +80,7 @@ ERROR_DECLARE(FileExistsError);
 ERROR_DECLARE(MemoryError);
 ERROR_DECLARE(CipherError);
 ERROR_DECLARE(ParamInvalidError);
+ERROR_DECLARE(PathCloseError);
 ERROR_DECLARE(RuntimeError);
 ERROR_DECLARE(InvalidError);
 ERROR_DECLARE(UnhandledError);

src/common/ini.c

@@ -212,7 +212,7 @@ iniLoad(Ini *this, const String *fileName)
 
         MEM_CONTEXT_TEMP_BEGIN()
         {
-            iniParse(this, strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), this->fileName))));
+            iniParse(this, strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), this->fileName))));
         }
         MEM_CONTEXT_TEMP_END();
     }

src/config/parse.c

@@ -164,15 +164,15 @@ cfgFileLoad(                                                        // NOTE: Pas
             configFileName = optConfigDefault;
 
         // Load the config file
-        Buffer *buffer = storageGetNP(storageOpenReadP(storageLocal(), configFileName, .ignoreMissing = !configRequired));
+        Buffer *buffer = storageGetNP(storageNewReadP(storageLocal(), configFileName, .ignoreMissing = !configRequired));
 
         // Convert the contents of the file buffer to the config string object
         if (buffer != NULL)
             result = strNewBuf(buffer);
         else if (strEq(configFileName, optConfigDefaultCurrent))
         {
             // If confg is current default and it was not found, attempt to load the config file from the old default location
-            buffer = storageGetNP(storageOpenReadP(storageLocal(), origConfigDefault, .ignoreMissing = !configRequired));
+            buffer = storageGetNP(storageNewReadP(storageLocal(), origConfigDefault, .ignoreMissing = !configRequired));
 
             if (buffer != NULL)
                 result = strNewBuf(buffer);
@@ -210,7 +210,7 @@ cfgFileLoad(                                                        // NOTE: Pas
             for (unsigned int listIdx = 0; listIdx < strLstSize(list); listIdx++)
             {
                 Buffer *fileBuffer = storageGetNP(
-                    storageOpenReadP(
+                    storageNewReadP(
                         storageLocal(), strNewFmt("%s/%s", strPtr(configIncludePath), strPtr(strLstGet(list, listIdx))),
                         .ignoreMissing = true));
 

src/storage/driver/posix.c → src/storage/driver/posix/driver.c

@@ -1,29 +1,19 @@
 /***********************************************************************************************************************************
-Storage Posix Driver
+Storage Driver Posix
 ***********************************************************************************************************************************/
 #include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <stdio.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #include "common/memContext.h"
 #include "common/regExp.h"
-#include "storage/driver/posix.h"
+#include "storage/driver/posix/driver.h"
+#include "storage/driver/posix/driverFile.h"
 #include "storage/storage.h"
 
-/***********************************************************************************************************************************
-Storage file data - holds the file handle.
-***********************************************************************************************************************************/
-typedef struct StorageFileDataPosix
-{
-    MemContext *memContext;
-    int handle;
-} StorageFileDataPosix;
-
-#define STORAGE_DATA(file)                                                                                                         \
-    ((StorageFileDataPosix *)storageFileData(file))
-
 /***********************************************************************************************************************************
 Does a file/path exist?
 ***********************************************************************************************************************************/
@@ -48,61 +38,6 @@ storageDriverPosixExists(const String *path)
     return result;
 }
 
-/***********************************************************************************************************************************
-Read from storage into a buffer
-***********************************************************************************************************************************/
-Buffer *
-storageDriverPosixGet(const StorageFile *file)
-{
-    Buffer *result = NULL;
-
-    MEM_CONTEXT_TEMP_BEGIN()
-    {
-        TRY_BEGIN()
-        {
-            size_t bufferSize = storageBufferSize(storageFileStorage(file));
-            result = bufNew(bufferSize);
-
-            // Create result buffer with buffer size
-            ssize_t actualBytes = 0;
-            size_t totalBytes = 0;
-
-            do
-            {
-                // Grow the buffer on subsequent reads
-                if (totalBytes != 0)
-                    bufResize(result, bufSize(result) + bufferSize);
-
-                // Read and handle errors
-                actualBytes = read(
-                    STORAGE_DATA(file)->handle, bufPtr(result) + totalBytes, bufferSize);
-
-                // Error occurred during write
-                if (actualBytes == -1)
-                    THROW_SYS_ERROR(FileReadError, "unable to read '%s'", strPtr(storageFileName(file)));
-
-                // Track total bytes read
-                totalBytes += (size_t)actualBytes;
-            }
-            while (actualBytes != 0);
-
-            // Resize buffer to total bytes read
-            bufResize(result, totalBytes);
-        }
-        FINALLY()
-        {
-            close(STORAGE_DATA(file)->handle);
-            storageFileFree(file);
-        }
-        TRY_END();
-
-        bufMove(result, MEM_CONTEXT_OLD());
-    }
-    MEM_CONTEXT_TEMP_END();
-
-    return result;
-}
-
 /***********************************************************************************************************************************
 Get a list of files from a directory
 ***********************************************************************************************************************************/
@@ -166,59 +101,59 @@ storageDriverPosixList(const String *path, bool errorOnMissing, const String *ex
 }
 
 /***********************************************************************************************************************************
-Open a file for reading
+Move a file
 ***********************************************************************************************************************************/
-void *
-storageDriverPosixOpenRead(const String *file, bool ignoreMissing)
+bool
+storageDriverPosixMove(StorageFileReadPosix *source, StorageFileWritePosix *destination)
 {
-    StorageFileDataPosix *result = NULL;
-
-    // Open the file and handle errors
-    int fileHandle = open(strPtr(file), O_RDONLY, 0);
+    bool result = true;
 
-    if (fileHandle == -1)
-    {
-        // Error unless ignore missing is specified
-        if (!ignoreMissing || errno != ENOENT)
-            THROW_SYS_ERROR(FileOpenError, "unable to open '%s' for read", strPtr(file));
-    }
-    // Else create the storage file and data
-    else
+    MEM_CONTEXT_TEMP_BEGIN()
     {
-        MEM_CONTEXT_NEW_BEGIN("StorageFileDataPosix")
-        {
-            result = memNew(sizeof(StorageFileDataPosix));
-            result->memContext = MEM_CONTEXT_NEW();
-            result->handle = fileHandle;
-        }
-        MEM_CONTEXT_NEW_END();
-    }
+        const String *sourceFile = storageFileReadPosixName(source);
+        const String *destinationFile = storageFileWritePosixName(destination);
+        const String *destinationPath = storageFileWritePosixPath(destination);
 
-    return result;
-}
-
-/***********************************************************************************************************************************
-Open a file for writing
-***********************************************************************************************************************************/
-void *
-storageDriverPosixOpenWrite(const String *file, mode_t mode)
-{
-    // Open the file and handle errors
-    int fileHandle = open(strPtr(file), O_CREAT | O_TRUNC | O_WRONLY, mode);
+        // Attempt to move the file
+        if (rename(strPtr(sourceFile), strPtr(destinationFile)) == -1)
+        {
+            // Detemine which file/path is missing
+            if (errno == ENOENT)
+            {
+                if (!storageDriverPosixExists(sourceFile))
+                    THROW_SYS_ERROR(FileMissingError, "unable to move missing file '%s'", strPtr(sourceFile));
 
-    if (fileHandle == -1)
-        THROW_SYS_ERROR(FileOpenError, "unable to open '%s' for write", strPtr(file));
+                if (!storageFileWritePosixCreatePath(destination))
+                {
+                    THROW_SYS_ERROR(
+                        PathMissingError, "unable to move '%s' to missing path '%s'", strPtr(sourceFile), strPtr(destinationPath));
+                }
 
-    // Create the storage file and data
-    StorageFileDataPosix *result = NULL;
+                storageDriverPosixPathCreate(destinationPath, false, false, storageFileWritePosixModePath(destination));
+                result = storageDriverPosixMove(source, destination);
+            }
+            // Else the destination is on a different device so a copy will be needed
+            else if (errno == EXDEV)
+            {
+                result = false;
+            }
+            else
+                THROW_SYS_ERROR(FileMoveError, "unable to move '%s' to '%s'", strPtr(sourceFile), strPtr(destinationFile));
+        }
+        // Sync paths on success
+        else
+        {
+            // Sync source path if the destination path was synced and the paths are not equal
+            if (storageFileWritePosixSyncPath(destination))
+            {
+                String *sourcePath = strPath(sourceFile);
 
-    MEM_CONTEXT_NEW_BEGIN("StorageFileDataPosix")
-    {
-        result = memNew(sizeof(StorageFileDataPosix));
-        result->memContext = MEM_CONTEXT_NEW();
-        result->handle = fileHandle;
+                if (!strEq(destinationPath, sourcePath))
+                    storageDriverPosixPathSync(sourcePath, false);
+            }
+        }
     }
-    MEM_CONTEXT_NEW_END();
+    MEM_CONTEXT_TEMP_END();
 
     return result;
 }
@@ -291,22 +226,23 @@ storageDriverPosixPathRemove(const String *path, bool errorOnMissing, bool recur
 }
 
 /***********************************************************************************************************************************
-Write a buffer to storage
+Sync a path
 ***********************************************************************************************************************************/
 void
-storageDriverPosixPut(const StorageFile *file, const Buffer *buffer)
+storageDriverPosixPathSync(const String *path, bool ignoreMissing)
 {
-    TRY_BEGIN()
-    {
-        if (write(STORAGE_DATA(file)->handle, bufPtr(buffer), bufSize(buffer)) != (ssize_t)bufSize(buffer))
-            THROW_SYS_ERROR(FileWriteError, "unable to write '%s'", strPtr(storageFileName(file)));
-    }
-    FINALLY()
+    // Open directory and handle errors
+    int handle = storageFilePosixOpen(path, O_RDONLY, 0, ignoreMissing, &PathOpenError, "sync");
+
+    // On success
+    if (handle != -1)
     {
-        close(STORAGE_DATA(file)->handle);
-        storageFileFree(file);
+        // Attempt to sync the directory
+        storageFilePosixSync(handle, path, &PathSyncError, true);
+
+        // Close the directory
+        storageFilePosixClose(handle, path, &PathCloseError);
     }
-    TRY_END();
 }
 
 /***********************************************************************************************************************************