[pgmoneta#123] Receive and extract base backup archive (for server ve…

…rsion < 15)

.github/workflows/ci.yml

@@ -29,6 +29,8 @@ jobs:
       run: sudo apt install -y liblz4-dev
     - name: Install libssh
       run: sudo apt install -y libssh-dev
+    - name: Install libarchive
+      run: sudo apt install -y libarchive-dev
     - name: Install libcurl
       run: sudo apt install -y libcurl4-nss-dev
     - name: Install clang

CMakeLists.txt

@@ -98,6 +98,13 @@ else ()
   message(FATAL_ERROR "OpenSSL needed")
 endif()
 
+find_package(LibArchive)
+if (LibArchive_FOUND)
+  message(STATUS "libarchive found")
+else ()
+  message(FATAL_ERROR "libarchive needed")
+endif()
+
 find_package(Rst2man)
 if (RST2MAN_FOUND)
   message(STATUS "rst2man found")

CONTRIBUTING.md

@@ -34,7 +34,7 @@ Don't forget to indicate your pgmoneta version.
 You can use the follow command, if you are using a [Fedora](https://getfedora.org/) based platform:
 
 ```
-dnf install git gcc cmake make libev libev-devel openssl openssl-devel systemd systemd-devel zlib zlib-devel libzstd libzstd-devel lz4 lz4-devel libssh libssh-devel python3-docutils libatomic libcurl libcurl-devel bzip2 bzip2-devel
+dnf install git gcc cmake make libev libev-devel openssl openssl-devel systemd systemd-devel zlib zlib-devel libzstd libzstd-devel lz4 lz4-devel libssh libssh-devel python3-docutils libatomic libcurl libcurl-devel bzip2 bzip2-devel libarchive libarchive-devel
 ```
 
 in order to get the necessary dependencies.

README.md

@@ -58,9 +58,10 @@ See [Architecture](./doc/ARCHITECTURE.md) for the architecture of `pgmoneta`.
 * [rst2man](https://docutils.sourceforge.io/)
 * [libssh](https://www.libssh.org/)
 * [libcurl](https://curl.se/libcurl/)
+* [libarchive](http://www.libarchive.org/)
 
 ```sh
-dnf install git gcc cmake make libev libev-devel openssl openssl-devel systemd systemd-devel zlib zlib-devel libzstd libzstd-devel lz4 lz4-devel libssh libssh-devel libcurl libcurl-devel python3-docutils libatomic bzip2 bzip2-devel
+dnf install git gcc cmake make libev libev-devel openssl openssl-devel systemd systemd-devel zlib zlib-devel libzstd libzstd-devel lz4 lz4-devel libssh libssh-devel libcurl libcurl-devel python3-docutils libatomic bzip2 bzip2-devel libarchive libarchive-devel
 ```
 
 Alternative [clang 8+](https://clang.llvm.org/) can be used.

doc/ARCHITECTURE.md

@@ -12,7 +12,7 @@ Backup is handled in [backup.h](../src/include/backup.h) ([backup.c](../src/libp
 Restore is handled in [restore.h](../src/include/restore.h) ([restore.c](../src/libpgmoneta/restore.c)) with linking
 handled in [link.h](../src/include/link.h) ([link.c](../src/libpgmoneta/link.c)).
 
-Archive is handled in [archive.h](../src/include/archive.h) ([archive.c](../src/libpgmoneta/archive.c)) backed by
+Archive is handled in [achv.h](../src/include/achv.h) ([archive.c](../src/libpgmoneta/archive.c)) backed by
 restore.
 
 Write-Ahead Log is handled in [wal.h](../src/include/wal.h) ([wal.c](../src/libpgmoneta/wal.c)).

src/CMakeLists.txt

@@ -28,6 +28,7 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
     ${SYSTEMD_INCLUDE_DIRS}
     ${LIBSSH_INCLUDE_DIRS}
     ${CURL_INCLUDE_DIRS}
+    ${LibArchive_INCLUDE_DIRS}
   )
 
   #
@@ -45,6 +46,7 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
     ${LIBSSH_LIBRARIES}
     ${CURL_LIBRARIES}
     ${LIBATOMIC_LIBRARY}
+    ${LibArchive_LIBRARY}
   )
 
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
@@ -69,6 +71,7 @@ else()
     ${OPENSSL_INCLUDE_DIR}
     ${LIBSSH_INCLUDE_DIRS}
     ${CURL_INCLUDE_DIRS}
+    ${LibArchive_INCLUDE_DIRS}
   )
 
   #
@@ -84,6 +87,7 @@ else()
     ${OPENSSL_SSL_LIBRARY}
     ${LIBSSH_LIBRARIES}
     ${CURL_LIBRARIES}
+    ${LibArchive_LIBRARIES}
   )
 
   if (${CMAKE_SYSTEM_NAME} STREQUAL "OpenBSD")

src/include/archive.h → src/include/achv.h

@@ -26,8 +26,8 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef PGMONETA_ARCHIVE_H
-#define PGMONETA_ARCHIVE_H
+#ifndef PGMONETA_ACHV_H
+#define PGMONETA_ACHV_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -48,6 +48,15 @@ extern "C" {
 void
 pgmoneta_archive(int client_fd, int server, char* backup_id, char* position, char* directory, char** argv);
 
+/**
+ * Extract from a tar file to a given directory
+ * @param file_path The tar file path
+ * @param destination The destination to extract to
+ * @return 0 upon success, otherwise 1
+ */
+int
+pgmoneta_extract_tar_file(char* file_path, char* destination);
+
 #ifdef __cplusplus
 }
 #endif

src/include/message.h

@@ -34,6 +34,7 @@ extern "C" {
 #endif
 
 #include <pgmoneta.h>
+#include <tablespace.h>
 #include <memory.h>
 
 #include <stdbool.h>
@@ -417,6 +418,30 @@ pgmoneta_consume_copy_stream(int socket, struct stream_buffer* buffer, struct me
 int
 pgmoneta_consume_data_row_messages(int socket, struct stream_buffer* buffer, struct query_response** response);
 
+/**
+ * Receive backup tar files from the copy stream and write to disk
+ * This functionality is for server version < 15,
+ * and will ignore tablespaces other than the base one for now
+ * @param socket The socket
+ * @param buffer The stream buffer
+ * @param basedir The base directory for the backup data
+ * @param tablespaces The user level tablespaces
+ * @return 0 upon success, otherwise 1
+ */
+int
+pgmoneta_receive_archive_files(int socket, struct stream_buffer* buffer, char* basedir, struct tablespace* tablespaces);
+
+/**
+ * Receive mainfest file from the copy stream and write to disk
+ * This functionality is for server version < 15,
+ * @param socket The socket
+ * @param buffer The stream buffer
+ * @param basedir The base directory for the manifest
+ * @return 0 upon success, otherwise 1
+ */
+int
+pgmoneta_receive_manifest_file(int socket, struct stream_buffer* buffer, char* basedir);
+
 #ifdef __cplusplus
 }
 #endif

src/include/tablespace.h

@@ -40,6 +40,7 @@ struct tablespace
 {
    char* name;
    char* path;
+   unsigned int oid; /**< [[maybe_unused]] The oid of the table space */
    struct tablespace* next;
 };
 

src/libpgmoneta/archive.c

@@ -28,7 +28,7 @@
 
 /* pgmoneta */
 #include <pgmoneta.h>
-#include <archive.h>
+#include <achv.h>
 #include <info.h>
 #include <logging.h>
 #include <management.h>
@@ -37,6 +37,10 @@
 #include <restore.h>
 #include <utils.h>
 
+#include <archive.h>
+#include <archive_entry.h>
+#include <stdio.h>
+
 void
 pgmoneta_archive(int client_fd, int server, char* backup_id, char* position, char* directory, char** argv)
 {
@@ -173,3 +177,49 @@ pgmoneta_archive(int client_fd, int server, char* backup_id, char* position, cha
 
    exit(1);
 }
+
+int
+pgmoneta_extract_tar_file(char* file_path, char* destination)
+{
+   struct archive* a;
+   struct archive_entry* entry;
+   a = archive_read_new();
+   archive_read_support_format_tar(a);
+   // open tar file in a suitable buffer size, I'm using 10240 here
+   if (archive_read_open_filename(a, file_path, 10240) != ARCHIVE_OK)
+   {
+      pgmoneta_log_error("Failed to open the tar file for reading");
+      goto error;
+   }
+
+   while (archive_read_next_header(a, &entry) == ARCHIVE_OK)
+   {
+      char dst_file_path[256];
+      memset(dst_file_path, 0, sizeof(dst_file_path));
+      const char* entry_path = archive_entry_pathname(entry);
+      if (pgmoneta_ends_with(destination, "/"))
+      {
+         snprintf(dst_file_path, sizeof(dst_file_path), "%s%s", destination, entry_path);
+      }
+      else
+      {
+         snprintf(dst_file_path, sizeof(dst_file_path), "%s/%s", destination, entry_path);
+      }
+
+      archive_entry_set_pathname(entry, dst_file_path);
+      if (archive_read_extract(a, entry, 0) != ARCHIVE_OK)
+      {
+         pgmoneta_log_error("Failed to extract entry: %s", archive_error_string(a));
+         goto error;
+      }
+   }
+
+   archive_read_close(a);
+   archive_read_free(a);
+   return 0;
+
+error:
+   archive_read_close(a);
+   archive_read_free(a);
+   return 1;
+}