[#291] Add function for CRC32C checksum

Signed-off-by: Shikhar Soni <shikharish05@gmail.com>
pgmoneta · May 19, 2024 · 7bcdd0e · 7bcdd0e
1 parent a75b1ad
commit 7bcdd0e
Show file tree

Hide file tree

Showing 4 changed files with 207 additions and 81 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -59,6 +59,35 @@ if(NOT COMPILER_SUPPORTS_C17)
   message(FATAL_ERROR "The compiler ${CMAKE_C_COMPILER} has no C17 support. Please use a different C compiler.")
 endif()
 
+CHECK_C_COMPILER_FLAG("-msse4.2" COMPILER_SUPPORTS_SSE42)
+if(NOT COMPILER_SUPPORTS_SSE42)
+  message(NOTICE "The compiler ${CMAKE_C_COMPILER} has no SSE4.2 support.")
+endif()
+
+if(COMPILER_SUPPORTS_SSE42)
+  set(CMAKE_C_FLAGS "-msse4.2")
+  check_c_source_compiles("
+  #if defined(_MSC_VER)
+  #include <intrin.h>
+  #else  // !defined(_MSC_VER)
+  #include <cpuid.h>
+  #include <x86intrin.h>
+  #endif  // defined(_MSC_VER)
+  
+  int main() {
+    _mm_crc32_u8(0, 0); _mm_crc32_u32(0, 0);
+  #if defined(_M_X64) || defined(__x86_64__)
+     _mm_crc32_u64(0, 0);
+  #endif // defined(_M_X64) || defined(__x86_64__)
+    return 0;
+  }
+  "  HAVE_SSE42)
+endif()
+
+if(NOT HAVE_SSE42)
+  unset(CMAKE_C_FLAGS)
+endif()
+
 if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
     # Homebrew ships libarchive keg only, include dirs have to be set manually
     execute_process(

diff --git a/src/include/security.h b/src/include/security.h
@@ -148,6 +148,22 @@ int pgmoneta_generate_string_hmac_sha256_hash(char* key, int key_length, char* v
                                               int value_length, unsigned char** hmac,
                                               int* hmac_length);
 
+/**
+ * Generate CRC32C for a buffer
+ * @param buffer The buffer
+ * @param size The size of the buffer
+ * @param crc_buff The hash value
+ * @return 0 upon success, otherwise 1
+ */
+int pgmoneta_create_crc32c_buffer(void* buffer, size_t size, uint32_t* crc_buf);
+
+/**
+ * @param path The file path.
+ * @param crc The hash value.
+ * @return 0 upon success, otherwise 1.
+ */
+int pgmoneta_create_crc32c_file(char* path, char** crc);
+
 /**
  * Close a SSL structure
  * @param ssl The SSL structure

diff --git a/src/libpgmoneta/hashmap.c b/src/libpgmoneta/hashmap.c
@@ -29,6 +29,7 @@
 /* Based on https://github.com/sheredom/hashmap.h */
 
 #include <hashmap.h>
+#include <security.h>
 
 #include <stdbool.h>
 #include <stdlib.h>
@@ -47,7 +48,6 @@
 #define HASHMAP_MAX_CHAIN_LENGTH 8
 
 static int hashmap_iterate(struct hashmap* hashmap, int (* f)(void*, struct hashmap_element*), void* context);
-static unsigned int hashmap_crc32_helper(char* s, unsigned int len);
 static unsigned int hashmap_hash_helper_int_helper(struct hashmap* m, char* keystring, unsigned int len);
 static int hashmap_match_helper(struct hashmap_element* element, char* key, unsigned int len);
 static int hashmap_hash_helper(struct hashmap* m, char* key, unsigned int len, unsigned* out_index);
@@ -335,89 +335,11 @@ hashmap_iterate(struct hashmap* hashmap,
    return 0;
 }
 
-static unsigned int
-hashmap_crc32_helper(char* s, unsigned int len)
-{
-   unsigned int crc32val = 0;
-
-#if defined(HASHMAP_SSE42)
-   for (unsigned int i = 0; i < len; i++)
-   {
-      crc32val = _mm_crc32_u8(crc32val, HASHMAP_CAST(unsigned char, s[i]));
-   }
-
-   return crc32val;
-#else
-   // Using polynomial 0x11EDC6F41 to match SSE 4.2's crc function.
-   static const unsigned int crc32_tab[] = {
-      0x00000000U, 0xF26B8303U, 0xE13B70F7U, 0x1350F3F4U, 0xC79A971FU,
-      0x35F1141CU, 0x26A1E7E8U, 0xD4CA64EBU, 0x8AD958CFU, 0x78B2DBCCU,
-      0x6BE22838U, 0x9989AB3BU, 0x4D43CFD0U, 0xBF284CD3U, 0xAC78BF27U,
-      0x5E133C24U, 0x105EC76FU, 0xE235446CU, 0xF165B798U, 0x030E349BU,
-      0xD7C45070U, 0x25AFD373U, 0x36FF2087U, 0xC494A384U, 0x9A879FA0U,
-      0x68EC1CA3U, 0x7BBCEF57U, 0x89D76C54U, 0x5D1D08BFU, 0xAF768BBCU,
-      0xBC267848U, 0x4E4DFB4BU, 0x20BD8EDEU, 0xD2D60DDDU, 0xC186FE29U,
-      0x33ED7D2AU, 0xE72719C1U, 0x154C9AC2U, 0x061C6936U, 0xF477EA35U,
-      0xAA64D611U, 0x580F5512U, 0x4B5FA6E6U, 0xB93425E5U, 0x6DFE410EU,
-      0x9F95C20DU, 0x8CC531F9U, 0x7EAEB2FAU, 0x30E349B1U, 0xC288CAB2U,
-      0xD1D83946U, 0x23B3BA45U, 0xF779DEAEU, 0x05125DADU, 0x1642AE59U,
-      0xE4292D5AU, 0xBA3A117EU, 0x4851927DU, 0x5B016189U, 0xA96AE28AU,
-      0x7DA08661U, 0x8FCB0562U, 0x9C9BF696U, 0x6EF07595U, 0x417B1DBCU,
-      0xB3109EBFU, 0xA0406D4BU, 0x522BEE48U, 0x86E18AA3U, 0x748A09A0U,
-      0x67DAFA54U, 0x95B17957U, 0xCBA24573U, 0x39C9C670U, 0x2A993584U,
-      0xD8F2B687U, 0x0C38D26CU, 0xFE53516FU, 0xED03A29BU, 0x1F682198U,
-      0x5125DAD3U, 0xA34E59D0U, 0xB01EAA24U, 0x42752927U, 0x96BF4DCCU,
-      0x64D4CECFU, 0x77843D3BU, 0x85EFBE38U, 0xDBFC821CU, 0x2997011FU,
-      0x3AC7F2EBU, 0xC8AC71E8U, 0x1C661503U, 0xEE0D9600U, 0xFD5D65F4U,
-      0x0F36E6F7U, 0x61C69362U, 0x93AD1061U, 0x80FDE395U, 0x72966096U,
-      0xA65C047DU, 0x5437877EU, 0x4767748AU, 0xB50CF789U, 0xEB1FCBADU,
-      0x197448AEU, 0x0A24BB5AU, 0xF84F3859U, 0x2C855CB2U, 0xDEEEDFB1U,
-      0xCDBE2C45U, 0x3FD5AF46U, 0x7198540DU, 0x83F3D70EU, 0x90A324FAU,
-      0x62C8A7F9U, 0xB602C312U, 0x44694011U, 0x5739B3E5U, 0xA55230E6U,
-      0xFB410CC2U, 0x092A8FC1U, 0x1A7A7C35U, 0xE811FF36U, 0x3CDB9BDDU,
-      0xCEB018DEU, 0xDDE0EB2AU, 0x2F8B6829U, 0x82F63B78U, 0x709DB87BU,
-      0x63CD4B8FU, 0x91A6C88CU, 0x456CAC67U, 0xB7072F64U, 0xA457DC90U,
-      0x563C5F93U, 0x082F63B7U, 0xFA44E0B4U, 0xE9141340U, 0x1B7F9043U,
-      0xCFB5F4A8U, 0x3DDE77ABU, 0x2E8E845FU, 0xDCE5075CU, 0x92A8FC17U,
-      0x60C37F14U, 0x73938CE0U, 0x81F80FE3U, 0x55326B08U, 0xA759E80BU,
-      0xB4091BFFU, 0x466298FCU, 0x1871A4D8U, 0xEA1A27DBU, 0xF94AD42FU,
-      0x0B21572CU, 0xDFEB33C7U, 0x2D80B0C4U, 0x3ED04330U, 0xCCBBC033U,
-      0xA24BB5A6U, 0x502036A5U, 0x4370C551U, 0xB11B4652U, 0x65D122B9U,
-      0x97BAA1BAU, 0x84EA524EU, 0x7681D14DU, 0x2892ED69U, 0xDAF96E6AU,
-      0xC9A99D9EU, 0x3BC21E9DU, 0xEF087A76U, 0x1D63F975U, 0x0E330A81U,
-      0xFC588982U, 0xB21572C9U, 0x407EF1CAU, 0x532E023EU, 0xA145813DU,
-      0x758FE5D6U, 0x87E466D5U, 0x94B49521U, 0x66DF1622U, 0x38CC2A06U,
-      0xCAA7A905U, 0xD9F75AF1U, 0x2B9CD9F2U, 0xFF56BD19U, 0x0D3D3E1AU,
-      0x1E6DCDEEU, 0xEC064EEDU, 0xC38D26C4U, 0x31E6A5C7U, 0x22B65633U,
-      0xD0DDD530U, 0x0417B1DBU, 0xF67C32D8U, 0xE52CC12CU, 0x1747422FU,
-      0x49547E0BU, 0xBB3FFD08U, 0xA86F0EFCU, 0x5A048DFFU, 0x8ECEE914U,
-      0x7CA56A17U, 0x6FF599E3U, 0x9D9E1AE0U, 0xD3D3E1ABU, 0x21B862A8U,
-      0x32E8915CU, 0xC083125FU, 0x144976B4U, 0xE622F5B7U, 0xF5720643U,
-      0x07198540U, 0x590AB964U, 0xAB613A67U, 0xB831C993U, 0x4A5A4A90U,
-      0x9E902E7BU, 0x6CFBAD78U, 0x7FAB5E8CU, 0x8DC0DD8FU, 0xE330A81AU,
-      0x115B2B19U, 0x020BD8EDU, 0xF0605BEEU, 0x24AA3F05U, 0xD6C1BC06U,
-      0xC5914FF2U, 0x37FACCF1U, 0x69E9F0D5U, 0x9B8273D6U, 0x88D28022U,
-      0x7AB90321U, 0xAE7367CAU, 0x5C18E4C9U, 0x4F48173DU, 0xBD23943EU,
-      0xF36E6F75U, 0x0105EC76U, 0x12551F82U, 0xE03E9C81U, 0x34F4F86AU,
-      0xC69F7B69U, 0xD5CF889DU, 0x27A40B9EU, 0x79B737BAU, 0x8BDCB4B9U,
-      0x988C474DU, 0x6AE7C44EU, 0xBE2DA0A5U, 0x4C4623A6U, 0x5F16D052U,
-      0xAD7D5351U
-   };
-
-   for (unsigned int i = 0; i < len; i++)
-   {
-      crc32val = crc32_tab[(HASHMAP_CAST(unsigned char, crc32val) ^
-                            HASHMAP_CAST(unsigned char, s[i]))] ^
-                 (crc32val >> 8);
-   }
-   return crc32val;
-#endif
-}
-
 static unsigned int
 hashmap_hash_helper_int_helper(struct hashmap* m, char* keystring, unsigned int len)
 {
-   unsigned int key = hashmap_crc32_helper(keystring, len);
+   unsigned int key;
+   pgmoneta_create_crc32c_buffer(keystring, len, &key);
 
    /* Robert Jenkins' 32 bit Mix Function */
    key += (key << 12);

diff --git a/src/libpgmoneta/security.c b/src/libpgmoneta/security.c
@@ -57,6 +57,10 @@
 #include <sys/socket.h>
 #include <sys/types.h>
 
+#ifdef HAVE_SSE42
+#include <immintrin.h>
+#endif
+
 #define SECURITY_INVALID  -2
 #define SECURITY_REJECT   -1
 #define SECURITY_TRUST     0
@@ -3060,6 +3064,161 @@ pgmoneta_generate_string_hmac_sha256_hash(char* key, int key_length, char* value
    return 1;
 }
 
+int
+pgmoneta_create_crc32c_buffer(void* buffer, size_t size, uint32_t* crc)
+{
+   if (buffer == NULL)
+   {
+      return 1;
+   }
+
+   #ifdef HAVE_SSE42
+
+   uint64_t crc_long = (uint64_t) ~(*crc);
+
+   for (int i = 0; i < size / 8; i++)
+   {
+      crc_long = _mm_crc32_u64(crc_long, *((uint64_t*)buffer));
+      buffer += 8;
+   }
+   for (int i = 0; i < (size % 8); i++)
+   {
+      crc_long = (uint64_t)_mm_crc32_u8((uint32_t)crc_long, *((unsigned char*)buffer));
+      buffer++;
+   }
+
+   crc_long = ~crc_long;
+
+   *crc = (uint32_t)crc_long;
+   return 0;
+
+   #else
+   uint32_t crc_int = ~(*crc);
+
+   static const uint32_t crc32_tab[256] = {
+      0x00000000L, 0xF26B8303L, 0xE13B70F7L, 0x1350F3F4L,
+      0xC79A971FL, 0x35F1141CL, 0x26A1E7E8L, 0xD4CA64EBL,
+      0x8AD958CFL, 0x78B2DBCCL, 0x6BE22838L, 0x9989AB3BL,
+      0x4D43CFD0L, 0xBF284CD3L, 0xAC78BF27L, 0x5E133C24L,
+      0x105EC76FL, 0xE235446CL, 0xF165B798L, 0x030E349BL,
+      0xD7C45070L, 0x25AFD373L, 0x36FF2087L, 0xC494A384L,
+      0x9A879FA0L, 0x68EC1CA3L, 0x7BBCEF57L, 0x89D76C54L,
+      0x5D1D08BFL, 0xAF768BBCL, 0xBC267848L, 0x4E4DFB4BL,
+      0x20BD8EDEL, 0xD2D60DDDL, 0xC186FE29L, 0x33ED7D2AL,
+      0xE72719C1L, 0x154C9AC2L, 0x061C6936L, 0xF477EA35L,
+      0xAA64D611L, 0x580F5512L, 0x4B5FA6E6L, 0xB93425E5L,
+      0x6DFE410EL, 0x9F95C20DL, 0x8CC531F9L, 0x7EAEB2FAL,
+      0x30E349B1L, 0xC288CAB2L, 0xD1D83946L, 0x23B3BA45L,
+      0xF779DEAEL, 0x05125DADL, 0x1642AE59L, 0xE4292D5AL,
+      0xBA3A117EL, 0x4851927DL, 0x5B016189L, 0xA96AE28AL,
+      0x7DA08661L, 0x8FCB0562L, 0x9C9BF696L, 0x6EF07595L,
+      0x417B1DBCL, 0xB3109EBFL, 0xA0406D4BL, 0x522BEE48L,
+      0x86E18AA3L, 0x748A09A0L, 0x67DAFA54L, 0x95B17957L,
+      0xCBA24573L, 0x39C9C670L, 0x2A993584L, 0xD8F2B687L,
+      0x0C38D26CL, 0xFE53516FL, 0xED03A29BL, 0x1F682198L,
+      0x5125DAD3L, 0xA34E59D0L, 0xB01EAA24L, 0x42752927L,
+      0x96BF4DCCL, 0x64D4CECFL, 0x77843D3BL, 0x85EFBE38L,
+      0xDBFC821CL, 0x2997011FL, 0x3AC7F2EBL, 0xC8AC71E8L,
+      0x1C661503L, 0xEE0D9600L, 0xFD5D65F4L, 0x0F36E6F7L,
+      0x61C69362L, 0x93AD1061L, 0x80FDE395L, 0x72966096L,
+      0xA65C047DL, 0x5437877EL, 0x4767748AL, 0xB50CF789L,
+      0xEB1FCBADL, 0x197448AEL, 0x0A24BB5AL, 0xF84F3859L,
+      0x2C855CB2L, 0xDEEEDFB1L, 0xCDBE2C45L, 0x3FD5AF46L,
+      0x7198540DL, 0x83F3D70EL, 0x90A324FAL, 0x62C8A7F9L,
+      0xB602C312L, 0x44694011L, 0x5739B3E5L, 0xA55230E6L,
+      0xFB410CC2L, 0x092A8FC1L, 0x1A7A7C35L, 0xE811FF36L,
+      0x3CDB9BDDL, 0xCEB018DEL, 0xDDE0EB2AL, 0x2F8B6829L,
+      0x82F63B78L, 0x709DB87BL, 0x63CD4B8FL, 0x91A6C88CL,
+      0x456CAC67L, 0xB7072F64L, 0xA457DC90L, 0x563C5F93L,
+      0x082F63B7L, 0xFA44E0B4L, 0xE9141340L, 0x1B7F9043L,
+      0xCFB5F4A8L, 0x3DDE77ABL, 0x2E8E845FL, 0xDCE5075CL,
+      0x92A8FC17L, 0x60C37F14L, 0x73938CE0L, 0x81F80FE3L,
+      0x55326B08L, 0xA759E80BL, 0xB4091BFFL, 0x466298FCL,
+      0x1871A4D8L, 0xEA1A27DBL, 0xF94AD42FL, 0x0B21572CL,
+      0xDFEB33C7L, 0x2D80B0C4L, 0x3ED04330L, 0xCCBBC033L,
+      0xA24BB5A6L, 0x502036A5L, 0x4370C551L, 0xB11B4652L,
+      0x65D122B9L, 0x97BAA1BAL, 0x84EA524EL, 0x7681D14DL,
+      0x2892ED69L, 0xDAF96E6AL, 0xC9A99D9EL, 0x3BC21E9DL,
+      0xEF087A76L, 0x1D63F975L, 0x0E330A81L, 0xFC588982L,
+      0xB21572C9L, 0x407EF1CAL, 0x532E023EL, 0xA145813DL,
+      0x758FE5D6L, 0x87E466D5L, 0x94B49521L, 0x66DF1622L,
+      0x38CC2A06L, 0xCAA7A905L, 0xD9F75AF1L, 0x2B9CD9F2L,
+      0xFF56BD19L, 0x0D3D3E1AL, 0x1E6DCDEEL, 0xEC064EEDL,
+      0xC38D26C4L, 0x31E6A5C7L, 0x22B65633L, 0xD0DDD530L,
+      0x0417B1DBL, 0xF67C32D8L, 0xE52CC12CL, 0x1747422FL,
+      0x49547E0BL, 0xBB3FFD08L, 0xA86F0EFCL, 0x5A048DFFL,
+      0x8ECEE914L, 0x7CA56A17L, 0x6FF599E3L, 0x9D9E1AE0L,
+      0xD3D3E1ABL, 0x21B862A8L, 0x32E8915CL, 0xC083125FL,
+      0x144976B4L, 0xE622F5B7L, 0xF5720643L, 0x07198540L,
+      0x590AB964L, 0xAB613A67L, 0xB831C993L, 0x4A5A4A90L,
+      0x9E902E7BL, 0x6CFBAD78L, 0x7FAB5E8CL, 0x8DC0DD8FL,
+      0xE330A81AL, 0x115B2B19L, 0x020BD8EDL, 0xF0605BEEL,
+      0x24AA3F05L, 0xD6C1BC06L, 0xC5914FF2L, 0x37FACCF1L,
+      0x69E9F0D5L, 0x9B8273D6L, 0x88D28022L, 0x7AB90321L,
+      0xAE7367CAL, 0x5C18E4C9L, 0x4F48173DL, 0xBD23943EL,
+      0xF36E6F75L, 0x0105EC76L, 0x12551F82L, 0xE03E9C81L,
+      0x34F4F86AL, 0xC69F7B69L, 0xD5CF889DL, 0x27A40B9EL,
+      0x79B737BAL, 0x8BDCB4B9L, 0x988C474DL, 0x6AE7C44EL,
+      0xBE2DA0A5L, 0x4C4623A6L, 0x5F16D052L, 0xAD7D5351L
+   };
+
+   for (int i = 0; i < size; i++)
+   {
+      crc_int = crc32_tab[(((unsigned char)crc_int) ^ ((unsigned char*)buffer)[i])] ^ (crc_int >> 8);
+   }
+   crc_int = ~crc_int;
+
+   *crc = crc_int;
+
+   return 0;
+   #endif
+}
+
+int
+pgmoneta_create_crc32c_file(char* path, char** crc)
+{
+   FILE* file = NULL;
+   char read_buf[16384];
+   unsigned long read_bytes = 0;
+   char* crc_string;
+   uint32_t crc_buf = 0;
+
+   file = fopen(path, "rb");
+   if (file == NULL)
+   {
+      goto error;
+   }
+
+   memset(read_buf, 0, sizeof(read_buf));
+
+   while ((read_bytes = fread(read_buf, 1, sizeof(read_buf), file)) > 0)
+   {
+      pgmoneta_create_crc32c_buffer(read_buf, read_bytes, &crc_buf);
+   }
+
+   fclose(file);
+
+   crc_string = malloc(9);
+   memset(crc_string, 0, 9);
+
+   for (int i = 0; i < 8; i++)
+   {
+      sprintf(crc_string, "%08x", crc_buf);
+   }
+
+   if (crc_string == NULL)
+   {
+      goto error;
+   }
+
+   *crc = crc_string;
+
+   return 0;
+
+error:
+   return 1;
+}
+
 void
 pgmoneta_close_ssl(SSL* ssl)
 {