MD5-related optimizations

Makefile.in

@@ -31,12 +31,12 @@ VERSION=@RSYNC_VERSION@
 .SUFFIXES:
 .SUFFIXES: .c .o
 
-SIMD_x86_64=simd-checksum-x86_64.o lib/md5-asm-x86_64.o
+SIMD_x86_64=simd-checksum-x86_64.o simd-md5-parallel-x86_64.o lib/md5-asm-x86_64.o
 
 GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsync-ssl.1 rsyncd.conf.5
 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
 	lib/pool_alloc.h
-LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
+LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o lib/md5p8.o \
 	lib/permstring.o lib/pool_alloc.o lib/sysacls.o lib/sysxattrs.o @LIBOBJS@
 zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
 	zlib/trees.o zlib/zutil.o zlib/adler32.o zlib/compress.o zlib/crc32.o
@@ -122,6 +122,9 @@ rounding.h: rounding.c rsync.h proto.h
 simd-checksum-x86_64.o: simd-checksum-x86_64.cpp
 	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $<
 
+simd-md5-parallel-x86_64.o: simd-md5-parallel-x86_64.cpp
+	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $<
+
 lib/md5-asm-x86_64.o: lib/md5-asm-x86_64.s
 	$(CC) -c -o $@ $<
 

checksum.c

@@ -30,10 +30,6 @@
 #ifdef SUPPORT_XXHASH
 #include "xxhash.h"
 #endif
-#ifdef USE_OPENSSL
-#include "openssl/md4.h"
-#include "openssl/md5.h"
-#endif
 
 extern int am_server;
 extern int whole_file;
@@ -49,27 +45,22 @@ extern const char *checksum_choice;
 #define CSUM_MD4 4
 #define CSUM_MD5 5
 #define CSUM_XXH64 6
+#define CSUM_MD5P8 7
 
 struct name_num_obj valid_checksums = {
 	"checksum", NULL, NULL, 0, 0, {
 #ifdef SUPPORT_XXHASH
 		{ CSUM_XXH64, "xxh64", NULL },
 		{ CSUM_XXH64, "xxhash", NULL },
 #endif
+        { CSUM_MD5P8, "md5p8", NULL },
 		{ CSUM_MD5, "md5", NULL },
 		{ CSUM_MD4, "md4", NULL },
 		{ CSUM_NONE, "none", NULL },
 		{ 0, NULL, NULL }
 	}
 };
 
-#ifndef USE_OPENSSL
-#define MD5_CTX md_context
-#define MD5_Init md5_begin
-#define MD5_Update md5_update
-#define MD5_Final(digest, cptr) md5_result(cptr, digest)
-#endif
-
 int xfersum_type = 0; /* used for the file transfer checksums */
 int checksum_type = 0; /* used for the pre-transfer (--checksum) checksums */
 
@@ -150,6 +141,7 @@ int csum_len_for_type(int cst, BOOL flist_csum)
 	  case CSUM_MD4_OLD:
 	  case CSUM_MD4_BUSTED:
 		return MD4_DIGEST_LEN;
+	  case CSUM_MD5P8:
 	  case CSUM_MD5:
 		return MD5_DIGEST_LEN;
 #ifdef SUPPORT_XXHASH
@@ -175,6 +167,7 @@ int canonical_checksum(int csum_type)
 	  case CSUM_MD4_BUSTED:
 		break;
 	  case CSUM_MD4:
+	  case CSUM_MD5P8:
 	  case CSUM_MD5:
 		return -1;
 #ifdef SUPPORT_XXHASH
@@ -208,16 +201,27 @@ uint32 get_checksum1(char *buf1, int32 len)
 	}
 	return (s1 & 0xffff) + (s2 << 16);
 }
-#endif
 
-void get_checksum2(char *buf, int32 len, char *sum)
+void checksum2_enable_prefetch(UNUSED(struct map_struct *map), UNUSED(OFF_T len), UNUSED(int32 blocklen))
+{
+}
+
+void checksum2_disable_prefetch()
+{
+}
+
+void get_checksum2(char *buf, int32 len, char *sum, UNUSED(OFF_T prefetch_offset))
+#else
+void get_checksum2_nosimd(char *buf, int32 len, char *sum, UNUSED(OFF_T prefetch_offset))
+#endif
 {
 	switch (xfersum_type) {
 #ifdef SUPPORT_XXHASH
 	  case CSUM_XXH64:
 		SIVAL64(sum, 0, XXH64(buf, len, checksum_seed));
 		break;
 #endif
+	  case CSUM_MD5P8:  // == CSUM_MD5 for checksum2
 	  case CSUM_MD5: {
 		MD5_CTX m5;
 		uchar seedbuf[4];
@@ -333,6 +337,21 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
 		break;
 	  }
 #endif
+	  case CSUM_MD5P8: {
+		MD5P8_CTX m5p8;
+
+		MD5P8_Init(&m5p8);
+
+		for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE)
+			MD5P8_Update(&m5p8, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE);
+
+		remainder = (int32)(len - i);
+		if (remainder > 0)
+			MD5P8_Update(&m5p8, (uchar *)map_ptr(buf, i, remainder), remainder);
+
+		MD5P8_Final((uchar *)sum, &m5p8);
+		break;
+	  }
 	  case CSUM_MD5: {
 		MD5_CTX m5;
 
@@ -408,6 +427,7 @@ static union {
 #ifdef SUPPORT_XXHASH
 static XXH64_state_t* xxh64_state;
 #endif
+static MD5P8_CTX m5p8;
 static int cursum_type;
 
 void sum_init(int csum_type, int seed)
@@ -426,6 +446,9 @@ void sum_init(int csum_type, int seed)
 		XXH64_reset(xxh64_state, 0);
 		break;
 #endif
+	  case CSUM_MD5P8:
+		MD5P8_Init(&m5p8);
+		break;
 	  case CSUM_MD5:
 		MD5_Init(&ctx.m5);
 		break;
@@ -468,6 +491,9 @@ void sum_update(const char *p, int32 len)
 		XXH64_update(xxh64_state, p, len);
 		break;
 #endif
+	  case CSUM_MD5P8:
+		MD5P8_Update(&m5p8, (uchar *)p, len);
+		break;
 	  case CSUM_MD5:
 		MD5_Update(&ctx.m5, (uchar *)p, len);
 		break;
@@ -522,6 +548,9 @@ int sum_end(char *sum)
 		SIVAL64(sum, 0, XXH64_digest(xxh64_state));
 		break;
 #endif
+	  case CSUM_MD5P8:
+		MD5P8_Final((uchar *)sum, &m5p8);
+		break;
 	  case CSUM_MD5:
 		MD5_Final((uchar *)sum, &ctx.m5);
 		break;

generator.c

@@ -706,10 +706,12 @@ static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
 	if (append_mode > 0 && f_copy < 0)
 		return 0;
 
-	if (len > 0)
+	if (len > 0) {
 		mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
-	else
+        checksum2_enable_prefetch(mapbuf, len, sum.blength);
+	} else {
 		mapbuf = NULL;
+    }
 
 	for (i = 0; i < sum.count; i++) {
 		int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
@@ -727,7 +729,7 @@ static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
 		}
 
 		sum1 = get_checksum1(map, n1);
-		get_checksum2(map, n1, sum2);
+		get_checksum2(map, n1, sum2, offset - n1);
 
 		if (DEBUG_GTE(DELTASUM, 3)) {
 			rprintf(FINFO,
@@ -739,8 +741,10 @@ static int generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
 		write_buf(f_out, sum2, sum.s2length);
 	}
 
-	if (mapbuf)
+	if (mapbuf) {
 		unmap_file(mapbuf);
+	    checksum2_disable_prefetch();
+    }
 
 	return 0;
 }

lib/md5p8.c

@@ -0,0 +1,130 @@
+/*
+ * MD5-based hash friendly to parallel processing, reference implementation
+ *
+ * Author: Jorrit Jongma, 2020
+ *
+ * Released in the public domain falling back to the MIT license
+ * ( http://www.opensource.org/licenses/MIT ) in case public domain does not
+ * apply in your country.
+ */
+/*
+ * MD5P8 is an MD5-based hash friendly to parallel processing. The input
+ * stream is divided into 8 independent streams. For each 512 bytes of input,
+ * the first 64 bytes are send to the first stream, the second 64 bytes to
+ * the second stream, etc. The input stream is padded with zeros to the next
+ * multiple of 512 bytes, then a normal MD5 hash is computed on a buffer
+ * containing the A, B, C, and D states of the 8 individual streams, followed
+ * by the (unpadded) length of the input.
+ *
+ * On non-SIMD accelerated CPUs the performance of MD5P8 is slightly lower
+ * than normal MD5 (particularly on files smaller than 10 kB), but with
+ * SIMD-based parallel processing it can be two to six times as fast. Even in
+ * the best-case scenario, xxHash is still at least twice as fast and should
+ * be preferred when available.
+ */
+
+#include "rsync.h"
+
+#ifndef HAVE_SIMD
+
+// each MD5_CTX needs to be 8-byte aligned
+#define MD5P8_Contexts(ctx, index) ((MD5_CTX*)((((uintptr_t)((ctx)->context_storage) + 7) & ~7) + (index)*((sizeof(MD5_CTX) + 7) & ~7)))
+
+int MD5P8_Init(MD5P8_CTX *ctx)
+{
+    int i;
+    for (i = 0; i < 8; i++) {
+        MD5_Init(MD5P8_Contexts(ctx, i));
+    }
+    ctx->used = 0;
+    ctx->next = 0;
+    return 1;
+}
+
+int MD5P8_Update(MD5P8_CTX *ctx, const uchar *input, uint32 length)
+{
+    uint32 pos = 0;
+
+    if ((ctx->used) || (length < 64)) {
+        int cpy = MIN(length, 64 - ctx->used);
+        memmove(&ctx->buffer[ctx->used], input, cpy);
+        ctx->used += cpy;
+        length -= cpy;
+        pos += cpy;
+
+        if (ctx->used == 64) {
+            MD5_Update(MD5P8_Contexts(ctx, ctx->next), ctx->buffer, 64);
+            ctx->used = 0;
+            ctx->next = (ctx->next + 1) % 8;
+        }
+    }
+
+    while (length >= 64) {
+        MD5_Update(MD5P8_Contexts(ctx, ctx->next), &input[pos], 64);
+        ctx->next = (ctx->next + 1) % 8;
+        pos += 64;
+        length -= 64;
+    }
+
+    if (length) {
+        memcpy(ctx->buffer, &input[pos], length);
+        ctx->used = length;
+    }
+    return 1;
+}
+
+int MD5P8_Final(uchar digest[MD5_DIGEST_LEN], MD5P8_CTX *ctx)
+{
+    int i;
+    uint32 low = 0, high = 0, sub = ctx->used ? 64 - ctx->used : 0;
+    if (ctx->used) {
+        uchar tmp[64];
+        memset(tmp, 0, 64);
+        MD5P8_Update(ctx, tmp, 64 - ctx->used);
+    }
+    memset(ctx->buffer, 0, 64);
+    while (ctx->next != 0) {
+        MD5P8_Update(ctx, ctx->buffer, 64);
+        sub += 64;
+    }
+
+    uchar state[34*4] = {0};
+
+    for (i = 0; i < 8; i++) {
+        MD5_CTX* md = MD5P8_Contexts(ctx, i);
+#ifdef USE_OPENSSL
+        if (low + md->Nl < low) high++;
+        low += md->Nl;
+        high += md->Nh;
+#else
+        if (low + md->totalN < low) high++;
+        low += md->totalN;
+        high += md->totalN2;
+#endif
+        SIVALu(state, i*16, md->A);
+        SIVALu(state, i*16 + 4, md->B);
+        SIVALu(state, i*16 + 8, md->C);
+        SIVALu(state, i*16 + 12, md->D);
+    }
+
+#ifndef USE_OPENSSL
+	high = (low >> 29) | (high << 3);
+	low = (low << 3);
+#endif
+
+    sub <<= 3;
+    if (low - sub > low) high--;
+    low -= sub;
+
+    SIVALu(state, 32*4, low);
+    SIVALu(state, 33*4, high);
+
+    MD5_CTX md;
+    MD5_Init(&md);
+    MD5_Update(&md, state, 34*4);
+    MD5_Final(digest, &md);
+
+    return 1;
+}
+
+#endif

lib/mdigest.h

@@ -1,5 +1,10 @@
 /* The include file for both the MD4 and MD5 routines. */
 
+#ifdef USE_OPENSSL
+#include "openssl/md4.h"
+#include "openssl/md5.h"
+#endif
+
 #define MD4_DIGEST_LEN 16
 #define MD5_DIGEST_LEN 16
 #define MAX_DIGEST_LEN MD5_DIGEST_LEN
@@ -18,7 +23,23 @@ void mdfour_update(md_context *md, const uchar *in, uint32 length);
 void mdfour_result(md_context *md, uchar digest[MD4_DIGEST_LEN]);
 
 #ifndef USE_OPENSSL
+#define MD5_CTX md_context
+#define MD5_Init md5_begin
+#define MD5_Update md5_update
+#define MD5_Final(digest, cptr) md5_result(cptr, digest)
+
 void md5_begin(md_context *ctx);
 void md5_update(md_context *ctx, const uchar *input, uint32 length);
 void md5_result(md_context *ctx, uchar digest[MD5_DIGEST_LEN]);
 #endif
+
+typedef struct {
+    uchar context_storage[1024];
+    uchar buffer[512];
+    int used;
+    int next;
+} MD5P8_CTX;
+
+int MD5P8_Init(MD5P8_CTX *ctx);
+int MD5P8_Update(MD5P8_CTX *ctx, const uchar *input, uint32 length);
+int MD5P8_Final(uchar digest[MD5_DIGEST_LEN], MD5P8_CTX *ctx);