mem: Align data to natural alignment.

Ensure that the pointer returned by mem_alloc & co. is properly aligned similar to malloc on a given platform. This is important e.g. for SIMD processing, including libc string operations. Also, fixed mem_realloc behavior when there are multiple references to the memory buffer being reallocated. In this case, the implementation now allocates a new buffer and copies the contents and decrements the reference counter on the original memory buffer. This way the other references for the old buffer remain valid. Also, fixed formatting in mem_status.
baresip · Jul 15, 2022 · 18d5b05 · 18d5b05
1 parent 3a3ee48
commit 18d5b05
Showing 1 changed file with 80 additions and 38 deletions.
diff --git a/src/mem/mem.c b/src/mem/mem.c
@@ -28,10 +28,10 @@
 /** Defines a reference-counting memory object */
 struct mem {
 	size_t nrefs;          /**< Number of references  */
+	size_t size;           /**< Size of memory object */
 	mem_destroy_h *dh;     /**< Destroy handler       */
 #if MEM_DEBUG
 	size_t magic;          /**< Magic number          */
-	size_t size;           /**< Size of memory object */
 	struct le le;          /**< Linked list element   */
 	struct btrace btraces; /**< Backtrace array       */
 #endif
@@ -68,45 +68,74 @@ static inline void mem_unlock(void)
 }
 
 /** Update statistics for mem_zalloc() */
-#define STAT_ALLOC(m, size) \
+#define STAT_ALLOC(_m, _size) \
 	mem_lock(); \
-	memstat.bytes_cur += (size); \
+	memstat.bytes_cur += (_size); \
 	memstat.bytes_peak = max(memstat.bytes_cur, memstat.bytes_peak); \
 	++memstat.blocks_cur; \
 	memstat.blocks_peak = max(memstat.blocks_cur, memstat.blocks_peak); \
 	mem_unlock(); \
-	(m)->size = (size); \
-	(m)->magic = mem_magic;
+	(_m)->size = (_size); \
+	(_m)->magic = mem_magic;
 
 /** Update statistics for mem_realloc() */
-#define STAT_REALLOC(m, size) \
+#define STAT_REALLOC(_m, _size) \
 	mem_lock(); \
-	memstat.bytes_cur += ((size) - (m)->size); \
+	memstat.bytes_cur += ((_size) - (_m)->size); \
 	memstat.bytes_peak = max(memstat.bytes_cur, memstat.bytes_peak); \
 	mem_unlock(); \
-	(m)->size = (size)
+	(_m)->size = (_size)
 
 /** Update statistics for mem_deref() */
-#define STAT_DEREF(m) \
+#define STAT_DEREF(_m) \
 	mem_lock(); \
-	memstat.bytes_cur -= (m)->size; \
+	memstat.bytes_cur -= (_m)->size; \
 	--memstat.blocks_cur; \
 	mem_unlock(); \
-	memset((m), 0xb5, sizeof(struct mem) + (m)->size)
+	memset((_m), 0xb5, sizeof(struct mem) + (_m)->size)
 
 /** Check magic number in memory object */
-#define MAGIC_CHECK(m) \
-	if (mem_magic != (m)->magic) { \
-		DEBUG_WARNING("%s: magic check failed 0x%08x (%p)\n", \
-			      __REFUNC__, (m)->magic, (m)+1);	      \
+#define MAGIC_CHECK(_m) \
+	if (mem_magic != (_m)->magic) { \
+		DEBUG_WARNING("%s: magic check failed 0x%08zx (%p)\n", \
+			__REFUNC__, (_m)->magic, get_mem_data((_m))); \
 		BREAKPOINT;					      \
 	}
 #else
-#define STAT_ALLOC(m, size)
-#define STAT_REALLOC(m, size)
-#define STAT_DEREF(m)
-#define MAGIC_CHECK(m)
+#define STAT_ALLOC(_m, _size) (_m)->size = (_size);
+#define STAT_REALLOC(_m, _size) (_m)->size = (_size);
+#define STAT_DEREF(_m)
+#define MAGIC_CHECK(_m)
+#endif
+
+#ifndef SIZE_MAX
+#define SIZE_MAX    (~((size_t)0))
+#endif
+
+
+enum {
+#if defined(__x86_64__)
+	/* Use 16-byte alignment on x86-x32 as well */
+	mem_alignment = 16u,
+#else
+	mem_alignment = sizeof(void*) >= 8u ? 16u : 8u,
 #endif
+	alignment_mask = mem_alignment - 1u,
+	mem_header_size = (sizeof(struct mem) + alignment_mask) &
+		(~(size_t)alignment_mask)
+};
+
+
+static inline struct mem *get_mem(void *p)
+{
+	return (struct mem *)(void *)(((unsigned char *)p) - mem_header_size);
+}
+
+
+static inline void *get_mem_data(struct mem *m)
+{
+	return (void *)(((unsigned char *)m) + mem_header_size);
+}
 
 
 /**
@@ -121,6 +150,9 @@ void *mem_alloc(size_t size, mem_destroy_h *dh)
 {
 	struct mem *m;
 
+	if (size > (SIZE_MAX - mem_header_size))
+		return NULL;
+
 #if MEM_DEBUG
 	mem_lock();
 	if (-1 != threshold && (memstat.blocks_cur >= (size_t)threshold)) {
@@ -130,7 +162,7 @@ void *mem_alloc(size_t size, mem_destroy_h *dh)
 	mem_unlock();
 #endif
 
-	m = malloc(sizeof(*m) + size);
+	m = malloc(mem_header_size + size);
 	if (!m)
 		return NULL;
 
@@ -147,7 +179,7 @@ void *mem_alloc(size_t size, mem_destroy_h *dh)
 
 	STAT_ALLOC(m, size);
 
-	return (void *)(m + 1);
+	return get_mem_data(m);
 }
 
 
@@ -190,10 +222,22 @@ void *mem_realloc(void *data, size_t size)
 	if (!data)
 		return NULL;
 
-	m = ((struct mem *)data) - 1;
+	if (size > (SIZE_MAX - mem_header_size))
+		return NULL;
+
+	m = get_mem(data);
 
 	MAGIC_CHECK(m);
 
+	if (m->nrefs > 1u) {
+		void* p = mem_alloc(size, m->dh);
+		if (p) {
+			memcpy(p, data, m->size);
+			mem_deref(data);
+		}
+		return p;
+	}
+
 #if MEM_DEBUG
 	mem_lock();
 
@@ -206,10 +250,11 @@ void *mem_realloc(void *data, size_t size)
 	}
 
 	list_unlink(&m->le);
+
 	mem_unlock();
 #endif
 
-	m2 = realloc(m, sizeof(*m2) + size);
+	m2 = realloc(m, mem_header_size + size);
 
 #if MEM_DEBUG
 	mem_lock();
@@ -223,15 +268,10 @@ void *mem_realloc(void *data, size_t size)
 
 	STAT_REALLOC(m2, size);
 
-	return (void *)(m2 + 1);
+	return get_mem_data(m2);
 }
 
 
-#ifndef SIZE_MAX
-#define SIZE_MAX    (~((size_t)0))
-#endif
-
-
 /**
  * Re-allocate a reference-counted array
  *
@@ -275,7 +315,7 @@ void mem_destructor(void *data, mem_destroy_h *dh)
 	if (!data)
 		return;
 
-	m = ((struct mem *)data) - 1;
+	m = get_mem(data);
 
 	MAGIC_CHECK(m);
 
@@ -297,7 +337,7 @@ void *mem_ref(void *data)
 	if (!data)
 		return NULL;
 
-	m = ((struct mem *)data) - 1;
+	m = get_mem(data);
 
 	MAGIC_CHECK(m);
 
@@ -324,7 +364,7 @@ void *mem_deref(void *data)
 	if (!data)
 		return NULL;
 
-	m = ((struct mem *)data) - 1;
+	m = get_mem(data);
 
 	MAGIC_CHECK(m);
 
@@ -366,7 +406,7 @@ uint32_t mem_nrefs(const void *data)
 	if (!data)
 		return 0;
 
-	m = ((struct mem *)data) - 1;
+	m = get_mem((void*)data);
 
 	MAGIC_CHECK(m);
 
@@ -378,7 +418,7 @@ uint32_t mem_nrefs(const void *data)
 static bool debug_handler(struct le *le, void *arg)
 {
 	struct mem *m = le->data;
-	const uint8_t *p = (const uint8_t *)(m + 1);
+	const uint8_t *p = get_mem_data(m);
 	size_t i;
 
 	(void)arg;
@@ -483,13 +523,15 @@ int mem_status(struct re_printf *pf, void *unused)
 	c = list_count(&meml);
 	mem_unlock();
 
-	err |= re_hprintf(pf, "Memory status: (%u bytes overhead pr block)\n",
-			  sizeof(struct mem));
-	err |= re_hprintf(pf, " Cur:  %u blocks, %u bytes (total %u bytes)\n",
+	err |= re_hprintf(pf, "Memory status: (%zu bytes overhead pr block)\n",
+			  (size_t)mem_header_size);
+	err |= re_hprintf(pf,
+			  " Cur:  %zu blocks, %zu bytes (total %zu bytes)\n",
 			  stat.blocks_cur, stat.bytes_cur,
 			  stat.bytes_cur
 			  +(stat.blocks_cur*sizeof(struct mem)));
-	err |= re_hprintf(pf, " Peak: %u blocks, %u bytes (total %u bytes)\n",
+	err |= re_hprintf(pf,
+			  " Peak: %zu blocks, %zu bytes (total %zu bytes)\n",
 			  stat.blocks_peak, stat.bytes_peak,
 			  stat.bytes_peak
 			  +(stat.blocks_peak*sizeof(struct mem)));