Skip to content

Commit

Permalink
mem: Align data to natural alignment.
Browse files Browse the repository at this point in the history 
Ensure that the pointer returned by mem_alloc & co. is properly aligned
similar to malloc on a given platform. This is important e.g. for SIMD
processing, including libc string operations.

Also, made mem_realloc optimize away realloc call if there is enough
storage in the allocated buffer. This only works if the input pointer
is the only reference to the memory block.

Also, fixed formatting in mem_status.
  • Loading branch information
@Lastique
Lastique committed Jun 25, 2022
1 parent 52bef29 commit 8f3c320
Showing 1 changed file with 96 additions and 40 deletions.
136 changes: 96 additions & 40 deletions src/mem/mem.c
View file
Expand Up @@ -29,9 +29,9 @@
struct mem {
size_t nrefs; /**< Number of references */
mem_destroy_h *dh; /**< Destroy handler */
size_t size; /**< Size of memory object */
#if MEM_DEBUG
size_t magic; /**< Magic number */
size_t size; /**< Size of memory object */
struct le le; /**< Linked list element */
struct btrace btraces; /**< Backtrace array */
#endif
Expand Down Expand Up @@ -68,45 +68,80 @@ static inline void mem_unlock(void)
}

/** Update statistics for mem_zalloc() */
#define STAT_ALLOC(m, size) \
#define STAT_ALLOC(_m, _size) \
mem_lock(); \
memstat.bytes_cur += (size); \
memstat.bytes_cur += (_size); \
memstat.bytes_peak = max(memstat.bytes_cur, memstat.bytes_peak); \
++memstat.blocks_cur; \
memstat.blocks_peak = max(memstat.blocks_cur, memstat.blocks_peak); \
mem_unlock(); \
(m)->size = (size); \
(m)->magic = mem_magic;
(_m)->size = (_size); \
(_m)->magic = mem_magic;

/** Update statistics for mem_realloc() */
#define STAT_REALLOC(m, size) \
#define STAT_REALLOC(_m, _size) \
mem_lock(); \
memstat.bytes_cur += ((size) - (m)->size); \
memstat.bytes_cur += ((_size) - (_m)->size); \
memstat.bytes_peak = max(memstat.bytes_cur, memstat.bytes_peak); \
mem_unlock(); \
(m)->size = (size)
(_m)->size = (_size)

/** Update statistics for mem_deref() */
#define STAT_DEREF(m) \
#define STAT_DEREF(_m) \
mem_lock(); \
memstat.bytes_cur -= (m)->size; \
memstat.bytes_cur -= (_m)->size; \
--memstat.blocks_cur; \
mem_unlock(); \
memset((m), 0xb5, sizeof(struct mem) + (m)->size)
memset((_m), 0xb5, sizeof(struct mem) + (_m)->size)

/** Check magic number in memory object */
#define MAGIC_CHECK(m) \
if (mem_magic != (m)->magic) { \
#define MAGIC_CHECK(_m) \
if (mem_magic != (_m)->magic) { \
DEBUG_WARNING("%s: magic check failed 0x%08x (%p)\n", \
__REFUNC__, (m)->magic, (m)+1); \
__REFUNC__, (_m)->magic, get_mem_data((_m))); \
BREAKPOINT; \
}
#else
#define STAT_ALLOC(m, size)
#define STAT_REALLOC(m, size)
#define STAT_DEREF(m)
#define MAGIC_CHECK(m)
#define STAT_ALLOC(_m, _size) (_m)->size = (_size);
#define STAT_REALLOC(_m, _size) (_m)->size = (_size);
#define STAT_DEREF(_m)
#define MAGIC_CHECK(_m)
#endif

#ifndef SIZE_MAX
#define SIZE_MAX (~((size_t)0))
#endif


enum {
#if defined(__x86_64__)
/* Use 16-byte alignment on x86-x32 as well */
mem_alignment = 16u,
#else
mem_alignment = sizeof(void*) >= 8u ? 16u : 8u,
#endif
alignment_mask = mem_alignment - 1u,
mem_header_size = (sizeof(struct mem) + alignment_mask) &
(~(size_t)alignment_mask)
};


static inline size_t align_up(size_t n)
{
return (n + (size_t)alignment_mask) & (~(size_t)alignment_mask);
}


static inline struct mem *get_mem(void *p)
{
return (struct mem *)(void *)(((unsigned char *)p) - mem_header_size);
}


static inline void *get_mem_data(struct mem *m)
{
return (void *)(((unsigned char *)m) + mem_header_size);
}


/**
Expand All @@ -120,6 +155,11 @@ static inline void mem_unlock(void)
void *mem_alloc(size_t size, mem_destroy_h *dh)
{
struct mem *m;
size_t capacity;

capacity = align_up(size);
if (capacity < size || capacity > (SIZE_MAX - mem_header_size))
return NULL;

#if MEM_DEBUG
mem_lock();
Expand All @@ -130,7 +170,7 @@ void *mem_alloc(size_t size, mem_destroy_h *dh)
mem_unlock();
#endif

m = malloc(sizeof(*m) + size);
m = malloc(mem_header_size + capacity);
if (!m)
return NULL;

Expand All @@ -145,9 +185,9 @@ void *mem_alloc(size_t size, mem_destroy_h *dh)
m->nrefs = 1;
m->dh = dh;

STAT_ALLOC(m, size);
STAT_ALLOC(m, capacity);

return (void *)(m + 1);
return get_mem_data(m);
}


Expand Down Expand Up @@ -186,14 +226,28 @@ void *mem_zalloc(size_t size, mem_destroy_h *dh)
void *mem_realloc(void *data, size_t size)
{
struct mem *m, *m2;
size_t capacity;

if (!data)
return NULL;

m = ((struct mem *)data) - 1;
capacity = align_up(size);
if (capacity < size || capacity > (SIZE_MAX - mem_header_size))
return NULL;

m = get_mem(data);

MAGIC_CHECK(m);

if (m->nrefs > 1u) {
void* p = mem_alloc(size, m->dh);
if (p) {
memcpy(p, data, m->size);
mem_deref(data);
}
return p;
}

#if MEM_DEBUG
mem_lock();

Expand All @@ -205,11 +259,16 @@ void *mem_realloc(void *data, size_t size)
}
}

list_unlink(&m->le);
if (size > m->size)
list_unlink(&m->le);

mem_unlock();
#endif

m2 = realloc(m, sizeof(*m2) + size);
if (size <= m->size)
return data;

m2 = realloc(m, mem_header_size + capacity);

#if MEM_DEBUG
mem_lock();
Expand All @@ -221,17 +280,12 @@ void *mem_realloc(void *data, size_t size)
return NULL;
}

STAT_REALLOC(m2, size);
STAT_REALLOC(m2, capacity);

return (void *)(m2 + 1);
return get_mem_data(m2);
}


#ifndef SIZE_MAX
#define SIZE_MAX (~((size_t)0))
#endif


/**
* Re-allocate a reference-counted array
*
Expand Down Expand Up @@ -275,7 +329,7 @@ void mem_destructor(void *data, mem_destroy_h *dh)
if (!data)
return;

m = ((struct mem *)data) - 1;
m = get_mem(data);

MAGIC_CHECK(m);

Expand All @@ -297,7 +351,7 @@ void *mem_ref(void *data)
if (!data)
return NULL;

m = ((struct mem *)data) - 1;
m = get_mem(data);

MAGIC_CHECK(m);

Expand All @@ -324,7 +378,7 @@ void *mem_deref(void *data)
if (!data)
return NULL;

m = ((struct mem *)data) - 1;
m = get_mem(data);

MAGIC_CHECK(m);

Expand Down Expand Up @@ -366,7 +420,7 @@ uint32_t mem_nrefs(const void *data)
if (!data)
return 0;

m = ((struct mem *)data) - 1;
m = get_mem((void*)data);

MAGIC_CHECK(m);

Expand All @@ -378,7 +432,7 @@ uint32_t mem_nrefs(const void *data)
static bool debug_handler(struct le *le, void *arg)
{
struct mem *m = le->data;
const uint8_t *p = (const uint8_t *)(m + 1);
const uint8_t *p = get_mem_data(m);
size_t i;

(void)arg;
Expand Down Expand Up @@ -483,13 +537,15 @@ int mem_status(struct re_printf *pf, void *unused)
c = list_count(&meml);
mem_unlock();

err |= re_hprintf(pf, "Memory status: (%u bytes overhead pr block)\n",
sizeof(struct mem));
err |= re_hprintf(pf, " Cur: %u blocks, %u bytes (total %u bytes)\n",
err |= re_hprintf(pf, "Memory status: (%zu bytes overhead pr block)\n",
(size_t)mem_header_size);
err |= re_hprintf(pf,
" Cur: %zu blocks, %zu bytes (total %zu bytes)\n",
stat.blocks_cur, stat.bytes_cur,
stat.bytes_cur
+(stat.blocks_cur*sizeof(struct mem)));
err |= re_hprintf(pf, " Peak: %u blocks, %u bytes (total %u bytes)\n",
err |= re_hprintf(pf,
" Peak: %zu blocks, %zu bytes (total %zu bytes)\n",
stat.blocks_peak, stat.bytes_peak,
stat.bytes_peak
+(stat.blocks_peak*sizeof(struct mem)));
Expand Down

0 comments on commit 8f3c320

Please sign in to comment.