Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PROBLEM] No custom memory allocator for msg can be provided #4125

Open
wants to merge 64 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
64 commits
Select commit Hold shift + click to select a range
4f84758
Implement a very simple zero-lock message pool to test performance gains
f18m Aug 13, 2019
ea0dc06
Merge branch 'master' of https://github.com/f18m/libzmq.git
f18m Aug 13, 2019
a24f2af
Allow to choose message sizes as well
f18m Aug 13, 2019
1bd2ae1
Allow using env variables to do some basic overriding
f18m Aug 13, 2019
252e8d4
fix typo
f18m Aug 13, 2019
4a30795
add TCP kernel socket buffer setting
f18m Aug 13, 2019
577232e
Merge remote-tracking branch 'upstream/master'
f18m Aug 25, 2019
ff8d79f
Merge remote-tracking branch 'upstream/master'
f18m Aug 28, 2019
00e514e
First implementation of global memory pool for ZMQ
f18m Aug 28, 2019
18c52c4
Remove changes related to graph generation
f18m Aug 28, 2019
a720a31
allow testing up to 8k msg sizes
f18m Aug 28, 2019
b9e1f01
correctly deallocate memory pool blocks
f18m Aug 30, 2019
1649701
fix build with no draft API
f18m Aug 30, 2019
0baafa4
never use allocator for VSM
f18m Aug 31, 2019
59cbfac
Merge branch 'master' of https://github.com/zeromq/libzmq into memory…
May 2, 2020
f0a7a7f
Fixes cmake build
May 2, 2020
f682600
Changes to base class with virtuals
May 2, 2020
3a3d877
Makes max message size dynamic
May 2, 2020
b416348
Dynamically grows mempool
May 2, 2020
cfd4c85
Updates dynamic global pool
May 3, 2020
1dd2304
Removes unnecessary class
May 3, 2020
d06f868
Adds new files to makefile
May 4, 2020
cfa228b
Adds concurrentqueue to sources
May 4, 2020
d96d616
Fixes some warnings
May 4, 2020
caf7798
Adds includes
May 4, 2020
5fbc4cc
Makes initial number of messages a bit more dynamic
May 5, 2020
d2c53c5
Hides global allocator implementation and option when C++11 not avail…
May 5, 2020
348865f
Fixes msvc __cplusplus reporting
May 5, 2020
59c6a6c
Improves <C++11 support
May 5, 2020
1f5abc1
Fixes missing declaration
May 5, 2020
2c29abc
Adds more c++11 guards
May 5, 2020
d7f9452
Adds test and moves queue to external
May 12, 2020
1450beb
Merge branch 'master' of https://github.com/zeromq/libzmq into memory…
May 12, 2020
f4973dc
Fixes bad path
May 12, 2020
3b9ec2f
Fixes bad path
May 12, 2020
a260668
Fixes formatting
May 12, 2020
7dafdf7
Adds destroy to test
May 13, 2020
a3bfc67
Fixes wrong increment
May 13, 2020
cd90418
Fixes bad c+11
May 13, 2020
a575335
Adds a basic concurrent queue
May 13, 2020
74dd371
Removes some debug code
May 13, 2020
e9c3a01
Adds newline and nullptr
May 13, 2020
aaa10dd
Adds a start on function pointer interface
May 13, 2020
77293ab
Adds missing free
May 13, 2020
1309316
Cleans up some includes
May 16, 2020
d666af8
Fixes bad options
May 16, 2020
73807f8
Moves to draft
May 16, 2020
ffcede1
Fixes formatting
May 16, 2020
312f8c3
Updates copyright years
May 16, 2020
bf495f8
Switches to new/delete
May 16, 2020
61870a4
Switches to alternative log2
May 16, 2020
c13f837
More copyright years
May 16, 2020
ea9c5dc
Fixes more formatting
May 16, 2020
20f49ec
Fixes more bad years
May 16, 2020
ba05e8f
Fixes some concurrency issues and bugs
May 17, 2020
afb858d
Merge branch 'master' into memorypool
Sep 19, 2020
32d827d
Adds destroy fn
Sep 19, 2020
84b4f8f
Adds destroy
Jan 16, 2021
633bf65
Removes pool allocator
Jan 17, 2021
cfd982d
Fixes delete
Jan 17, 2021
e7ad8b4
Fixes non-draft build
Jan 17, 2021
60a9cdf
Fixes formatting
Jan 18, 2021
ef4b77f
Inits group type
Jan 18, 2021
6edbcfd
Fix formatting as clang version mismatch between my install and CI
Jan 18, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -615,6 +615,8 @@ if(MSVC)

zmq_check_cxx_flag_prepend("/analyze")

zmq_check_cxx_flag_prepend("/Zc:__cplusplus") # enables right reporting of __cplusplus, ref. https://devblogs.microsoft.com/cppblog/msvc-now-correctly-reports-__cplusplus/

# C++11/14/17-specific, but maybe possible via conditional defines
zmq_check_cxx_flag_prepend("/wd26440") # Function '...' can be declared 'noexcept'
zmq_check_cxx_flag_prepend("/wd26432") # If you define or delete any default operation in the type '...', define or
Expand Down Expand Up @@ -893,6 +895,7 @@ endif()
set(cxx-sources
precompiled.cpp
address.cpp
allocator_default.cpp
channel.cpp
client.cpp
clock.cpp
Expand Down Expand Up @@ -993,6 +996,7 @@ set(cxx-sources
zmtp_engine.cpp
# at least for VS, the header files must also be listed
address.hpp
allocator_default.hpp
array.hpp
atomic_counter.hpp
atomic_ptr.hpp
Expand Down
2 changes: 2 additions & 0 deletions Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ include_HEADERS = \
src_libzmq_la_SOURCES = \
src/address.cpp \
src/address.hpp \
src/allocator_default.cpp \
src/allocator_default.hpp \
src/array.hpp \
src/atomic_counter.hpp \
src/atomic_ptr.hpp \
Expand Down
25 changes: 25 additions & 0 deletions include/zmq.h
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,22 @@ ZMQ_EXPORT void *zmq_threadstart (zmq_thread_fn *func_, void *arg_);
ZMQ_EXPORT void zmq_threadclose (void *thread_);


struct zmq_allocator_t
{
// Allocate a chunk of memory of size len and return the pointer
void *(*allocate_fn) (void *allocator, size_t len);

// Deallocate the memory chunk pointed to by data_
void (*deallocate_fn) (void *allocator, void *data_);

// Return true if this is an allocator and alive, otherwise false
bool (*check_tag_fn) (void *allocator);

void (*destroy_fn) (void *allocator);

void *allocator;
};

/******************************************************************************/
/* These functions are DRAFT and disabled in stable releases, and subject to */
/* change at ANY time until declared stable. */
Expand Down Expand Up @@ -699,6 +715,15 @@ ZMQ_EXPORT int zmq_ctx_get_ext (void *context_,
void *optval_,
size_t *optvallen_);

// ZMQ-provided message-pool implementations. */
// default allocator using malloc/free
#define ZMQ_MSG_ALLOCATOR_DEFAULT 0

ZMQ_EXPORT void *zmq_msg_allocator_new (int type_);
ZMQ_EXPORT int zmq_msg_allocator_destroy (void **allocator_);
ZMQ_EXPORT int
zmq_msg_init_allocator (zmq_msg_t *msg_, size_t size_, void *allocator_);

/* DRAFT Socket methods. */
ZMQ_EXPORT int zmq_join (void *s, const char *group);
ZMQ_EXPORT int zmq_leave (void *s, const char *group);
Expand Down
16 changes: 16 additions & 0 deletions perf/remote_thr.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#include "platform.hpp"
#include "../include/zmq.h"
#include <stdio.h>
#include <stdlib.h>
Expand Down Expand Up @@ -67,6 +68,11 @@ int main (int argc, char *argv[])
return -1;
}

#if (defined ZMQ_BUILD_DRAFT_API)
// EXPERIMENTAL ALLOCATOR FOR MSG_T
void *allocator = zmq_msg_allocator_new (ZMQ_MSG_ALLOCATOR_DEFAULT);
#endif

s = zmq_socket (ctx, ZMQ_PUSH);
if (!s) {
printf ("error in zmq_socket: %s\n", zmq_strerror (errno));
Expand Down Expand Up @@ -105,7 +111,11 @@ int main (int argc, char *argv[])
}

for (i = 0; i != message_count; i++) {
#if (defined ZMQ_BUILD_DRAFT_API)
rc = zmq_msg_init_allocator (&msg, message_size, allocator);
#else
rc = zmq_msg_init_size (&msg, message_size);
#endif
if (rc != 0) {
printf ("error in zmq_msg_init_size: %s\n", zmq_strerror (errno));
return -1;
Expand Down Expand Up @@ -134,5 +144,11 @@ int main (int argc, char *argv[])
return -1;
}

#if (defined ZMQ_BUILD_DRAFT_API)
// IMPORTANT: destroy the allocator only after zmq_ctx_term() since otherwise
// some zmq_msg_t may still be "in fly"
zmq_msg_allocator_destroy (&allocator);
#endif

return 0;
}
60 changes: 60 additions & 0 deletions src/allocator_default.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
Copyright (c) 2019-2020 Contributors as noted in the AUTHORS file

This file is part of libzmq, the ZeroMQ core engine in C++.

libzmq is free software; you can redistribute it and/or modify it under
the terms of the GNU Lesser General Public License (LGPL) as published
by the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.

As a special exception, the Contributors give you permission to link
this library with independent modules to produce an executable,
regardless of the license terms of these independent modules, and to
copy and distribute the resulting executable under terms of your choice,
provided that you also meet, for each linked independent module, the
terms and conditions of the license of that module. An independent
module is a module which is not derived from or based on this library.
If you modify this library, you must extend this exception to your
version of the library.

libzmq is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
License for more details.

You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#include "precompiled.hpp"
#include "allocator_default.hpp"

#include <new>

zmq::allocator_default_t::allocator_default_t ()
{
_tag = 0xCAFEEBEB;
}


zmq::allocator_default_t::~allocator_default_t ()
{
// Mark this instance as dead
_tag = 0xdeadbeef;
}

void *zmq::allocator_default_t::allocate (size_t len_)
{
return operator new (len_, std::nothrow);
}

void zmq::allocator_default_t::deallocate (void *data_)
{
operator delete (data_);
}

bool zmq::allocator_default_t::check_tag () const
{
return _tag == 0xCAFEEBEB;
}
76 changes: 76 additions & 0 deletions src/allocator_default.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
Copyright (c) 2019-2020 Contributors as noted in the AUTHORS file

This file is part of libzmq, the ZeroMQ core engine in C++.

libzmq is free software; you can redistribute it and/or modify it under
the terms of the GNU Lesser General Public License (LGPL) as published
by the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.

As a special exception, the Contributors give you permission to link
this library with independent modules to produce an executable,
regardless of the license terms of these independent modules, and to
copy and distribute the resulting executable under terms of your choice,
provided that you also meet, for each linked independent module, the
terms and conditions of the license of that module. An independent
module is a module which is not derived from or based on this library.
If you modify this library, you must extend this exception to your
version of the library.

libzmq is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
License for more details.

You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#ifndef __ZMQ_I_ALLOCATOR_HPP_INCLUDED__
#define __ZMQ_I_ALLOCATOR_HPP_INCLUDED__

namespace zmq
{
class allocator_default_t
{
public:
allocator_default_t ();

~allocator_default_t ();

static void *allocate_fn (void *allocator_, size_t len_)
{
return static_cast<allocator_default_t *> (allocator_)->allocate (len_);
}

static void deallocate_fn (void *allocator_, void *data_)
{
return static_cast<allocator_default_t *> (allocator_)
->deallocate (data_);
}

static bool check_tag_fn (void *allocator_)
{
return static_cast<allocator_default_t *> (allocator_)->check_tag ();
}

static void destroy_fn (void *allocator_)
{
operator delete (static_cast<allocator_default_t *> (allocator_));
}

// allocate() typically gets called by the consumer thread: the user app thread(s)
void *allocate (size_t len_);

void deallocate (void *data_);

bool check_tag () const;

private:
// Used to check whether the object is a socket.
uint32_t _tag;
};
}

#endif
16 changes: 16 additions & 0 deletions src/ctx.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
#include "err.hpp"
#include "msg.hpp"
#include "random.hpp"
#include "allocator_default.hpp"

#ifdef ZMQ_HAVE_VMCI
#include <vmci_sockets.h>
Expand Down Expand Up @@ -311,6 +312,18 @@ int zmq::ctx_t::set (int option_, const void *optval_, size_t optvallen_)
}
break;

/*case ZMQ_MSG_ALLOCATOR: {
if (optvallen_ == sizeof (zmq::allocator_t)) {
const zmq::allocator_t *all =
static_cast<const zmq::allocator_t *> (optval_);
if (all->check_tag ()) {
_allocator = const_cast<zmq::allocator_t *> (all);
return 0;
}
}
break;
}*/

default: {
return thread_ctx_t::set (option_, optval_, optvallen_);
}
Expand Down Expand Up @@ -381,6 +394,9 @@ int zmq::ctx_t::get (int option_, void *optval_, const size_t *optvallen_)
return 0;
}
break;
/*
case ZMQ_MSG_ALLOCATOR: {
} break;*/

default: {
return thread_ctx_t::get (option_, optval_, optvallen_);
Expand Down
4 changes: 4 additions & 0 deletions src/ctx.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#include <string>
#include <stdarg.h>

//#include "allocator_default.hpp"
#include "mailbox.hpp"
#include "array.hpp"
#include "config.hpp"
Expand Down Expand Up @@ -221,6 +222,9 @@ class ctx_t ZMQ_FINAL : public thread_ctx_t
// Synchronisation of access to the list of inproc endpoints.
mutex_t _endpoints_sync;

// Allocator for messages
//allocator_t *_allocator;

// Maximum socket ID.
static atomic_counter_t max_socket_id;

Expand Down