Skip to content

Commit

Permalink
Obvious memory reductions (netdata#15204)
Browse files Browse the repository at this point in the history 
* remove rd->update_every

* reduce amount of memory for RRDDIM

* reorgnize rrddim->db entries

* optimize rrdset and statsd

* optimize dictionaries

* RW_SPINLOCK for dictionaries

* fix codeql warning

* rw_spinlock improvements

* remove obsolete assertion

* fix crash on health_alarm_log_process()

* use RW_SPINLOCK for AVL trees

* add RW_SPINLOCK read/write trylock

* pgc and mrg now use rw_spinlocks; cache line optimizations for mrg

* thread tag of dbegnine init

* append created datafile, lockless

* make DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE friendly for lockless use

* thread cancelability in spinlocks; optimize thread cancelability management

* introduce a JudyL to index datafiles and use it during queries to quickly find the relevant files

* use the last timestamp of each journal file for indexing

* when the previous cannot be found, start from the beginning

* add more stats to PDC to trace routing easier

* rename spinlock functions

* fix for spinlock renames

* revert statsd socket statistics to size_t

* turn fatal into internal_fatal()

* show candidates always

* show connected status and connection attempts
  • Loading branch information
@ktsaou
ktsaou committed Jun 19, 2023
1 parent 0b4f820 commit 43c749b
Show file tree
Hide file tree
Showing 55 changed files with 1,183 additions and 884 deletions.
4 changes: 2 additions & 2 deletions collectors/freebsd.plugin/freebsd_sysctl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -459,9 +459,9 @@ int do_dev_cpu_temperature(int update_every, usec_t dt) {
static RRDDIM **rd_pcpu_temperature;

if (unlikely(number_of_cpus != old_number_of_cpus)) {
rd_pcpu_temperature = reallocz(rd_pcpu_temperature, sizeof(RRDDIM) * number_of_cpus);
rd_pcpu_temperature = reallocz(rd_pcpu_temperature, sizeof(RRDDIM *) * number_of_cpus);
if (unlikely(number_of_cpus > old_number_of_cpus))
memset(&rd_pcpu_temperature[old_number_of_cpus], 0, sizeof(RRDDIM) * (number_of_cpus - old_number_of_cpus));
memset(&rd_pcpu_temperature[old_number_of_cpus], 0, sizeof(RRDDIM *) * (number_of_cpus - old_number_of_cpus));
}

if (unlikely(!st)) {
Expand Down
24 changes: 12 additions & 12 deletions collectors/plugins.d/plugins_d.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,28 +22,28 @@ inline size_t pluginsd_initialize_plugin_directories()
}

static inline void plugin_set_disabled(struct plugind *cd) {
netdata_spinlock_lock(&cd->unsafe.spinlock);
spinlock_lock(&cd->unsafe.spinlock);
cd->unsafe.enabled = false;
netdata_spinlock_unlock(&cd->unsafe.spinlock);
spinlock_unlock(&cd->unsafe.spinlock);
}

bool plugin_is_enabled(struct plugind *cd) {
netdata_spinlock_lock(&cd->unsafe.spinlock);
spinlock_lock(&cd->unsafe.spinlock);
bool ret = cd->unsafe.enabled;
netdata_spinlock_unlock(&cd->unsafe.spinlock);
spinlock_unlock(&cd->unsafe.spinlock);
return ret;
}

static inline void plugin_set_running(struct plugind *cd) {
netdata_spinlock_lock(&cd->unsafe.spinlock);
spinlock_lock(&cd->unsafe.spinlock);
cd->unsafe.running = true;
netdata_spinlock_unlock(&cd->unsafe.spinlock);
spinlock_unlock(&cd->unsafe.spinlock);
}

static inline bool plugin_is_running(struct plugind *cd) {
netdata_spinlock_lock(&cd->unsafe.spinlock);
spinlock_lock(&cd->unsafe.spinlock);
bool ret = cd->unsafe.running;
netdata_spinlock_unlock(&cd->unsafe.spinlock);
spinlock_unlock(&cd->unsafe.spinlock);
return ret;
}

Expand All @@ -53,15 +53,15 @@ static void pluginsd_worker_thread_cleanup(void *arg)

worker_unregister();

netdata_spinlock_lock(&cd->unsafe.spinlock);
spinlock_lock(&cd->unsafe.spinlock);

cd->unsafe.running = false;
cd->unsafe.thread = 0;

pid_t pid = cd->unsafe.pid;
cd->unsafe.pid = 0;

netdata_spinlock_unlock(&cd->unsafe.spinlock);
spinlock_unlock(&cd->unsafe.spinlock);

if (pid) {
siginfo_t info;
Expand Down Expand Up @@ -190,14 +190,14 @@ static void pluginsd_main_cleanup(void *data) {

struct plugind *cd;
for (cd = pluginsd_root; cd; cd = cd->next) {
netdata_spinlock_lock(&cd->unsafe.spinlock);
spinlock_lock(&cd->unsafe.spinlock);
if (cd->unsafe.enabled && cd->unsafe.running && cd->unsafe.thread != 0) {
info("PLUGINSD: 'host:%s', stopping plugin thread: %s",
rrdhost_hostname(cd->host), cd->id);

netdata_thread_cancel(cd->unsafe.thread);
}
netdata_spinlock_unlock(&cd->unsafe.spinlock);
spinlock_unlock(&cd->unsafe.spinlock);
}

info("PLUGINSD: cleanup completed.");
Expand Down
32 changes: 16 additions & 16 deletions collectors/plugins.d/pluginsd_parser.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ static ssize_t send_to_plugin(const char *txt, void *data) {
return 0;

errno = 0;
netdata_spinlock_lock(&parser->writer.spinlock);
spinlock_lock(&parser->writer.spinlock);
ssize_t bytes = -1;

#ifdef ENABLE_HTTPS
Expand All @@ -24,7 +24,7 @@ static ssize_t send_to_plugin(const char *txt, void *data) {
else
error("PLUGINSD: cannot send command (SSL)");

netdata_spinlock_unlock(&parser->writer.spinlock);
spinlock_unlock(&parser->writer.spinlock);
return bytes;
}
#endif
Expand All @@ -39,7 +39,7 @@ static ssize_t send_to_plugin(const char *txt, void *data) {
else
fflush(parser->fp_output);

netdata_spinlock_unlock(&parser->writer.spinlock);
spinlock_unlock(&parser->writer.spinlock);
return bytes;
}

Expand All @@ -52,18 +52,18 @@ static ssize_t send_to_plugin(const char *txt, void *data) {
sent = write(parser->fd, &txt[bytes], total - bytes);
if(sent <= 0) {
error("PLUGINSD: cannot send command (fd)");
netdata_spinlock_unlock(&parser->writer.spinlock);
spinlock_unlock(&parser->writer.spinlock);
return -3;
}
bytes += sent;
}
while(bytes < total);

netdata_spinlock_unlock(&parser->writer.spinlock);
spinlock_unlock(&parser->writer.spinlock);
return (int)bytes;
}

netdata_spinlock_unlock(&parser->writer.spinlock);
spinlock_unlock(&parser->writer.spinlock);
error("PLUGINSD: cannot send command (no output socket/pipe/file given to plugins.d parser)");
return -4;
}
Expand Down Expand Up @@ -93,15 +93,15 @@ static inline RRDSET *pluginsd_get_chart_from_parent(void *user) {
static inline void pluginsd_lock_rrdset_data_collection(void *user) {
PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
if(u->st && !u->v2.locked_data_collection) {
netdata_spinlock_lock(&u->st->data_collection_lock);
spinlock_lock(&u->st->data_collection_lock);
u->v2.locked_data_collection = true;
}
}

static inline bool pluginsd_unlock_rrdset_data_collection(void *user) {
PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
if(u->st && u->v2.locked_data_collection) {
netdata_spinlock_unlock(&u->st->data_collection_lock);
spinlock_unlock(&u->st->data_collection_lock);
u->v2.locked_data_collection = false;
return true;
}
Expand Down Expand Up @@ -1233,9 +1233,9 @@ PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, void *user) {
st->counter_done++;

// these are only needed for db mode RAM, SAVE, MAP, ALLOC
st->current_entry++;
if(st->current_entry >= st->entries)
st->current_entry -= st->entries;
st->db.current_entry++;
if(st->db.current_entry >= st->db.entries)
st->db.current_entry -= st->db.entries;

((PARSER_USER_OBJECT *) user)->replay.start_time = start_time;
((PARSER_USER_OBJECT *) user)->replay.end_time = end_time;
Expand Down Expand Up @@ -1660,9 +1660,9 @@ PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {
st->counter_done++;

// these are only needed for db mode RAM, SAVE, MAP, ALLOC
st->current_entry++;
if(st->current_entry >= st->entries)
st->current_entry -= st->entries;
st->db.current_entry++;
if(st->db.current_entry >= st->db.entries)
st->db.current_entry -= st->db.entries;

timing_step(TIMING_STEP_BEGIN2_STORE);

Expand Down Expand Up @@ -1774,7 +1774,7 @@ PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {
rd->last_stored_value = value;
rd->last_calculated_value = value;
rd->collections_counter++;
rd->updated = true;
rrddim_set_updated(rd);

timing_step(TIMING_STEP_SET2_STORE);

Expand Down Expand Up @@ -1831,7 +1831,7 @@ PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_
rrddim_foreach_read(rd, st) {
rd->calculated_value = 0;
rd->collected_value = 0;
rd->updated = false;
rrddim_clear_updated(rd);
}
rrddim_foreach_done(rd);

Expand Down
Loading

0 comments on commit 43c749b

Please sign in to comment.