Merge pull request #5494 from garazdawi/lukas/erts/fix-ets-read_concu…

…rrency-memory/OTP-17832

erts: Fix ets memory calculation for read_concurrency

erts/emulator/beam/erl_db.c

@@ -450,16 +450,18 @@ free_dbtable(void *vtb)
     erts_flxctr_add(&tb->common.counters,
                     ERTS_DB_TABLE_MEM_COUNTER_ID,
                     -((Sint)erts_flxctr_nr_of_allocated_bytes(&tb->common.counters)));
-    ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) ||
-           sizeof(DbTable) == DB_GET_APPROX_MEM_CONSUMED(tb));
-
-    ASSERT(is_immed(tb->common.heir_data));
 
     if (!DB_LOCK_FREE(tb)) {
+        ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(&tb->common.rwlock), 0);
         erts_rwmtx_destroy(&tb->common.rwlock);
         erts_mtx_destroy(&tb->common.fixlock);
     }
 
+    ASSERT(is_immed(tb->common.heir_data));
+
+    ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) ||
+           sizeof(DbTable) == DB_GET_APPROX_MEM_CONSUMED(tb));
+
     if (tb->common.btid)
         erts_bin_release(tb->common.btid);
 
@@ -644,6 +646,7 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock)
     if (!DB_LOCK_FREE(tb)) {
         erts_rwmtx_init_opt(&tb->common.rwlock, &rwmtx_opt, "db_tab",
                             tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
+        ERTS_DB_ALC_MEM_UPDATE_(tb, 0, erts_rwmtx_size(&tb->common.rwlock));
         erts_mtx_init(&tb->common.fixlock, "db_tab_fix",
                       tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
     }

erts/emulator/beam/erl_db_catree.c

@@ -1034,6 +1034,7 @@ static DbTableCATreeNode *create_base_node(DbTableCATree *tb,
                         "erl_db_catree_base_node",
                         NIL,
                         ERTS_LOCK_FLAGS_CATEGORY_DB);
+    ERTS_DB_ALC_MEM_UPDATE_((DbTable *) tb, 0, erts_rwmtx_size(&p->u.base.lock));
     BASE_NODE_STAT_SET(p, ((tb->common.status & DB_CATREE_FORCE_SPLIT)
                            ? INT_MAX : 0));
     p->u.base.is_valid = 1;
@@ -1092,7 +1093,7 @@ static void do_free_base_node(void* vptr)
 static void free_catree_base_node(DbTableCATree* tb, DbTableCATreeNode* p)
 {
     ASSERT(p->is_base_node);
-    ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof_base_node(), 0);
+    ERTS_DB_ALC_MEM_UPDATE_(tb, sizeof_base_node() + erts_rwmtx_size(&p->u.base.lock), 0);
     do_free_base_node(p);
 }
 
@@ -1334,11 +1335,13 @@ static void join_catree(DbTableCATree *tb,
                           thiz,
                           &thiz->u.base.free_item,
                           sizeof_base_node());
+    ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(&thiz->u.base.lock), 0);
     erts_schedule_db_free(&tb->common,
                           do_free_base_node,
                           neighbor,
                           &neighbor->u.base.free_item,
                           sizeof_base_node());
+    ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(&neighbor->u.base.lock), 0);
 }
 
 static void split_catree(DbTableCATree *tb,
@@ -1383,6 +1386,7 @@ static void split_catree(DbTableCATree *tb,
                               base,
                               &base->u.base.free_item,
                               sizeof_base_node());
+        ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(&base->u.base.lock), 0);
     }
 }
 

erts/emulator/beam/erl_db_hash.c

@@ -375,8 +375,9 @@ void erl_db_hash_adapt_number_of_locks(DbTable* tb) {
                                                               (DbTable *) tb,
                                                               sizeof(DbTableHashFineLockSlot) * tbl->nlocks);
         for (i=0; i < tbl->nlocks; i++) {
-            erts_rwmtx_init_opt(&tbl->locks[i].u.lck_ctr.lck, &rwmtx_opt,
+            erts_rwmtx_init_opt(GET_LOCK(tbl, i), &rwmtx_opt,
                                 "db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
+            ERTS_DB_ALC_MEM_UPDATE_(tb, 0, erts_rwmtx_size(GET_LOCK(tbl,i)));
             tbl->locks[i].u.lck_ctr.nitems =
                 get_lock_nitems_form_prev_lock_array(i, tbl->nlocks, old_number_of_locks, old_locks);
             tbl->locks[i].u.lck_ctr.lck_stat = 0;
@@ -403,6 +404,7 @@ void erl_db_hash_adapt_number_of_locks(DbTable* tb) {
         erts_atomic_set_nob(&tbl->lock_array_resize_state, DB_HASH_LOCK_ARRAY_RESIZE_STATUS_NORMAL);
         erts_rwmtx_rwunlock(&tb->common.rwlock);
         for (i = 0; i < old_number_of_locks; i++) {
+            ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(&old_locks[i].u.lck_ctr.lck), 0);
             erts_rwmtx_destroy(&old_locks[i].u.lck_ctr.lck);
         }
         erts_db_free(ERTS_ALC_T_DB_SEG, tb, old_locks, sizeof(DbTableHashFineLockSlot) * old_number_of_locks);
@@ -1037,8 +1039,10 @@ int db_create_hash(Process *p, DbTable *tbl)
                                                              (DbTable *) tb,
                                                              sizeof(DbTableHashFineLockSlot) * tb->nlocks);
 	for (i=0; i<tb->nlocks; ++i) {
-            erts_rwmtx_init_opt(&tb->locks[i].u.lck_ctr.lck, &rwmtx_opt,
+            erts_rwmtx_init_opt(
+                GET_LOCK(tb,i), &rwmtx_opt,
                 "db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
+            ERTS_DB_ALC_MEM_UPDATE_(tb, 0, erts_rwmtx_size(GET_LOCK(tb,i)));
             tb->locks[i].u.lck_ctr.nitems = 0;
             tb->locks[i].u.lck_ctr.lck_stat = 0;
 	}
@@ -3065,14 +3069,15 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds)
     if (tb->locks != NULL) {
 	int i;
 	for (i=0; i<tb->nlocks; ++i) {
-	    erts_rwmtx_destroy(GET_LOCK(tb,i)); 
+            ERTS_DB_ALC_MEM_UPDATE_(tb, erts_rwmtx_size(GET_LOCK(tb,i)), 0);
+	    erts_rwmtx_destroy(GET_LOCK(tb,i));
 	}
 	erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb,
 		     (void*)tb->locks, tb->nlocks * sizeof(DbTableHashFineLockSlot));
 	tb->locks = NULL;
     }
     ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) ||
-           ((sizeof(DbTable) +
+           ((sizeof(DbTable) + (!DB_LOCK_FREE(tb) ? erts_rwmtx_size(&tb->common.rwlock) : 0) +
              erts_flxctr_nr_of_allocated_bytes(&tb->common.counters)) ==
             erts_flxctr_read_approx(&tb->common.counters,
                                     ERTS_DB_TABLE_MEM_COUNTER_ID)));

erts/emulator/beam/erl_db_tree.c

@@ -2467,9 +2467,11 @@ static SWord db_free_table_continue_tree(DbTable *tbl, SWord reds)
 	ASSERT(erts_flxctr_is_snapshot_ongoing(&tb->common.counters) ||
                ((APPROX_MEM_CONSUMED(tb)
                  == (sizeof(DbTable) +
+                     (!DB_LOCK_FREE(tb) ? erts_rwmtx_size(&tb->common.rwlock) : 0) +
                      erts_flxctr_nr_of_allocated_bytes(&tb->common.counters))) ||
                 (APPROX_MEM_CONSUMED(tb)
                  == (sizeof(DbTable) +
+                     (!DB_LOCK_FREE(tb) ? erts_rwmtx_size(&tb->common.rwlock) : 0) +
                      sizeof(DbFixation) +
                      erts_flxctr_nr_of_allocated_bytes(&tb->common.counters)))));
     }

erts/emulator/beam/erl_init.c

@@ -2523,13 +2523,21 @@ erl_start(int argc, char **argv)
 __decl_noreturn void erts_thr_fatal_error(int err, const char *what)
 {
     const char *errstr = err ? strerror(err) : NULL;
-    erts_fprintf(stderr,
-		 "Failed to %s: %s%s(%d)\n",
-		 what,
-		 errstr ? errstr : "",
-		 errstr ? " " : "",
-		 err);
-    abort();
+    if (err == ENOMEM) {
+        erts_exit(ERTS_DUMP_EXIT, "Failed to %s: %s%s(%d)\n",
+                  what,
+                  errstr ? errstr : "",
+                  errstr ? " " : "",
+                  err);
+    } else {
+        erts_fprintf(stderr,
+                     "Failed to %s: %s%s(%d)\n",
+                     what,
+                     errstr ? errstr : "",
+                     errstr ? " " : "",
+                     err);
+        abort();
+    }
 }
 
 

erts/emulator/beam/erl_threads.h

@@ -445,6 +445,7 @@ ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
                                      char *name,
                                      Eterm extra,
                                      erts_lock_flags_t flags);
+ERTS_GLB_INLINE size_t erts_rwmtx_size(erts_rwmtx_t *rwmtx);
 ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
 #ifdef ERTS_ENABLE_LOCK_POSITION
 ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, const char *file, unsigned int line);
@@ -1883,6 +1884,11 @@ erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name, Eterm extra,
     erts_rwmtx_init_opt(rwmtx, NULL, name, extra, flags);
 }
 
+ERTS_GLB_INLINE size_t
+erts_rwmtx_size(erts_rwmtx_t *rwmtx) {
+    return ethr_rwmutex_size(&rwmtx->rwmtx);
+}
+
 ERTS_GLB_INLINE void
 erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
 {

erts/include/internal/ethr_mutex.h

@@ -341,6 +341,7 @@ struct ethr_rwmutex_ {
 int ethr_rwmutex_set_reader_group(int);
 int ethr_rwmutex_init_opt(ethr_rwmutex *, ethr_rwmutex_opt *);
 int ethr_rwmutex_init(ethr_rwmutex *);
+size_t ethr_rwmutex_size(ethr_rwmutex *);
 int ethr_rwmutex_destroy(ethr_rwmutex *);
 #if defined(ETHR_USE_OWN_RWMTX_IMPL__) \
     || !defined(ETHR_TRY_INLINE_FUNCS) \

erts/lib_src/common/ethr_mutex.c

@@ -2720,6 +2720,28 @@ ethr_rwmutex_init(ethr_rwmutex *rwmtx)
     return ethr_rwmutex_init_opt(rwmtx, NULL);
 }
 
+size_t
+ethr_rwmutex_size(ethr_rwmutex *rwmtx) {
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+        return sizeof(ethr_rwmtx_readers_array__) * (reader_groups_array_size + 1);
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+        return sizeof(ethr_rwmtx_readers_array__) * (main_threads_array_size + 1);
+    default:
+        return 0;
+    }
+}
+
 int
 ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
 {
@@ -3098,6 +3120,11 @@ ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
     return ethr_rwmutex_init(rwmtx);
 }
 
+size_t
+ethr_rwmutex_size(ethr_rwmutex *rwmtx) {
+    return 0;
+}
+
 int
 ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
 {

lib/stdlib/test/ets_SUITE.erl

@@ -180,7 +180,8 @@ all() ->
      test_delete_table_while_size_snapshot,
      test_decentralized_counters_setting,
      ms_excessive_nesting,
-     error_info].
+     error_info
+    ].
 
 
 groups() ->
@@ -1923,7 +1924,7 @@ t_select_replace_next_bug(Config) when is_list(Config) ->
 
 
 %% OTP-17379
-t_select_pam_stack_overflow_bug(Config) ->
+t_select_pam_stack_overflow_bug(_Config) ->
     T = ets:new(k, []),
     ets:insert(T,[{x,17}]),
     [{x,18}] = ets:select(T,[{{x,17}, [], [{{{element,1,'$_'},{const,18}}}]}]),
@@ -3042,8 +3043,9 @@ write_concurrency(Config) when is_list(Config) ->
     YesTreeMem = ets:info(Yes7,memory),
     YesYesTreeMem = ets:info(Yes14,memory),
     NoTreeMem = ets:info(No4,memory),
-    io:format("YesMem=~p NoHashMem=~p NoTreeMem=~p YesTreeMem=~p\n",[YesMem,NoHashMem,
-                                                                     NoTreeMem,YesTreeMem]),
+
+    io:format("YesMem=~p NoHashMem=~p NoTreeMem=~p YesTreeMem=~p YesYesTreeMem=~p\n",
+              [YesMem,NoHashMem,NoTreeMem,YesTreeMem,YesYesTreeMem]),
 
     YesMem = ets:info(Yes2,memory),
     YesMem = ets:info(Yes3,memory),
@@ -3068,14 +3070,24 @@ write_concurrency(Config) when is_list(Config) ->
 
     true = YesMem > YesTreeMem,
 
-    case erlang:system_info(schedulers) > 1 of
-        true ->
+    case erlang:system_info(schedulers) of
+        1 ->
+            YesMem = NoHashMem,
+            YesTreeMem = NoTreeMem,
+            YesYesTreeMem = YesTreeMem;
+        NoSchedulers ->
             true = YesMem > NoHashMem,
             true = YesMem > NoTreeMem,
-            true = YesTreeMem < NoTreeMem,
-            true = YesYesTreeMem > YesTreeMem;
-        _ ->
-            one_scheduler_only
+
+            %% The memory of ordered_set with write concurrency is
+            %% smaller than without write concurrency on systems with
+            %% few schedulers.
+            if NoSchedulers > 6 ->
+                    true = YesTreeMem >= NoTreeMem;
+               true ->
+                    true = YesTreeMem < NoTreeMem
+            end,
+            true = YesYesTreeMem > YesTreeMem
     end,
 
     {'EXIT',{badarg,_}} = (catch ets_new(foo,[public,{write_concurrency,foo}])),
@@ -4384,12 +4396,28 @@ exit_many_many_tables_owner(Config) when is_list(Config) ->
     repeat_for_opts(fun(Opts) -> exit_many_many_tables_owner_do(Opts,FEData,Config) end).
 
 exit_many_many_tables_owner_do(Opts,FEData,Config) ->
-    verify_rescheduling_exit(Config, FEData, [named_table | Opts], true, 200, 5),
-    verify_rescheduling_exit(Config, FEData, Opts, false, 200, 5),
+
+    E = ets_new(tmp,Opts),
+    FEData(fun(Data) -> ets:insert(E, Data) end),
+    Mem = ets:info(E,memory) * erlang:system_info(wordsize),
+    ets:delete(E),
+
+    ct:log("Memory per table: ~p bytes",[Mem]),
+
+    Tables =
+        case erlang:system_info(wordsize) of
+            8 ->
+                200;
+            4 ->
+                lists:min([200,2_000_000_000 div (Mem * 5)])
+        end,
+
+    verify_rescheduling_exit(Config, FEData, [named_table | Opts], true, Tables, 5),
+    verify_rescheduling_exit(Config, FEData, Opts, false, Tables, 5),
     wait_for_test_procs(),
     EtsMem = etsmem(),
-    verify_rescheduling_exit(Config, FEData, Opts, true, 200, 5),
-    verify_rescheduling_exit(Config, FEData, [named_table | Opts], false, 200, 5),
+    verify_rescheduling_exit(Config, FEData, Opts, true, Tables, 5),
+    verify_rescheduling_exit(Config, FEData, [named_table | Opts], false, Tables, 5),
     verify_etsmem(EtsMem).