Connection Pool Issue

[dubrovine]

MongoDB 4.4.4 Community
3 nodes

I have couple server

• 1 - 5k requests to mongodb
• 2 - about 15\20k requests to mongo

when is use application on the first server everything is fine. working without any problems
but when I deploy to the second server - application is giving me errors like this

2021-09-29T22:15:02.139619+03:00 [error] <0.22446.1>: auth_handler unknown error -- type: exit reason: {timeout,{gen_server,call,[<0.3723.0>,{checkout,#Ref<0.3569742552.2113929218.56829>,true},5000]}} stacktrace [{gen_server,call,3,[{file,"gen_server.erl"},{line,223}]},{poolboy,checkout,3,[{file,"/*****/_build/default/lib/poolboy/src/poolboy.erl"},{line,63}]},{poolboy,transaction,3,[{file,"/*****/_build/default/lib/poolboy/src/poolboy.erl"},{line,82}]},{mongodb_interface,select,2,[{file,"/*****/src/mongodb/mongodb_interface.erl"},{line,83}]},{session,get,2,[{file,"/*****/src/core/session.erl"},{line,20}]},{auth_proc,process,4,[{file,"/*****/src/service_processes/auth_proc.erl"},{line,108}]},{auth_handler,init,2,[{file,"/*****/src/handlers/auth_handler.erl"},{line,29}]},{cowboy_handler,execute,2,[{file,"/*****/_build/default/lib/cowboy/src/cowboy_handler.erl"},{line,37}]}]

2021-09-29T22:40:03.769130+03:00 [error] <0.17875.0>: auth_handler unknown error -- type: exit reason: {timeout,{gen_server,call,[<0.2241.0>,get_pool,5000]}} stacktrace [{gen_server,call,3,[{file,"gen_server.erl"},{line,223}]},{mc_topology,get_pool,2,[{file,"/*****/_build/default/lib/mongodb/src/mongoc/mc_topology.erl"},{line,109}]},{mongoc,transaction_query,4,[{file,"/*****/_build/default/lib/mongodb/src/api/mongoc.erl"},{line,87}]},{mongodb_interface,select,2,[{file,"/*****/src/mongodb/mongodb_interface.erl"},{line,83}]},{session,get,2,[{file,"/*****/src/core/session.erl"},{line,29}]},{auth_proc,process,4,[{file,"/*****/src/service_processes/auth_proc.erl"},{line,108}]},{auth_handler,init,2,[{file,"/*****/src/handlers/auth_handler.erl"},{line,29}]},{cowboy_handler,execute,2,[{file,"/*****/_build/default/lib/cowboy/src/cowboy_handler.erl"},{line,37}]}]

as I can see from error message

mc_server:get_pool(Pid, 5000) couldn't get pool in time

I tried to increase connection pool settings to:

"mongodb": { "rs": "sessions", "hosts": ["XX.XX.XX.XX:27017", "XX.XX.XX.XX:27017"], "database": "sessions", "login": "xxx", "password": "xxx", "pool_size_master": 300, "max_overflow_master": 100, "pool_size_slave": 200 },

but had no luck

could someone point me to the right direction

[dubrovine]

I guess the problem is here
"max_overflow_master": 100

mongoc:status(Atom) - indicating that overflow is growing

when I set "max_overflow_master" to 0 on the test server - it shows a better performance

[dubrovine]

I am still having the same issue

always getting timeout on mongoc:get_pool
also having the same timeout with Redis connection timeout gen_server:call

seems like some sort of bottleneck

here is the mongo db connection part

start_pool(Config) ->
    ReplicaSet      = data:get(<<"rs">>, Config),
    Hosts           = data:get(<<"hosts">>, Config),
    Database        = data:get(<<"database">>, Config),
    Login           = data:get(<<"login">>, Config),
    Password        = data:get(<<"password">>, Config),
    PoolSizeMaster  = data:get(<<"pool_size_master">>, Config),
    Credentials     = case Login of undefined -> []; _ -> [{login, Login}, {password, Password}, {auth_source, Database}] end,
    {ok, _Pid} = mongoc:connect(
        {rs, ReplicaSet, Hosts},
        [
            {name, ?MONGO_MASTER}, 
            {register, ?MONGO_MASTER},
            {max_overflow, 0},
            {pool_size, PoolSizeMaster},
            {rp_mode, primary}
        ],
        [{database, Database} | Credentials]
    ),
    PoolSizeSlave  = data:get(<<"pool_size_slave">>, Config),
    {ok, _} = mongoc:connect(
        {rs, ReplicaSet, Hosts},
        [
            {name, ?MONGO_SLAVE}, 
            {register, ?MONGO_SLAVE}, 
            {pool_size, PoolSizeSlave}, 
            {max_overflow, 0},
            {rp_mode, secondaryPreferred}
        ],
        [{database, Database}, {r_mode, slave_ok} | Credentials]
    ),
    ok.

what am I doing wrong ?
thanx