Skip to content

Commit

Permalink
Use unique port for each server unit test.
Browse files Browse the repository at this point in the history 
If the same port is reused too quickly bind may fail with this error:

FileOpenError: unable to bind socket: [98] Address already in use

We specify SO_REUSEADDR when creating the socket but apparently this is not always enough if the port is reused very rapidly.

Fix this (hopefully) by using a unique port for each test that needs one. This does in theory limit the number of tests that can use ports, but we allow 768 per test, whereas the test that uses the most ports is common/io-tls with 4.
  • Loading branch information
@dwsteele
dwsteele committed Nov 30, 2023
1 parent a147327 commit 7ce0f5a
Show file tree
Hide file tree
Showing 9 changed files with 136 additions and 110 deletions.
28 changes: 18 additions & 10 deletions test/src/common/harnessServer.c
View file
Expand Up @@ -44,6 +44,14 @@ Constants
***********************************************************************************************************************************/
#define HRN_SERVER_HOST "tls.test.pgbackrest.org"

/***********************************************************************************************************************************
Local data
***********************************************************************************************************************************/
struct HrnServerLocal
{
unsigned int portOffsetNext; // Next server port offset to be returned
} hrnServerLocal;

/**********************************************************************************************************************************/
void
hrnServerInit(void)
Expand Down Expand Up @@ -230,21 +238,18 @@ hrnServerScriptSleep(IoWrite *write, TimeMSec sleepMs)

/**********************************************************************************************************************************/
void
hrnServerRun(IoRead *read, HrnServerProtocol protocol, HrnServerRunParam param)
hrnServerRun(IoRead *const read, const HrnServerProtocol protocol, const unsigned int port, HrnServerRunParam param)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(IO_READ, read);
FUNCTION_HARNESS_PARAM(ENUM, protocol);
FUNCTION_HARNESS_PARAM(UINT, param.port);
FUNCTION_HARNESS_PARAM(UINT, port);
FUNCTION_HARNESS_PARAM(STRING, param.certificate);
FUNCTION_HARNESS_PARAM(STRING, param.key);
FUNCTION_HARNESS_END();

ASSERT(read != NULL);

// Set port to index 0 if not specified
if (param.port == 0)
param.port = hrnServerPort(0);
ASSERT(port >= HRN_SERVER_PORT_MIN);

// Initialize ssl and create a context
IoServer *tlsServer = NULL;
Expand All @@ -263,7 +268,7 @@ hrnServerRun(IoRead *read, HrnServerProtocol protocol, HrnServerRunParam param)
tlsServer = tlsServerNew(STRDEF(HRN_SERVER_HOST), param.ca, param.key, param.certificate, 5000);
}

IoServer *socketServer = sckServerNew(STRDEF("localhost"), param.port, 5000);
IoServer *socketServer = sckServerNew(STRDEF("localhost"), port, 5000);

// Loop until no more commands
IoSession *serverSession = NULL;
Expand Down Expand Up @@ -377,9 +382,12 @@ hrnServerHost(void)

/**********************************************************************************************************************************/
unsigned int
hrnServerPort(unsigned int portIdx)
hrnServerPortNext(void)
{
ASSERT(portIdx < HRN_SERVER_PORT_MAX);
CHECK(AssertError, testIdx() < 32, "test max exceeds limit of 32");
CHECK(
AssertError, hrnServerLocal.portOffsetNext < HRN_SERVER_PORT_MAX,
"requested port exceeds limit of " STRINGIFY(HRN_SERVER_PORT_MAX));

return 44443 + (HRN_SERVER_PORT_MAX * testIdx()) + portIdx;
return HRN_SERVER_PORT_MIN + (HRN_SERVER_PORT_MAX * testIdx()) + hrnServerLocal.portOffsetNext++;
}
22 changes: 14 additions & 8 deletions test/src/common/harnessServer.h
View file
Expand Up @@ -23,9 +23,16 @@ typedef enum
} HrnServerProtocol;

/***********************************************************************************************************************************
Maximum number of ports allowed for each test
Port constants
***********************************************************************************************************************************/
#define HRN_SERVER_PORT_MAX 4
// Maximum number of ports allowed for each test
#define HRN_SERVER_PORT_MAX 768

// Bogus port to be used where the port does not matter or must fail
#define HRN_SERVER_PORT_BOGUS 34342

// Minimum port to be assigned to a test
#define HRN_SERVER_PORT_MIN (HRN_SERVER_PORT_BOGUS + 1)

/***********************************************************************************************************************************
Path and prefix for test certificates
Expand All @@ -47,16 +54,15 @@ void hrnServerInit(void);
typedef struct HrnServerRunParam
{
VAR_PARAM_HEADER;
unsigned int port; // Server port, defaults to hrnServerPort(0)
const String *ca; // TLS CA store when protocol = hrnServerProtocolTls
const String *certificate; // TLS certificate when protocol = hrnServerProtocolTls
const String *key; // TLS key when protocol = hrnServerProtocolTls
} HrnServerRunParam;

#define hrnServerRunP(read, protocol, ...) \
hrnServerRun(read, protocol, (HrnServerRunParam){VAR_PARAM_INIT, __VA_ARGS__})
#define hrnServerRunP(read, protocol, port, ...) \
hrnServerRun(read, protocol, port, (HrnServerRunParam){VAR_PARAM_INIT, __VA_ARGS__})

void hrnServerRun(IoRead *read, HrnServerProtocol protocol, HrnServerRunParam param);
void hrnServerRun(IoRead *read, HrnServerProtocol protocol, unsigned int port, HrnServerRunParam param);

// Begin/end server script
IoWrite *hrnServerScriptBegin(IoWrite *write);
Expand Down Expand Up @@ -89,7 +95,7 @@ Getters/Setters
const String *hrnServerHost(void);

// Port to use for testing. This will be unique for each test running in parallel to avoid conflicts. A range is allocated to each
// test so multiple ports can be requested.
unsigned int hrnServerPort(unsigned int portIdx);
// test so multiple ports can be requested and no port is ever reused to eliminate rebinding issues.
unsigned int hrnServerPortNext(void);

#endif
16 changes: 10 additions & 6 deletions test/src/module/command/serverTest.c
View file
Expand Up @@ -29,6 +29,8 @@ testRun(void)

HRN_FORK_BEGIN(.timeout = 15000)
{
const unsigned int testPort = hrnServerPortNext();

HRN_FORK_CHILD_BEGIN(.prefix = "client repo")
{
StringList *argList = strLstNew();
Expand All @@ -41,7 +43,7 @@ testRun(void)
#endif
hrnCfgArgRawZ(argList, cfgOptRepoHostCertFile, HRN_SERVER_CLIENT_CERT);
hrnCfgArgRawZ(argList, cfgOptRepoHostKeyFile, HRN_SERVER_CLIENT_KEY);
hrnCfgArgRawFmt(argList, cfgOptRepoHostPort, "%u", hrnServerPort(0));
hrnCfgArgRawFmt(argList, cfgOptRepoHostPort, "%u", testPort);
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);

Expand Down Expand Up @@ -87,7 +89,7 @@ testRun(void)
#endif
hrnCfgArgRawZ(argList, cfgOptPgHostCertFile, HRN_SERVER_CLIENT_CERT);
hrnCfgArgRawZ(argList, cfgOptPgHostKeyFile, HRN_SERVER_CLIENT_KEY);
hrnCfgArgRawFmt(argList, cfgOptPgHostPort, "%u", hrnServerPort(0));
hrnCfgArgRawFmt(argList, cfgOptPgHostPort, "%u", testPort);
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
hrnCfgArgRawZ(argList, cfgOptProcess, "1");
HRN_CFG_LOAD(cfgCmdBackup, argList, .role = cfgCmdRoleLocal);
Expand Down Expand Up @@ -129,7 +131,7 @@ testRun(void)

StringList *argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptConfig, TEST_PATH "/pgbackrest.conf");
hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", hrnServerPort(0));
hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", testPort);
hrnCfgArgRawZ(argList, cfgOptLogLevelStderr, CFGOPTVAL_ARCHIVE_MODE_OFF_Z);
HRN_CFG_LOAD(cfgCmdServer, argList);

Expand Down Expand Up @@ -206,12 +208,14 @@ testRun(void)

HRN_FORK_BEGIN(.timeout = 15000)
{
const unsigned int testPort = hrnServerPortNext();

HRN_FORK_CHILD_BEGIN(.prefix = "client")
{
TEST_TITLE("ping localhost");

argList = strLstNew();
hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", hrnServerPort(0));
hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", testPort);
HRN_CFG_LOAD(cfgCmdServerPing, argList);

TEST_RESULT_VOID(cmdServerPing(), "ping");
Expand All @@ -220,7 +224,7 @@ testRun(void)
TEST_TITLE("ping 12.0.0.1");

argList = strLstNew();
hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", hrnServerPort(0));
hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", testPort);
strLstAddZ(argList, "127.0.0.1");
HRN_CFG_LOAD(cfgCmdServerPing, argList);

Expand All @@ -242,7 +246,7 @@ testRun(void)
hrnCfgArgRawZ(argList, cfgOptTlsServerCertFile, HRN_SERVER_CERT);
hrnCfgArgRawZ(argList, cfgOptTlsServerKeyFile, HRN_SERVER_KEY);
hrnCfgArgRawZ(argList, cfgOptTlsServerAuth, "bogus=*");
hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", hrnServerPort(0));
hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", testPort);
HRN_CFG_LOAD(cfgCmdServer, argList);

// Init exit signal handlers
Expand Down
19 changes: 11 additions & 8 deletions test/src/module/common/ioHttpTest.c
View file
Expand Up @@ -307,21 +307,22 @@ testRun(void)
char logBuf[STACK_TRACE_PARAM_MAX];
HttpClient *client = NULL;

TEST_ASSIGN(client, httpClientNew(sckClientNew(STRDEF("localhost"), hrnServerPort(0), 500, 500), 500), "new client");
TEST_ASSIGN(client, httpClientNew(sckClientNew(STRDEF("localhost"), HRN_SERVER_PORT_BOGUS, 500, 500), 500), "new client");

TEST_ERROR_FMT(
httpRequestResponse(httpRequestNewP(client, STRDEF("GET"), STRDEF("/")), false), HostConnectError,
"unable to connect to 'localhost:%u (127.0.0.1)': [111] Connection refused\n"
"unable to connect to 'localhost:34342 (127.0.0.1)': [111] Connection refused\n"
"[RETRY DETAIL OMITTED]\n"
"[RETRY DETAIL OMITTED]",
hrnServerPort(0));
"[RETRY DETAIL OMITTED]");

HRN_FORK_BEGIN()
{
const unsigned int testPort = hrnServerPortNext();

HRN_FORK_CHILD_BEGIN(.prefix = "test server", .timeout = 5000)
{
// Start HTTP test server
TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolSocket), "http server run");
TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolSocket, testPort), "http server");
}
HRN_FORK_CHILD_END();

Expand All @@ -334,7 +335,7 @@ testRun(void)

ioBufferSizeSet(35);

TEST_ASSIGN(client, httpClientNew(sckClientNew(hrnServerHost(), hrnServerPort(0), 5000, 5000), 5000), "new client");
TEST_ASSIGN(client, httpClientNew(sckClientNew(hrnServerHost(), testPort, 5000, 5000), 5000), "new client");

// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("no output from server");
Expand Down Expand Up @@ -684,13 +685,15 @@ testRun(void)

HRN_FORK_BEGIN()
{
const unsigned int testPort = hrnServerPortNext();

// Start HTTPS test server
HRN_FORK_CHILD_BEGIN(.prefix = "test server", .timeout = 5000)
{
// Set buffer size large enough for server to read expect messages
ioBufferSizeSet(65536);

TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolTls), "http server run");
TEST_RESULT_VOID(hrnServerRunP(HRN_FORK_CHILD_READ(), hrnServerProtocolTls, testPort), "http server");
}
HRN_FORK_CHILD_END();

Expand All @@ -707,7 +710,7 @@ testRun(void)
client,
httpClientNew(
tlsClientNewP(
sckClientNew(hrnServerHost(), hrnServerPort(0), 5000, 5000), hrnServerHost(), 0, 0, TEST_IN_CONTAINER),
sckClientNew(hrnServerHost(), testPort, 5000, 5000), hrnServerHost(), 0, 0, TEST_IN_CONTAINER),
5000),
"new client");

Expand Down

0 comments on commit 7ce0f5a

Please sign in to comment.