Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,12 @@ public CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode) {
return this;
}

@Override
public CommonConfig setPipeMemoryManagementEnabled(boolean pipeMemoryManagementEnabled) {
setProperty("pipe_memory_management_enabled", String.valueOf(pipeMemoryManagementEnabled));
return this;
}

@Override
public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
setProperty("pipe_enable_memory_checked", String.valueOf(isPipeEnableMemoryCheck));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -466,6 +466,13 @@ public CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode) {
return this;
}

@Override
public CommonConfig setPipeMemoryManagementEnabled(boolean pipeMemoryManagementEnabled) {
dnConfig.setPipeMemoryManagementEnabled(pipeMemoryManagementEnabled);
cnConfig.setPipeMemoryManagementEnabled(pipeMemoryManagementEnabled);
return this;
}

@Override
public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
dnConfig.setIsPipeEnableMemoryCheck(isPipeEnableMemoryCheck);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,11 @@ public CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode) {
return this;
}

@Override
public CommonConfig setPipeMemoryManagementEnabled(boolean pipeMemoryManagementEnabled) {
return this;
}

@Override
public CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck) {
return this;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,8 @@ CommonConfig setEnableAutoLeaderBalanceForIoTConsensus(

CommonConfig setSchemaRegionPerDataNode(double schemaRegionPerDataNode);

CommonConfig setPipeMemoryManagementEnabled(boolean pipeMemoryManagementEnabled);

CommonConfig setIsPipeEnableMemoryCheck(boolean isPipeEnableMemoryCheck);

CommonConfig setPipeAirGapReceiverEnabled(boolean isPipeAirGapReceiverEnabled);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ protected void setupConfig() {
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setEnforceStrongPassword(false)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
Expand All @@ -58,6 +59,7 @@ protected void setupConfig() {
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setEnforceStrongPassword(false)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

// 10 min, assert that the operations will not time out
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ public void setUp() {
.setTimestampPrecision("ms")
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
Expand All @@ -78,6 +79,7 @@ public void setUp() {
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false)
.setSchemaReplicationFactor(3)
.setDataReplicationFactor(2);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,10 @@ private void innerSetUp(
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(dataRegionConsensus)
.setSchemaReplicationFactor(schemaRegionReplicationFactor)
.setDataReplicationFactor(dataRegionReplicationFactor);
.setDataReplicationFactor(dataRegionReplicationFactor)
.setDnConnectionTimeoutMs(600000)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
Expand All @@ -85,11 +88,10 @@ private void innerSetUp(
.setSchemaRegionConsensusProtocolClass(schemaRegionConsensus)
.setDataRegionConsensusProtocolClass(dataRegionConsensus)
.setSchemaReplicationFactor(schemaRegionReplicationFactor)
.setDataReplicationFactor(dataRegionReplicationFactor);

// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
.setDataReplicationFactor(dataRegionReplicationFactor)
.setDnConnectionTimeoutMs(600000)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

senderEnv.initClusterEnvironment(configNodesNum, dataNodesNum);
receiverEnv.initClusterEnvironment(configNodesNum, dataNodesNum);
Expand Down Expand Up @@ -170,7 +172,10 @@ public void testPipeOnBothSenderAndReceiver() throws Exception {
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setSchemaReplicationFactor(3)
.setDataReplicationFactor(2);
.setDataReplicationFactor(2)
.setDnConnectionTimeoutMs(600000)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
Expand All @@ -179,11 +184,10 @@ public void testPipeOnBothSenderAndReceiver() throws Exception {
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setSchemaReplicationFactor(1)
.setDataReplicationFactor(1);

// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
.setDataReplicationFactor(1)
.setDnConnectionTimeoutMs(600000)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

senderEnv.initClusterEnvironment(3, 3);
receiverEnv.initClusterEnvironment(1, 1);
Expand Down Expand Up @@ -379,7 +383,10 @@ private void doTestUseNodeUrls(String connectorName) throws Exception {
.setDataReplicationFactor(1)
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
.setEnableCrossSpaceCompaction(false);
.setEnableCrossSpaceCompaction(false)
.setDnConnectionTimeoutMs(600000)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
Expand All @@ -389,11 +396,10 @@ private void doTestUseNodeUrls(String connectorName) throws Exception {
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setSchemaReplicationFactor(3)
.setDataReplicationFactor(2);

// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
.setDataReplicationFactor(2)
.setDnConnectionTimeoutMs(600000)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

senderEnv.initClusterEnvironment(1, 1);
receiverEnv.initClusterEnvironment(1, 3);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,19 +71,21 @@ public void setUp() {
// Disable sender compaction for tsfile determination in loose range test
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
.setEnableCrossSpaceCompaction(false);
.setEnableCrossSpaceCompaction(false)
.setDnConnectionTimeoutMs(600000)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH");

receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);

// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDnConnectionTimeoutMs(600000)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

senderEnv.initClusterEnvironment();
receiverEnv.initClusterEnvironment();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,10 @@ public void setUp() {
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setDnConnectionTimeoutMs(600000)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

receiverEnv
.getConfig()
Expand All @@ -84,11 +87,10 @@ public void setUp() {
.setSchemaReplicationFactor(3)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS);

// 10 min, assert that the operations will not time out
senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
receiverEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000);
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setDnConnectionTimeoutMs(600000)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

senderEnv.initClusterEnvironment(3, 3, 180);
receiverEnv.initClusterEnvironment(3, 3, 180);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,15 @@ public void setUp() {
.setTimestampPrecision("ms")
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

// 10 min, assert that the operations will not time out
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ private void innerSetUp(
.setDataRegionConsensusProtocolClass(dataRegionConsensus)
.setSchemaReplicationFactor(schemaRegionReplicationFactor)
.setDataReplicationFactor(dataRegionReplicationFactor)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
Expand All @@ -87,6 +88,7 @@ private void innerSetUp(
.setDataRegionConsensusProtocolClass(dataRegionConsensus)
.setSchemaReplicationFactor(schemaRegionReplicationFactor)
.setDataReplicationFactor(dataRegionReplicationFactor)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

// 10 min, assert that the operations will not time out
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ public void setUp() {
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
.setEnableCrossSpaceCompaction(false)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH");
receiverEnv
Expand All @@ -81,6 +82,7 @@ public void setUp() {
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
receiverEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ public void setUp() {
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
Expand All @@ -71,6 +72,7 @@ public void setUp() {
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

// 10 min, assert that the operations will not time out
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ public void setUp() {
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH");

Expand All @@ -91,6 +92,7 @@ public void setUp() {
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

// 10 min, assert that the operations will not time out
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ public void setUp() {
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setEnforceStrongPassword(false)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

receiverEnv
Expand All @@ -78,6 +79,7 @@ public void setUp() {
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setEnforceStrongPassword(false)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

// 10 min, assert that the operations will not time out
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ public void setUp() {
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

receiverEnv
Expand All @@ -78,6 +79,7 @@ public void setUp() {
.setPipeAirGapReceiverEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

// 10 min, assert that the operations will not time out
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,15 @@ public void setUp() {
.setEnableSeqSpaceCompaction(false)
.setEnableUnseqSpaceCompaction(false)
.setEnableCrossSpaceCompaction(false)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
receiverEnv
.getConfig()
.getCommonConfig()
.setAutoCreateSchemaEnabled(true)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

// 10 min, assert that the operations will not time out
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ protected void setupConfig() {
.setAutoCreateSchemaEnabled(false)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
senderEnv.getConfig().getDataNodeConfig().setDataNodeMemoryProportion("3:3:1:1:3:1");

Expand All @@ -66,6 +67,7 @@ protected void setupConfig() {
.setAutoCreateSchemaEnabled(false)
.setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

// 10 min, assert that the operations will not time out
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public void setUp() throws Exception {
.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS)
.setSchemaReplicationFactor(3)
.setDataReplicationFactor(2)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

EnvFactory.getEnv().initClusterEnvironment(3, 3);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public void setUp() throws Exception {
.getConfig()
.getCommonConfig()
.setSubscriptionEnabled(true)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);

EnvFactory.getEnv().initClusterEnvironment();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ protected void setUpConfig() {
sender
.getConfig()
.getCommonConfig()
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false)
.setSubscriptionPrefetchTsFileBatchMaxDelayInMs(500)
.setSubscriptionPrefetchTsFileBatchMaxSizeInBytes(32 * 1024);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public static void setUp() throws Exception {
.getConfig()
.getCommonConfig()
.setSubscriptionEnabled(true)
.setPipeMemoryManagementEnabled(false)
.setIsPipeEnableMemoryCheck(false);
EnvFactory.getEnv().initClusterEnvironment();

Expand Down
Loading
Loading