From c11e0497ae165708829f08598286db117593acb3 Mon Sep 17 00:00:00 2001 From: t-bast Date: Wed, 29 Apr 2020 18:05:20 +0200 Subject: [PATCH 1/2] Disable DB backup in tests We've been witnessing random test suites freezes (since ages). We've observed that when these freezes happen, there are usually a lot of "too many open files" errors raised by the OS. The backup handler is a likely culprit as the IntegrationSpec is running multiple nodes and exchanging HTLCs at a fast rate. At least it won't hurt disabling it in tests, and will speed up the test suite. --- eclair-core/src/main/resources/reference.conf | 1 + .../main/scala/fr/acinq/eclair/Setup.scala | 26 ++++++++++++------- .../eclair/integration/IntegrationSpec.scala | 1 + 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/eclair-core/src/main/resources/reference.conf b/eclair-core/src/main/resources/reference.conf index 9da3d0ff71..8a356cf400 100644 --- a/eclair-core/src/main/resources/reference.conf +++ b/eclair-core/src/main/resources/reference.conf @@ -15,6 +15,7 @@ eclair { password = "" // password for basic auth, must be non empty if json-rpc api is enabled } + enable-db-backup = true // enable the automatic sqlite db backup; do not change this unless you know what you are doing // override this with a script/exe that will be called everytime a new database backup has been created # backup-notify-script = "/absolute/path/to/script.sh" diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala b/eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala index 79b68afd1a..5c99d3fdcb 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala @@ -66,6 +66,7 @@ import scala.concurrent.duration._ * @param datadir directory where eclair-core will write/read its data. * @param overrideDefaults use this parameter to programmatically override the node configuration . * @param seed_opt optional seed, if set eclair will use it instead of generating one and won't create a seed.dat file. + * @param db optional databases to use, if not set eclair will create the necessary databases */ class Setup(datadir: File, overrideDefaults: Config = ConfigFactory.empty(), @@ -117,8 +118,10 @@ class Setup(datadir: File, val feeratesPerKw = new AtomicReference[FeeratesPerKw](null) val feeEstimator = new FeeEstimator { + // @formatter:off override def getFeeratePerKb(target: Int): Long = feeratesPerKB.get().feePerBlock(target) override def getFeeratePerKw(target: Int): Long = feeratesPerKw.get().feePerBlock(target) + // @formatter:on } val nodeParams = NodeParams.makeNodeParams(config, keyManager, initTor(), database, blockCount, feeEstimator) @@ -153,7 +156,7 @@ class Setup(datadir: File, blocks = (json \ "blocks").extract[Long] headers = (json \ "headers").extract[Long] chainHash <- bitcoinClient.invoke("getblockhash", 0).map(_.extract[String]).map(s => ByteVector32.fromValidHex(s)).map(_.reverse) - bitcoinVersion <- bitcoinClient.invoke("getnetworkinfo").map(json => (json \ "version")).map(_.extract[Int]) + bitcoinVersion <- bitcoinClient.invoke("getnetworkinfo").map(json => json \ "version").map(_.extract[Int]) unspentAddresses <- bitcoinClient.invoke("listunspent").collect { case JArray(values) => values .filter(value => (value \ "spendable").extract[Boolean]) @@ -270,17 +273,20 @@ class Setup(datadir: File, implicit val timeout = Timeout(30 seconds) new ElectrumEclairWallet(electrumWallet, nodeParams.chainHash) } - _ = wallet.getFinalAddress.map { - case address => logger.info(s"initial wallet address=$address") - } + _ = wallet.getFinalAddress.map(address => logger.info(s"initial wallet address=$address")) // do not change the name of this actor. it is used in the configuration to specify a custom bounded mailbox - backupHandler = system.actorOf(SimpleSupervisor.props( - BackupHandler.props( - nodeParams.db, - new File(chaindir, "eclair.sqlite.bak"), - if (config.hasPath("backup-notify-script")) Some(config.getString("backup-notify-script")) else None - ), "backuphandler", SupervisorStrategy.Resume)) + backupHandler = if (config.getBoolean("enable-db-backup")) { + system.actorOf(SimpleSupervisor.props( + BackupHandler.props( + nodeParams.db, + new File(chaindir, "eclair.sqlite.bak"), + if (config.hasPath("backup-notify-script")) Some(config.getString("backup-notify-script")) else None + ), "backuphandler", SupervisorStrategy.Resume)) + } else { + logger.warn("database backup is disabled") + system.deadLetters + } audit = system.actorOf(SimpleSupervisor.props(Auditor.props(nodeParams), "auditor", SupervisorStrategy.Resume)) register = system.actorOf(SimpleSupervisor.props(Props(new Register), "register", SupervisorStrategy.Resume)) commandBuffer = system.actorOf(SimpleSupervisor.props(Props(new CommandBuffer(nodeParams, register)), "command-buffer", SupervisorStrategy.Resume)) diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/integration/IntegrationSpec.scala b/eclair-core/src/test/scala/fr/acinq/eclair/integration/IntegrationSpec.scala index 160bcf4aee..b6404fcce9 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/integration/IntegrationSpec.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/integration/IntegrationSpec.scala @@ -91,6 +91,7 @@ class IntegrationSpec extends TestKit(ActorSystem("test")) with BitcoindService val commonConfig = ConfigFactory.parseMap(Map( "eclair.chain" -> "regtest", + "eclair.enable-db-backup" -> false, "eclair.server.public-ips.1" -> "127.0.0.1", "eclair.bitcoind.port" -> bitcoindPort, "eclair.bitcoind.rpcport" -> bitcoindRpcPort, From fa2bab35b7b8c100e813f235bb478af002c0d2f3 Mon Sep 17 00:00:00 2001 From: t-bast Date: Wed, 29 Apr 2020 18:06:26 +0200 Subject: [PATCH 2/2] Configure CI file limits --- .semaphore/semaphore.yml | 3 +++ .travis.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index dd93f52060..ff64d487fe 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -21,6 +21,9 @@ blocks: jobs: - name: Build & Test commands: + # configure file limits + - ulimit -S -n 1024000 + - echo "fs.file-max = 1024000" | sudo tee -a /etc/sysctl.conf - sem-version java 11 - checkout - cache restore maven diff --git a/.travis.yml b/.travis.yml index 9a9400203d..eb39066dbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ before_install: - export M2_HOME=$PWD/apache-maven-3.6.3 - export PATH=$M2_HOME/bin:$PATH script: + - echo "fs.file-max = 1024000" | sudo tee -a /etc/sysctl.conf - mvn scoverage:report cache: directories: