/
Main.kt
157 lines (150 loc) · 8.29 KB
/
Main.kt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
package net.corda.loadtest
import com.typesafe.config.ConfigFactory
import com.typesafe.config.ConfigParseOptions
import net.corda.loadtest.tests.StabilityTest
import net.corda.loadtest.tests.crossCashTest
import net.corda.loadtest.tests.selfIssueTest
import net.corda.nodeapi.internal.config.parseAs
import java.io.File
/**
* This is how load testing works:
*
* Setup:
* The load test assumes that there is an active SSH Agent running with an added identity it can use to connect to all
* remote nodes. To run intellij with the ssh-agent:
* $ ssh-agent $SHELL
* $ ssh-add
* $ exec idea.sh # 'exec' is required so we can detach from the surrounding shell without quiting the agent.
*
* In order to make our life easier we download the remote node certificates to localhost and use those directly. The
* reasoning being that if somebody has SSH access to the nodes they have access to the certificates anyway.
* TODO Still, is this ok? Perhaps we should include a warning in the docs.
* We then tunnel the remote Artemis messaging ports to localhost and establish an RPC link.
*
* Running the tests:
* The [LoadTest] API assumes that each load test will keep generating some kind of work to push to the nodes. It also
* assumes that the nodes' behaviour will be somewhat predictable, which is tracked in a state. For example say we
* want to self issue Cash on each node(see [SelfIssueTest]). We can predict that if we submit an Issue request of
* 100 USD and 200 USD we should end up with 300 USD issued by the node. Each load test can define its own such
* invariant and should check for it in [LoadTest.gatherRemoteState].
* We then simply keep generating pieces of work and check that the invariants hold(see [LoadTest.RunParameters] on
* how to configure the generation).
* In addition for each test a number of disruptions may be specified that make the nodes' jobs harder. Each
* disruption is basically an infinite loop of wait->mess something up->repeat. Invariants should hold under these
* conditions as well.
*
* Configuration:
* The load test will look for configuration in location provided by the program argument, or the configuration can be
* provided via system properties using vm arguments, e.g. -Dloadtest.nodeHosts.0="host" see [LoadTestConfiguration] for
* list of configurable properties.
*
* Diagnostic:
* TODO currently the diagnostic is quite poor, all we can say is that the predicted state is different from the real
* one, or that some piece of work failed to execute in some state. Logs need to be checked manually.
*
* TODO: Any load test that involves intra-node transactions will currently fail because the node re-picks the same states
* if tx creation requests arrive quickly, which result in notarisation failures. So this needs figuring out before we
* can run the load tests properly.
*/
fun main(args: Array<String>) {
val customConfig = if (args.isNotEmpty()) {
ConfigFactory.parseFile(File(args[0]), ConfigParseOptions.defaults().setAllowMissing(false))
} else {
// This allow us to provide some configurations via teamcity.
ConfigFactory.parseProperties(System.getProperties()).getConfig("loadtest")
}
val defaultConfig = ConfigFactory.parseResources("loadtest-reference.conf", ConfigParseOptions.defaults().setAllowMissing(false))
val resolvedConfig = customConfig.withFallback(defaultConfig).resolve()
val loadTestConfiguration = resolvedConfig.parseAs<LoadTestConfiguration>()
if (loadTestConfiguration.nodeHosts.isEmpty()) {
throw IllegalArgumentException("Please specify at least one node host")
}
when (loadTestConfiguration.mode) {
TestMode.LOAD_TEST -> runLoadTest(loadTestConfiguration)
TestMode.STABILITY_TEST -> runStabilityTest(loadTestConfiguration)
}
}
@Suppress("MagicNumber") // test constants
private fun runLoadTest(loadTestConfiguration: LoadTestConfiguration) {
runLoadTests(loadTestConfiguration, listOf(
selfIssueTest to LoadTest.RunParameters(
parallelism = 100,
generateCount = 10000,
clearDatabaseBeforeRun = false,
executionFrequency = 1000,
gatherFrequency = 1000,
disruptionPatterns = listOf(
listOf(), // no disruptions
listOf(
DisruptionSpec(
disruption = hang(2000L..4000L),
nodeFilter = { true },
noDisruptionWindowMs = 500L..1000L
),
DisruptionSpec(
disruption = kill,
nodeFilter = isNotary, // TODO Fix it with network map, isNetworkMap.or(isNotary).
noDisruptionWindowMs = 10000L..20000L // Takes a while for it to restart
),
// DOCS START 1
DisruptionSpec(
disruption = strainCpu(parallelism = 4, durationSeconds = 10),
nodeFilter = { true },
noDisruptionWindowMs = 5000L..10000L
)
// DOCS END 1
)
)
),
crossCashTest to LoadTest.RunParameters(
parallelism = 4,
generateCount = 2000,
clearDatabaseBeforeRun = false,
executionFrequency = 1000,
gatherFrequency = 10,
disruptionPatterns = listOf(
listOf(),
listOf(
DisruptionSpec(
disruption = hang(2000L..4000L),
nodeFilter = { true },
noDisruptionWindowMs = 500L..1000L
),
DisruptionSpec(
disruption = kill,
nodeFilter = isNotary, // TODO Fix it with network map, isNetworkMap.or(isNotary).
noDisruptionWindowMs = 10000L..20000L // Takes a while for it to restart
),
DisruptionSpec(
disruption = strainCpu(parallelism = 4, durationSeconds = 10),
nodeFilter = { true },
noDisruptionWindowMs = 5000L..10000L
)
)
)
)
))
}
@Suppress("MagicNumber") // test constants
private fun runStabilityTest(loadTestConfiguration: LoadTestConfiguration) {
runLoadTests(loadTestConfiguration, listOf(
// Self issue cash. This is a pre test step to make sure vault have enough cash to work with.
StabilityTest.selfIssueTest(100) to LoadTest.RunParameters(
parallelism = loadTestConfiguration.parallelism,
generateCount = 1000,
clearDatabaseBeforeRun = false,
executionFrequency = 50,
gatherFrequency = 100,
disruptionPatterns = listOf(listOf()) // no disruptions
),
// Send cash to a random party or exit cash, commands are generated randomly.
StabilityTest.crossCashTest(100) to LoadTest.RunParameters(
parallelism = loadTestConfiguration.parallelism,
generateCount = loadTestConfiguration.generateCount,
clearDatabaseBeforeRun = false,
executionFrequency = loadTestConfiguration.executionFrequency,
gatherFrequency = 100,
disruptionPatterns = listOf(listOf())
)
))
}