-
Notifications
You must be signed in to change notification settings - Fork 50
/
MapTestRunner.java
279 lines (247 loc) · 10.3 KB
/
MapTestRunner.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
package tests;
import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.results.RunResult;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.RunnerException;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
import tests.maptests.IMapTest;
import tests.maptests.ITestSet;
import tests.maptests.article_examples.*;
import tests.maptests.identity_object.*;
import tests.maptests.object.*;
import tests.maptests.object_prim.*;
import tests.maptests.prim_object.*;
import tests.maptests.primitive.*;
import java.util.*;
import java.util.concurrent.TimeUnit;
/*
Master plan:
1) add following test cases:
1.1) 2 added, 1 (previously inserted) removed - until reached the map size
*/
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@State(Scope.Thread)
public class MapTestRunner {
private static final boolean BILLION_TEST = false;
private static final int START_SIZE = BILLION_TEST ? 1000 * 1000 * 1000 : 10 * 1000;
private static final int TOTAL_SIZE = BILLION_TEST ? 1000 * 1000 * 1000 : 100 * 1000 * 1000;
private final static int[] MAP_SIZES;
static {
MAP_SIZES = new int[(int) (Math.log10( TOTAL_SIZE ) - Math.log10( START_SIZE ) + 1)];
int start = START_SIZE;
int p = 0;
while ( start <= TOTAL_SIZE )
{
MAP_SIZES[ p++ ] = start;
start *= 10;
}
}
private static final float FILL_FACTOR = 0.5f;
//share of unsuccessful get operations (approx)
//increase this value significantly (Integer.MAX_VALUE is a good candidate) to revert to the original "always successful get" tests
private static final int ONE_FAIL_OUT_OF = 2;
private static final Class<?>[] TESTS_ARTICLE = {
IntIntMap1Test.class,
IntIntMap2Test.class,
IntIntMap3Test.class,
IntIntMap4Test.class,
IntIntMap4aTest.class,
};
private static final Class<?>[] TESTS_PRIMITIVE = {
FastUtilMapTest.class,
GsMutableMapTest.class,
KolobokeMutableMapTest.class, //+
HppcMapTest.class,
TroveMapTest.class, //+
AgronaMapTest.class,
};
private static final Class<?>[] TESTS_WRAPPER = {
FastUtilObjMapTest.class,
KolobokeMutableObjTest.class, //+
KolobokeNotNullKeyObjTest.class,
KolobokeHashCodeMixingObjTest.class,
HppcObjMapTest.class,
GsObjMapTest.class,
JdkMapTest.class, //+
JdkMapTestDifferentCapacity.class, //+
TroveObjMapTest.class, //+
ObjObjMapTest.class, //
AgronaObjMapTest.class,
};
private static final Class<?>[] TESTS_PRIMITIVE_WRAPPER = {
FastUtilIntObjectMapTest.class,
GsIntObjectMapTest.class,
KolobokeIntObjectMapTest.class, //+
HppcIntObjectMapTest.class,
TroveIntObjectMapTest.class, //+
AgronaIntObjectMapTest.class,
};
private static final Class<?>[] TESTS_WRAPPER_PRIMITIVE = {
FastUtilObjectIntMapTest.class,
GsObjectIntMapTest.class,
KolobokeObjectIntMapTest.class, //+
HppcObjectIntMapTest.class,
TroveObjectIntMapTest.class, //+
AgronaObjectIntMapTest.class,
};
private static final Class<?>[] TESTS_IDENTITY = {
FastUtilRef2ObjectMapTest.class,
GsIdentityMapTest.class,
KolobokeIdentityMapTest.class,
HppcIdentityMapTest.class,
JDKIdentityMapTest.class,
TroveIdentityMapTest.class,
};
public static void main(String[] args) throws RunnerException, InstantiationException, IllegalAccessException
{
final LinkedHashMap<String, String> res = new LinkedHashMap<>(3);
res.put( "get", runTestSet( "get" ) );
res.put( "put", runTestSet( "put" ) );
res.put( "remove", runTestSet( "remove" ) );
for ( final Map.Entry<String, String> entry : res.entrySet() )
{
System.out.println( "Results for '" + entry.getKey() + "':" );
System.out.println( entry.getValue() );
System.out.println();
}
}
private static String runTestSet(final String testSetName) throws RunnerException, InstantiationException, IllegalAccessException
{
final List<Class<?>> tests = new ArrayList<>();
tests.addAll( Arrays.asList( TESTS_ARTICLE ) );
tests.addAll( Arrays.asList( TESTS_PRIMITIVE ) );
tests.addAll( Arrays.asList( TESTS_WRAPPER ) );
tests.addAll( Arrays.asList( TESTS_PRIMITIVE_WRAPPER ) );
tests.addAll( Arrays.asList( TESTS_WRAPPER_PRIMITIVE ) );
tests.addAll( Arrays.asList( TESTS_IDENTITY ) );
//first level: test class, second level - map size
final Map<String, Map<Integer, String>> results = new HashMap<>();
if ( BILLION_TEST )
{ //JMH does not feel well on these sizes
testBillion( tests );
return "";
}
//pick map size first - we need to generate a set of keys to be used in all tests
for (final int mapSize : MAP_SIZES) {
//run tests one after another
for ( final Class<?> testClass : tests ) {
Options opt = new OptionsBuilder()
.include(".*" + MapTestRunner.class.getSimpleName() + ".*")
.forks(1)
.mode(Mode.SingleShotTime)
.warmupBatchSize(TOTAL_SIZE / mapSize)
.warmupIterations(10)
.measurementBatchSize(TOTAL_SIZE / mapSize)
.measurementIterations(8)
.jvmArgsAppend("-Xmx30G")
.param("m_mapSize", Integer.toString(mapSize))
.param("m_className", testClass.getCanonicalName())
.param("m_testType", testSetName)
//.verbosity(VerboseMode.SILENT)
.shouldFailOnError(true)
.build();
Collection<RunResult> res = new Runner(opt).run();
for ( RunResult rr : res )
{
System.out.println( testClass.getCanonicalName() + " (" + mapSize + ") = " + rr.getAggregatedResult().getPrimaryResult().getScore() );
Map<Integer, String> forClass = results.computeIfAbsent(testClass.getCanonicalName(), k -> new HashMap<>(4));
forClass.put(mapSize, Integer.toString((int) rr.getAggregatedResult().getPrimaryResult().getScore()) );
}
if ( res.isEmpty() ) {
Map<Integer, String> forClass = results.computeIfAbsent(testClass.getCanonicalName(), k -> new HashMap<>(4));
forClass.put(mapSize, "-1");
}
}
}
final String res = formatResults(results, MAP_SIZES, tests);
System.out.println( "Results for test type = " + testSetName + ":\n" + res);
return res;
}
private static void testBillion( final List<Class<?>> tests ) throws IllegalAccessException, InstantiationException {
final int mapSize = 1000 * 1000 * 1000;
for ( final Class<?> klass : tests )
{
System.gc();
final IMapTest obj = (IMapTest) klass.newInstance();
System.out.println( "Prior to setup for " + klass.getName() );
obj.setup(KeyGenerator.getKeys(mapSize), FILL_FACTOR, ONE_FAIL_OUT_OF);
System.out.println( "After setup for " + klass.getName() );
final long start = System.currentTimeMillis();
obj.test();
final long time = System.currentTimeMillis() - start;
System.out.println( klass.getName() + " : time = " + ( time / 1000.0 ) + " sec");
}
}
private static String formatResults( final Map<String, Map<Integer, String>> results, final int[] mapSizes, final List<Class<?>> tests )
{
final StringBuilder sb = new StringBuilder( 2048 );
//format results
//first line - map sizes, should be sorted
for ( final int size : mapSizes )
sb.append( "," ).append( size );
sb.append( "\n" );
//following lines - tests in the definition order
for ( final Class<?> test : tests )
{
final Map<Integer, String> res = results.get( test.getCanonicalName() );
sb.append( test.getName() );
for ( final int size : mapSizes )
sb.append( ",\"" ).append( res.get( size ) ).append( "\"" );
sb.append( "\n" );
}
return sb.toString();
}
public static class KeyGenerator
{
public static int s_mapSize;
public static int[] s_keys;
public static int[] getKeys( final int mapSize )
{
if ( mapSize == s_mapSize )
return s_keys;
s_mapSize = mapSize;
s_keys = null; //should be done separately so we don't keep 2 arrays in memory
s_keys = new int[ mapSize ];
final Random r = new Random( 1234 );
for ( int i = 0; i < mapSize; ++i )
s_keys[ i ] = r.nextInt();
return s_keys;
}
}
@Param("1")
public int m_mapSize;
@Param("dummy")
public String m_className;
@Param( {"get", "put", "remove"} )
public String m_testType;
private IMapTest m_impl;
@Setup
public void setup()
{
try {
final ITestSet testSet = (ITestSet) Class.forName( m_className ).newInstance();
switch ( m_testType )
{
case "get":
m_impl = testSet.getTest();
break;
case "put":
m_impl = testSet.putTest();
break;
case "remove":
m_impl = testSet.removeTest();
break;
}
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
e.printStackTrace();
}
//share the same keys for all tests with the same map size
m_impl.setup( KeyGenerator.getKeys( m_mapSize ), FILL_FACTOR, ONE_FAIL_OUT_OF );
}
@Benchmark
public int testRandom()
{
return m_impl.test();
}
}