Skip to content

Commit

Permalink
New Logging
Browse files Browse the repository at this point in the history
  • Loading branch information
@tzaeschke
tzaeschke committed Jul 11, 2018
1 parent 6070fd8 commit 1115346
Show file tree
Hide file tree
Showing 4 changed files with 354 additions and 55 deletions.
84 changes: 77 additions & 7 deletions src/main/java/ch/ethz/globis/tinspin/TestManager.java
View file
Expand Up @@ -23,18 +23,80 @@ public static void main(String[] args) {
long t1 = System.currentTimeMillis();
try {

int[] sizesQuery = {1, 10, 100, 1000, 10000};
double[] sizesData = new double[]{0.5, 1, 2.5, 5, 10};//, 25, 50};
int[] sizesQuery = {1};//, 10, 100, 1000};//, 10000};
double[] sizesData = new double[]{0.5, 1, 2.5, 5, 10, 25, 50};

testSeriesWQS(TST.CLUSTER, IDX.PHC2, 3, 5.0, 1, sizesQuery);


testSeriesWQS(TST.CLUSTER, IDX.PHC, 3, 5.0, 1, sizesQuery);
// testSeriesWQS(TST.CLUSTER, IDX.PHC2, 3, 5.0, 1, sizesQuery);
testSeriesWQS(TST.CLUSTER, IDX.PHC_IPP, 3, 5.0, 1, sizesQuery);
// testSeriesWQS(TST.CLUSTER, IDX.RSZ, 3, 5.0, 1, sizesQuery);
// testSeriesWQS(TST.CLUSTER, IDX.QT0Z, 3, 5.0, 1, sizesQuery);
// testSeriesWQS(TST.CLUSTER, IDX.QT2Z, 3, 5.0, 1, sizesQuery);
testSeriesWQS(TST.CUBE, IDX.PHC, 3, 1.0, 1, sizesQuery);
// testSeriesWQS(TST.CUBE, IDX.PHC2, 3, 1.0, 1, sizesQuery);
testSeriesWQS(TST.CUBE, IDX.PHC_IPP, 3, 1.0, 1, sizesQuery);
testSeriesWQS(TST.OSM, IDX.PHC, 2, 1.0, 1, sizesQuery);
testSeriesWQS(TST.OSM, IDX.PHC2, 2, 1.0, 1, sizesQuery);
testSeriesWQS(TST.OSM, IDX.PHC_IPP, 1, 5.0, 1, sizesQuery);
testSeriesWQSR(TST.CLUSTER, IDX.PHC, 3, 5.0, 1, sizesQuery);
// testSeriesWQSR(TST.CLUSTER, IDX.PHC2, 3, 5.0, 1, sizesQuery);
testSeriesWQSR(TST.CLUSTER, IDX.PHC_IPP, 3, 5.0, 1, sizesQuery);
testSeriesWQSR(TST.CUBE, IDX.PHC, 3, 1.0, 1, sizesQuery);
// testSeriesWQSR(TST.CUBE, IDX.PHC2, 3, 1.0, 1, sizesQuery);
testSeriesWQSR(TST.CUBE, IDX.PHC_IPP, 3, 1.0, 1, sizesQuery);
testSeriesWQSR(TST.OSM, IDX.PHC, 2, 1.0, 1, sizesQuery);
testSeriesWQSR(TST.OSM, IDX.PHC2, 2, 1.0, 1, sizesQuery);
testSeriesWQSR(TST.OSM, IDX.PHC_IPP, 2, 1.0, 1, sizesQuery);

if (true) return;

// testP(TST.CUBE, IDX.RSS, 30, 1000000, 1.0);
// testP(TST.CUBE, IDX.RSS, 40, 1000000, 1.0);
// log.writeLogFileForTestSeries("dimsP");
//

testP(TST.CLUSTER, IDX.PHC2, 8, 1_000_000, 5.0);
testP(TST.CLUSTER, IDX.PHC2, 8, 1_000_000, 5.0);


suiteDims(TST.CLUSTER, IDX.PHC2, 1, 5.0);
suiteDims(TST.CUBE, IDX.PHC2, 1, 1.0);


// suiteDims(TST.CUBE, IDX.PHC_IPP, 1, 1.0);
// suiteDims(TST.CUBE, IDX.QT0Z, 1, 1.0);
// suiteDims(TST.CLUSTER, IDX.QT0Z, 1, 5.0);

testSeriesWQS(TST.CLUSTER, IDX.PHC, 3, 5.0, 1, sizesQuery);
testSeriesWQS(TST.CUBE, IDX.PHC, 3, 1.0, 1, sizesQuery);
testSeriesWQS(TST.OSM, IDX.PHC, 2, 1.0, 1, sizesQuery);
testSeriesWQS(TST.CLUSTER, IDX.PHC_IPP, 3, 5.0, 1, sizesQuery);
testSeriesWQS(TST.CUBE, IDX.PHC_IPP, 3, 1.0, 1, sizesQuery);
testSeriesWQS(TST.OSM, IDX.PHC_IPP, 2, 1.0, 1, sizesQuery);
// testSeriesWQS(TST.CLUSTER, IDX.PHC2, 3, 5.0, 1, sizesQuery);
// testSeriesWQS(TST.CUBE, IDX.PHC2, 3, 1.0, 1, sizesQuery);
// testSeriesWQS(TST.OSM, IDX.PHC2, 2, 1.0, 1, sizesQuery);

testSeries(TST.CLUSTER, IDX.PHC, 3, 5.0, sizesData);
testSeries(TST.CUBE, IDX.PHC, 3, 1.0, sizesData);
testSeries(TST.OSM, IDX.PHC, 2, 1.0, sizesData);
testSeries(TST.CLUSTER, IDX.PHC_IPP, 3, 5.0, sizesData);
testSeries(TST.CUBE, IDX.PHC_IPP, 3, 1.0, sizesData);
testSeries(TST.OSM, IDX.PHC_IPP, 2, 1.0, sizesData);
// testSeries(TST.CLUSTER, IDX.PHC2, 3, 5.0, sizesData);
// testSeries(TST.CUBE, IDX.PHC2, 3, 1.0, sizesData);
// testSeries(TST.OSM, IDX.PHC2, 2, 1.0, sizesData);

suiteDims(TST.CLUSTER, IDX.PHC, 1, 5.0);
suiteDims(TST.CUBE, IDX.PHC, 1, 1.0);
suiteDims(TST.CLUSTER, IDX.PHC_IPP, 1, 5.0);
suiteDims(TST.CUBE, IDX.PHC_IPP, 1, 1.0);
// suiteDims(TST.CLUSTER, IDX.PHC2, 1, 5.0);
// suiteDims(TST.CUBE, IDX.PHC2, 1, 1.0);
suiteDims(TST.CLUSTER, IDX.RSZ, 1, 5.0);
suiteDims(TST.CUBE, IDX.RSZ, 1, 1.0);

testSeries(TST.CLUSTER, IDX.KDZ, 3, 5.0, sizesData);
testSeries(TST.CLUSTER, IDX.KD_SAVA, 3, 5.0, sizesData);
Expand Down Expand Up @@ -284,10 +346,10 @@ public static void main(String[] args) {

} finally {
log.printSummary();
long t2 = System.currentTimeMillis();
long t = t2-t1;
System.out.println("Total time: " + t + "[ms] = " + t/1000 + "[s] = " + t/60000 + "[m]");
}
long t2 = System.currentTimeMillis();
long t = t2-t1;
System.out.println("Total time: " + t + "[ms] = " + t/1000 + "[s] = " + t/60000 + "[m]");
}

private static void suiteDims(TST test, IDX idx, double nEntries, double p1) {
Expand All @@ -313,10 +375,18 @@ private static void suiteDims(TST test, IDX idx, double nEntries, double p1) {
testP(test, idx, 28, n, p1);
}

// testP(test, idx, 30, n, p1);
testP(test, idx, 30, n, p1);
// testP(test, idx, 40, n, p1);
// testP(test, idx, 50, n, p1);

// testP(test, idx, 60, n, p1);
// testP(test, idx, 80, n, p1);
// testP(test, idx, 100, n, p1);
// testP(test, idx, 150, n, p1);
// testP(test, idx, 200, n, p1);



log.writeLogFileForTestSeries("dimsP");
}

Expand Down
108 changes: 74 additions & 34 deletions src/main/java/ch/ethz/globis/tinspin/TestRunner.java
View file
Expand Up @@ -10,6 +10,7 @@
import java.util.ArrayList;
import java.util.Date;
import java.util.Random;
import java.util.function.Consumer;

import ch.ethz.globis.phtree.PhTreeHelper;
import ch.ethz.globis.tinspin.TestStats.IDX;
Expand Down Expand Up @@ -37,6 +38,7 @@ public class TestRunner {
private static final boolean DEBUG = PhTreeHelper.DEBUG;

public static boolean USE_NEW_QUERIES = true;
public static long minimumMS = 2000;

private final TestStats S;
private Random R;
Expand All @@ -54,22 +56,28 @@ public static void main(String[] args) {
return;
}

final int DIM = 8;
final int DIM = 3;
final int N = 1*1000*1000;

//TestStats s0 = new TestStats(TST.CLUSTER, IDX.QTZ, N, DIM, true, 5);
//TestStats s0 = new TestStats(TST.CUBE, IDX.QTZ, N, DIM, true, 1.0);
//TestStats s0 = new TestStats(TST.OSM, IDX.PHC, N, 2, true, 1.0);
//TestStats s0 = new TestStats(TST.CUBE, IDX.PHC, N, DIM, true, 1.0E-5);
TestStats s0 = new TestStats(TST.CLUSTER, IDX.KDZ, N, DIM, false, 5.0);
//TestStats s0 = new TestStats(TST.CUBE, IDX.KDZ, N, DIM, false, 1.0);
//TestStats s0 = new TestStats(TST.OSM, IDX.PHC, N, 2, false, 1.0);
TestStats s0 = new TestStats(TST.CLUSTER, IDX.QT2Z, N, DIM, false, 5.0);
//TestStats s0 = new TestStats(TST.CUBE, IDX.PHC, N, DIM, false, 1.0);
//TestStats s0 = new TestStats(TST.OSM, IDX.QT2Z, N, 2, false, 1.0);
//s0.cfgWindowQueryRepeat = 1000;
//s0.cfgPointQueryRepeat = 1000000;
//s0.cfgUpdateSize = 1000;

//s0.cfgWindowQuerySize = 1;
//s0.cfgKnnQueryBaseRepeat = 1;
//s0.cfgWindowQueryRepeat = 10_000;

//s0.cfgWindowQuerySize = 1;
//s0.cfgKnnQueryBaseRepeat = 1000_000;

//s0.cfgWindowQuerySize = 10000;
//s0.cfgWindowQueryRepeat = 10000;

s0.setSeed(0);
TestRunner test = new TestRunner(s0);
Expand Down Expand Up @@ -155,37 +163,44 @@ public TestStats run() {
// System.out.println("histo(d=" + d + "): " + Arrays.toString(histo));
// }


//window queries
if (tree.supportsWindowQuery()) {
resetR();
repeatQuery(S.cfgWindowQueryRepeat, 0);
repeatQuery(S.cfgWindowQueryRepeat, 1);
S.assortedInfo += " WINDOW_RESULTS=" + S.cfgWindowQuerySize;
} else {
System.err.println("WARNING: window queries disabled");
}
repeatMinimum(() -> {
if (tree.supportsWindowQuery()) {
resetR();
repeatQuery(S.cfgWindowQueryRepeat, 0);
repeatQuery(S.cfgWindowQueryRepeat, 1);
S.assortedInfo += " WINDOW_RESULTS=" + S.cfgWindowQuerySize;
} else {
System.err.println("WARNING: window queries disabled");
}
});

//point queries.
if (tree.supportsPointQuery()) {
resetR();
repeatPointQuery(S.cfgPointQueryRepeat, 0);
repeatPointQuery(S.cfgPointQueryRepeat, 1);
} else {
System.err.println("WARNING: point queries disabled");
}
repeatMinimum(() -> {
if (tree.supportsPointQuery()) {
resetR();
repeatPointQuery(S.cfgPointQueryRepeat, 0);
repeatPointQuery(S.cfgPointQueryRepeat, 1);
} else {
System.err.println("WARNING: point queries disabled");
}
});

//kNN queries
if (tree.supportsKNN()) {
int repeat = getKnnRepeat(S.cfgNDims);
S.assortedInfo += " KNN_REPEAT=" + repeat;
resetR(12345);
repeatKnnQuery(repeat, 0, 1);
repeatKnnQuery(repeat, 1, 1);
repeatKnnQuery(repeat, 0, 10);
repeatKnnQuery(repeat, 1, 10);
} else {
System.err.println("WARNING: kNN queries disabled");
}
repeatMinimum(() -> {
if (tree.supportsKNN()) {
int repeat = getKnnRepeat(S.cfgNDims);
S.assortedInfo += " KNN_REPEAT=" + repeat;
resetR(12345);
repeatKnnQuery(repeat, 0, 1);
repeatKnnQuery(repeat, 1, 1);
repeatKnnQuery(repeat, 0, 10);
repeatKnnQuery(repeat, 1, 10);
} else {
System.err.println("WARNING: kNN queries disabled");
}
});

//update
if (tree.supportsUpdate()) {
Expand All @@ -211,6 +226,18 @@ public TestStats run() {
return S;
}

/**
* We repeat all (repeatable) tests until a minimum amount of time has been spend, this should avoid problems
* with warmup.
* @param r
*/
private void repeatMinimum(Runnable r) {
long t1 = System.currentTimeMillis();
do {
r.run();
} while (System.currentTimeMillis() - t1 < minimumMS);
}

/**
* This method sets the random seed to the default seed plus a given delta.
* This solves the problem that, for example, the kNN generator
Expand Down Expand Up @@ -319,6 +346,7 @@ private void load(TestStats ts) {
}
S.statSjvmE = S.statSjvmF / N;
S.statTLoad = t2-t1;
S.statPSLoad = (long) (N*1000)/(t2-t1);

if (ts.INDEX == IDX.PHCC) {
// TODO: add pht-cpp statistics collection
Expand Down Expand Up @@ -356,10 +384,12 @@ private void repeatQuery(int repeat, int round) {
if (round == 0) {
S.statTq1 = (t2-t1);
S.statTq1E = (long) ((t2-t1)*1000*1000/(double)n);
S.statPSq1 = (long) (repeat*1000)/(t2-t1);
S.statNq1 = n;
} else {
S.statTq2 = (t2-t1);
S.statTq2E = (long) ((t2-t1)*1000*1000/(double)n);
S.statPSq2 = (long) (repeat*1000)/(t2-t1);
S.statNq2 = n;
}
S.statGcDiffWq = JmxTools.getDiff();
Expand All @@ -384,10 +414,12 @@ private void repeatPointQuery(int repeat, int round) {
if (round == 0) {
S.statTqp1 = (t2-t1);
S.statTqp1E = (long) ((t2-t1)*1000*1000/(double)repeat);
S.statPSqp1 = (long) (repeat*1000)/(t2-t1);
S.statNqp1 = n;
} else {
S.statTqp2 = (t2-t1);
S.statTqp2E = (long) ((t2-t1)*1000*1000/(double)repeat);
S.statPSqp2 = (long) (repeat*1000)/(t2-t1);
S.statNqp2 = n;
}
S.statGcDiffPq = JmxTools.getDiff();
Expand Down Expand Up @@ -430,16 +462,18 @@ private void repeatKnnQuery(int repeat, int round, int k) {
long t2 = System.currentTimeMillis();
double avgDist = dist/repeat/k;
log("Element distance: " + dist + " -> " + avgDist);
log("kNN query time: " + (t2-t1) + " ms -> " + (t2-t1)/(double)repeat + " ms/q -> " +
(t2-t1)*1000*1000/(double)k + " ns/q/r");
log("kNN query time (repeat=" + repeat + "): " + (t2-t1) + " ms -> " + (t2-t1)/(double)repeat + " ms/q -> " +
(t2-t1)*1000*1000/(double)(k*repeat) + " ns/q/r");
if (k == 1) {
if (round == 0) {
S.statTqk1_1 = t2-t1;
S.statTqk1_1E = (long) ((t2-t1)*1000*1000/(double)repeat);
S.statPSqk1_1 = (long) (repeat*1000)/(t2-t1);
S.statDqk1_1 = avgDist;
} else {
S.statTqk1_2 = t2-t1;
S.statTqk1_2E = (long) ((t2-t1)*1000*1000/(double)repeat);
S.statPSqk1_2 = (long) (repeat*1000)/(t2-t1);
S.statDqk1_2 = avgDist;
}
S.statGcDiffK1 = JmxTools.getDiff();
Expand All @@ -448,10 +482,12 @@ private void repeatKnnQuery(int repeat, int round, int k) {
if (round == 0) {
S.statTqk10_1 = t2-t1;
S.statTqk10_1E = (long) ((t2-t1)*1000*1000/(double)repeat);
S.statPSqk10_1 = (long) (repeat*1000)/(t2-t1);
S.statDqk10_1 = avgDist;
} else {
S.statTqk10_2 = t2-t1;
S.statTqk10_2E = (long) ((t2-t1)*1000*1000/(double)repeat);
S.statPSqk10_2 = (long) (repeat*1000)/(t2-t1);
S.statDqk10_2 = avgDist;
}
S.statGcDiffK10 = JmxTools.getDiff();
Expand Down Expand Up @@ -545,9 +581,10 @@ public static String summary(ArrayList<Hist> histList) {

private int repeatQueries(double[][] lower, double[][] upper) {
int n=0;
int mod = lower.length / 100;
for (int i = 0; i < lower.length; i++) {
n += tree.query(lower[i], upper[i]);
if (i%10 == 0) System.out.print('.');
if (i%mod == 0) System.out.print('.');
}
System.out.println();
TestRunner.log("n=" + n/(double)lower.length);
Expand Down Expand Up @@ -696,10 +733,12 @@ private void update(int round) {
if (round == 0) {
S.statTu1 = t;
S.statTu1E = (long) (t*1000*1000/(double)n);
S.statPSu1E = (long) (n*1000)/t;
S.statNu1 = n;
} else {
S.statTu2 = t;
S.statTu2E = (long) (t*1000*1000/(double)n);
S.statPSu2E = (long) (n*1000)/t;
S.statNu2 = n;
}
}
Expand All @@ -715,6 +754,7 @@ private void unload() {
log("Deletion time: " + (t2-t1) + " ms -> " +
(t2-t1)*1000*1000/(double)S.cfgNEntries + " ns/q/r");
S.statTUnload = t2-t1;
S.statPSUnload = (long) (n*1000)/(t2-t1);
S.statGcDiffUl = JmxTools.getDiff();
S.statGcTimeUl = JmxTools.getTime();
if (S.cfgNEntries != n) {
Expand Down

0 comments on commit 1115346

Please sign in to comment.