Skip to content

Commit

Permalink
use platform api to run timing-based performance tests
Browse files Browse the repository at this point in the history
it should have more robust computation of expected time depending on hardware

#flakyTest
  • Loading branch information
niktrop committed Nov 27, 2020
1 parent 8411b6f commit 91a20bb
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 116 deletions.
@@ -1,34 +1,29 @@
package org.jetbrains.plugins.scala
package performance.highlighting


import java.util.concurrent.TimeoutException

import com.intellij.openapi.progress.{ProcessCanceledException, ProgressManager}
import com.intellij.openapi.progress.util.StandardProgressIndicatorBase
import com.intellij.psi.PsiFileFactory
import com.intellij.testFramework.PlatformTestUtil
import org.jetbrains.plugins.scala.base.ScalaFixtureTestCase
import org.jetbrains.plugins.scala.util.TestUtils
import org.junit.experimental.categories.Category

import scala.util.Try

/**
* User: Alexander Podkhalyuzin
* Date: 03.08.2009
*/
@Category(Array(classOf[SlowTests]))
class HighlightingPerformanceTest extends ScalaFixtureTestCase {
def doTest(text: String, TIMEOUT: Int): Unit = {
val file = PsiFileFactory.getInstance(myFixture.getProject)
.createFileFromText("dummy.scala", ScalaLanguage.INSTANCE, text, true, false)
TestUtils.assertTiming("Failed highlighting performance test", TIMEOUT,
() => try {
getFixture.allowTreeAccessForAllFiles()
myFixture.testHighlighting(false, false, false, file.getVirtualFile)
}
catch {
case e: RuntimeException =>
}
)

PlatformTestUtil.assertTiming("Failed highlighting performance test", TIMEOUT,
() => Try {
val file = PsiFileFactory.getInstance(myFixture.getProject)
.createFileFromText("dummy.scala", ScalaLanguage.INSTANCE, text, true, false)
getFixture.allowTreeAccessForAllFiles()
myFixture.testHighlighting(false, false, false, file.getVirtualFile)
}
)
}

def testPerformance(): Unit = {
Expand Down
Expand Up @@ -101,56 +101,6 @@ public static String removeEndMarker(String text) {
return text.substring(0, index) + text.substring(index + END_MARKER.length());
}

private static final long ETALON_TIMING = 438;

private static final boolean COVERAGE_ENABLED_BUILD = "true".equals(System.getProperty("idea.coverage.enabled.build"));

private static void assertTiming(String message, long expected, long actual) {
if (COVERAGE_ENABLED_BUILD) return;
long expectedOnMyMachine = expected * Timings.MACHINE_TIMING / ETALON_TIMING;
final double acceptableChangeFactor = 1.1;

// Allow 10% more in case of test machine is busy.
// For faster machines (expectedOnMyMachine < expected) allow nonlinear performance rating:
// just perform better than acceptable expected
if (actual > expectedOnMyMachine * acceptableChangeFactor &&
(expectedOnMyMachine > expected || actual > expected * acceptableChangeFactor)) {
int percentage = (int)(((float)100 * (actual - expectedOnMyMachine)) / expectedOnMyMachine);
Assert.fail(message + ". Operation took " + percentage + "% longer than expected. Expected on my machine: " + expectedOnMyMachine +
". Actual: " + actual + ". Expected on Etalon machine: " + expected + "; Actual on Etalon: " +
(actual * ETALON_TIMING / Timings.MACHINE_TIMING));
}
else {
int percentage = (int)(((float)100 * (actual - expectedOnMyMachine)) / expectedOnMyMachine);
Console.println(message + ". Operation took " + percentage + "% longer than expected. Expected on my machine: " +
expectedOnMyMachine + ". Actual: " + actual + ". Expected on Etalon machine: " + expected +
"; Actual on Etalon: " + (actual * ETALON_TIMING / Timings.MACHINE_TIMING));
}
}

public static void assertTiming(String message, long expected, @NotNull Runnable actionToMeasure) {
assertTiming(message, expected, 4, actionToMeasure);
}

private static void assertTiming(String message, long expected, int attempts, @NotNull Runnable actionToMeasure) {
while (true) {
attempts--;
long start = System.currentTimeMillis();
actionToMeasure.run();
long finish = System.currentTimeMillis();
try {
assertTiming(message, expected, finish - start);
break;
}
catch (AssertionError e) {
if (attempts == 0) throw e;
System.gc();
System.gc();
System.gc();
}
}
}

public static List<String> readInput(String filePath) throws IOException {
String content = new String(FileUtil.loadFileText(new File(filePath)));
Assert.assertNotNull(content);
Expand Down

This file was deleted.

0 comments on commit 91a20bb

Please sign in to comment.