This repository has been archived by the owner on Nov 9, 2017. It is now read-only.
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #303 from zanata/test-retries
Provide a retry mechanism for the slightly unstable tests
- Loading branch information
Showing
5 changed files
with
199 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
functional-test/src/test/java/org/zanata/feature/infrastructure/InfrastructureTestSuite.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * Copyright 2013, Red Hat, Inc. and individual contributors as indicated by the | ||
| * @author tags. See the copyright.txt file in the distribution for a full | ||
| * listing of individual contributors. | ||
| * | ||
| * This is free software; you can redistribute it and/or modify it under the | ||
| * terms of the GNU Lesser General Public License as published by the Free | ||
| * Software Foundation; either version 2.1 of the License, or (at your option) | ||
| * any later version. | ||
| * | ||
| * This software is distributed in the hope that it will be useful, but WITHOUT | ||
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more | ||
| * details. | ||
| * | ||
| * You should have received a copy of the GNU Lesser General Public License | ||
| * along with this software; if not, write to the Free Software Foundation, | ||
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF | ||
| * site: http://www.fsf.org. | ||
| */ | ||
| package org.zanata.feature.infrastructure; | ||
|
|
||
| import org.junit.runner.RunWith; | ||
| import org.junit.runners.Suite; | ||
|
|
||
| /** | ||
| * This suite comprises the tests to validate the test harness and framework | ||
| * for Zanata functional tests. | ||
| * @author Damian Jansen <a | ||
| * href="mailto:djansen@redhat.com">djansen@redhat.com</a> | ||
| */ | ||
| @RunWith(Suite.class) | ||
| @Suite.SuiteClasses({ RetryRuleTest.class }) | ||
| public class InfrastructureTestSuite { | ||
| } |
72 changes: 72 additions & 0 deletions
72
functional-test/src/test/java/org/zanata/feature/infrastructure/RetryRuleTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| /* | ||
| * Copyright 2013, Red Hat, Inc. and individual contributors as indicated by the | ||
| * @author tags. See the copyright.txt file in the distribution for a full | ||
| * listing of individual contributors. | ||
| * | ||
| * This is free software; you can redistribute it and/or modify it under the | ||
| * terms of the GNU Lesser General Public License as published by the Free | ||
| * Software Foundation; either version 2.1 of the License, or (at your option) | ||
| * any later version. | ||
| * | ||
| * This software is distributed in the hope that it will be useful, but WITHOUT | ||
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
| * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more | ||
| * details. | ||
| * | ||
| * You should have received a copy of the GNU Lesser General Public License | ||
| * along with this software; if not, write to the Free Software Foundation, | ||
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF | ||
| * site: http://www.fsf.org. | ||
| */ | ||
|
|
||
| package org.zanata.feature.infrastructure; | ||
|
|
||
| import org.hamcrest.Matchers; | ||
| import org.junit.Rule; | ||
| import org.junit.Test; | ||
| import org.junit.experimental.categories.Category; | ||
| import org.zanata.feature.BasicAcceptanceTest; | ||
| import org.zanata.util.RetryRule; | ||
|
|
||
| import static org.hamcrest.MatcherAssert.assertThat; | ||
|
|
||
| /** | ||
| * @author Damian Jansen <a | ||
| * href="mailto:djansen@redhat.com">djansen@redhat.com</a> | ||
| */ | ||
| @Category(BasicAcceptanceTest.class) | ||
| public class RetryRuleTest { | ||
|
|
||
| @Rule | ||
| public RetryRule retryRule = new RetryRule(2); | ||
|
|
||
| @Test | ||
| public void retryPassAfterFail() { | ||
| // Fail on the first execution, but pass on the second | ||
| assertThat("Current try is greater than 1", retryRule.currentTry(), | ||
| Matchers.greaterThan(1)); | ||
| // Can only pass on second execution | ||
| assertThat("This is the second try", retryRule.currentTry(), | ||
| Matchers.equalTo(2)); | ||
| } | ||
|
|
||
| @Test | ||
| public void passWillPass() { | ||
| assertThat("A normal passing test will pass", true); | ||
| assertThat("And pass on the first try", retryRule.currentTry(), | ||
| Matchers.equalTo(1)); | ||
| } | ||
|
|
||
| @Test(expected = AssertionError.class) | ||
| public void retryFailsWhenAllTriesFail() throws Exception { | ||
| // Fail the first execution | ||
| if (retryRule.currentTry() == 1) { | ||
| throw new Exception(); | ||
| } | ||
| // Passes on the second execution, expect-fails on the third | ||
| assertThat("The execution count is correct", retryRule.currentTry(), | ||
| Matchers.equalTo(2)); | ||
| // Fails on the second execution | ||
| throw new Exception(); | ||
| } | ||
| } |
83 changes: 83 additions & 0 deletions
83
functional-test/src/test/java/org/zanata/util/RetryRule.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| /* | ||
| * This work is licensed under a Creative Commons Attribution-ShareAlike 2.5 | ||
| * Generic License - See http://creativecommons.org/licenses/by-sa/2.5/ | ||
| */ | ||
|
|
||
| package org.zanata.util; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.junit.rules.TestRule; | ||
| import org.junit.runner.Description; | ||
| import org.junit.runners.model.Statement; | ||
|
|
||
| /** | ||
| * @author Damian Jansen <a | ||
| * href="mailto:djansen@redhat.com">djansen@redhat.com</a> | ||
| * | ||
| * Provide a retry mechanism that allows a test to be executed repeatedly | ||
| * until it has passed, or fails the given threshold. | ||
| * This threshold is defined as the original execution, plus the number | ||
| * of given retries, i.e<br/> | ||
| * RetryRule retryRule = new RetryRule(2);<br/> | ||
| * will execute a failing test 3 times before giving up. | ||
| * | ||
| * This file is an edited form of the work provided by <a | ||
| * href="http://stackoverflow.com/users/1836/matthew-farwell">Matthew | ||
| * Farwell</a> on the Stack Exchange Network, specifically <a | ||
| * href="http://stackoverflow.com/a/8301639">this submission</a> to Stack | ||
| * Overflow. | ||
| */ | ||
| @Slf4j | ||
| public class RetryRule implements TestRule { | ||
|
|
||
| private int retries; | ||
| private int currentExecution; | ||
|
|
||
| /** | ||
| * Constructor | ||
| * @param retries maximum number of added attempts before failing a test | ||
| */ | ||
| public RetryRule(int retries) { | ||
| this.retries = retries; | ||
| } | ||
|
|
||
| /** | ||
| * Return the human number of the current execution (1 for first try, etc) | ||
| * @return integer | ||
| */ | ||
| public int currentTry() { | ||
| return currentExecution + 1; | ||
| } | ||
|
|
||
| /** | ||
| * Executes the TestRule repeatedly until the retries have been exceeded, | ||
| * or the test passes | ||
| * @param base TestRule base link | ||
| * @param description TestRule description link | ||
| * @return TestRule statement | ||
| * @see TestRule | ||
| */ | ||
| public Statement apply(final Statement base, | ||
| final Description description) { | ||
| return new Statement() { | ||
| @Override | ||
| public void evaluate() throws Throwable { | ||
| Throwable throwable = null; | ||
| for (currentExecution = 0; currentExecution <= retries; | ||
| currentExecution++) { | ||
| try { | ||
| base.evaluate(); | ||
| return; | ||
| } catch (Throwable t) { | ||
| throwable = t; | ||
| log.info(description.getDisplayName() + ": Execution " | ||
| + (currentTry()) + " failed"); | ||
| } | ||
| } | ||
| log.info(description.getDisplayName() + | ||
| ": Failure threshold (" + retries + ") exceeded"); | ||
| throw throwable; | ||
| } | ||
| }; | ||
| } | ||
| } |