Merge pull request #7 from scamandrill/retry-flakey-tests

retry flakey tests
scamandrill · May 9, 2016 · 1655b78 · 1655b78
2 parents ef7650c + 3f576ef
commit 1655b78
Show file tree

Hide file tree

Showing 18 changed files with 79 additions and 111 deletions.
diff --git a/src/test/scala/com/joypeg/scamandrill/MandrillSpec.scala b/src/test/scala/com/joypeg/scamandrill/MandrillSpec.scala
@@ -0,0 +1,16 @@
+package com.joypeg.scamandrill
+
+import com.joypeg.scamandrill.utils.SimpleLogger
+import org.scalatest.{FlatSpec, Matchers, Retries}
+
+/**
+  * Created by graingert on 09/05/16.
+  */
+trait MandrillSpec extends FlatSpec with Matchers with SimpleLogger with Retries {
+  override def withFixture(test: NoArgTest) = {
+    if (isRetryable(test))
+      withRetry { super.withFixture(test) }
+    else
+      super.withFixture(test)
+  }
+}
diff --git a/src/test/scala/com/joypeg/scamandrill/PreparationTest.scala b/src/test/scala/com/joypeg/scamandrill/PreparationTest.scala
@@ -1,23 +1,16 @@
 package com.joypeg.scamandrill
 
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
-import org.scalatest.BeforeAndAfterAll
 import com.joypeg.scamandrill.models._
 import scala.util.{Failure, Success}
-import com.joypeg.scamandrill.utils._
-import com.joypeg.scamandrill.client.MandrillBlockingClient
 import com.joypeg.scamandrill.MandrillTestUtils._
 import com.joypeg.scamandrill.models.MSendResponse
 import com.joypeg.scamandrill.models.MTemplateAddResponses
 import com.joypeg.scamandrill.models.MTemplate
-import scala.util.Success
 import com.joypeg.scamandrill.models.MTo
-import scala.util.Failure
 import com.joypeg.scamandrill.models.MSendMessage
 
 
-class PreparationTest extends FlatSpec with Matchers with BeforeAndAfterAll with SimpleLogger {
+class PreparationTest extends MandrillSpec {
 
   val realKey = "REPLACEME" //ADD YOUR REAL (NON TESTING) KEY HERE
 

diff --git a/src/test/scala/com/joypeg/scamandrill/client/ExportCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/ExportCallsTest.scala
@@ -1,15 +1,11 @@
 package com.joypeg.scamandrill.client
 
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
-import scala.concurrent.Await
+import com.joypeg.scamandrill.MandrillSpec
+
 import com.joypeg.scamandrill.models._
 import com.joypeg.scamandrill.MandrillTestUtils._
-import com.joypeg.scamandrill.utils.SimpleLogger
-import scala.util.Failure
-import scala.util.Success
 
-class ExportCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class ExportCallsTest extends MandrillSpec {
 
 //  "ExportList" should "work getting a valid MExportResponse (async client)" in {
 //    val res = Await.result(MandrillAsyncClient.exportList(MKey()), DefaultConfig.defaultTimeout)

diff --git a/src/test/scala/com/joypeg/scamandrill/client/InboundCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/InboundCallsTest.scala
@@ -1,19 +1,17 @@
 package com.joypeg.scamandrill.client
 
+import com.joypeg.scamandrill.MandrillSpec
 
-import com.joypeg.scamandrill
-import com.joypeg.scamandrill.client.UnsuccessfulResponseException
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
 import com.joypeg.scamandrill.MandrillTestUtils._
-import com.joypeg.scamandrill.utils.SimpleLogger
+
 import scala.util.Failure
 import scala.util.Success
 
+import org.scalatest.tagobjects.Retryable
 
-class InboundCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class InboundCallsTest extends MandrillSpec {
 
   "InboundAddDomains" should "work getting a valid MInboundDomainResponse (async client)" in {
     val res = Await.result(mandrillAsyncClient.inboundAddDomain(MInboundDomain(domain = "testingdomain")), DefaultConfig.defaultTimeout)
@@ -84,7 +82,7 @@ class InboundCallsTest extends FlatSpec with Matchers with SimpleLogger {
     checkFailedBecauseOfInvalidKey(mandrillBlockingClient.inboundDeleteDomain(MInboundDomain(domain = "testingdomain", key="invalid")))
   }
 
-  "InboundAddRoute" should "fail if the key is not valid, with an , with an 'Unknown_InboundDomain' code" in {
+  "InboundAddRoute" should "fail if the key is not valid, with an , with an 'Unknown_InboundDomain' code" taggedAs(Retryable) in {
     mandrillBlockingClient.inboundAddRoute(validRoute) match {
       case Success(res) =>
         fail("This operation should be unsuccessful")

diff --git a/src/test/scala/com/joypeg/scamandrill/client/IspCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/IspCallsTest.scala
@@ -1,17 +1,15 @@
 package com.joypeg.scamandrill.client
 
-import com.joypeg.scamandrill
-import com.joypeg.scamandrill.client.UnsuccessfulResponseException
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
+import com.joypeg.scamandrill.MandrillSpec
+
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
 import com.joypeg.scamandrill.MandrillTestUtils._
-import com.joypeg.scamandrill.utils.SimpleLogger
+
 import scala.util.Failure
 import scala.util.Success
 
-class IspCallsTest  extends FlatSpec with Matchers with SimpleLogger {
+class IspCallsTest  extends MandrillSpec {
 
   "IspList" should "work getting a valid List[MIspResponse] (async client)" in {
     val res = Await.result(mandrillAsyncClient.ispList(MKey()), DefaultConfig.defaultTimeout)

diff --git a/src/test/scala/com/joypeg/scamandrill/client/MessageCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/MessageCallsTest.scala
@@ -1,18 +1,13 @@
 package com.joypeg.scamandrill.client
 
-import com.joypeg.scamandrill
-import com.joypeg.scamandrill.client.UnsuccessfulResponseException
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
+import com.joypeg.scamandrill.MandrillSpec
 
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
 
 import scala.util.{Failure, Success, Try}
-import com.joypeg.scamandrill.utils._
 
-
-class MessageCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class MessageCallsTest extends MandrillSpec {
 
   import com.joypeg.scamandrill.MandrillTestUtils._
 

diff --git a/src/test/scala/com/joypeg/scamandrill/client/MetadataCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/MetadataCallsTest.scala
@@ -1,15 +1,11 @@
 package com.joypeg.scamandrill.client
 
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
-import scala.concurrent.Await
+import com.joypeg.scamandrill.MandrillSpec
+
 import com.joypeg.scamandrill.models._
 import com.joypeg.scamandrill.MandrillTestUtils._
-import com.joypeg.scamandrill.utils.SimpleLogger
-import scala.util.Failure
-import scala.util.Success
 
-class MetadataCallsTest  extends FlatSpec with Matchers with SimpleLogger {
+class MetadataCallsTest  extends MandrillSpec {
 
 
 //  "MetadataAdd" should "work getting a valid MIMetadataResponse (async client)" in {

diff --git a/src/test/scala/com/joypeg/scamandrill/client/RejectCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/RejectCallsTest.scala
@@ -1,21 +1,16 @@
 package com.joypeg.scamandrill.client
 
-import com.joypeg.scamandrill
-import com.joypeg.scamandrill.client.UnsuccessfulResponseException
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
+import com.joypeg.scamandrill.MandrillSpec
+
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
+
 import scala.util.{Failure, Success}
-import com.joypeg.scamandrill.utils._
 import com.joypeg.scamandrill.MandrillTestUtils._
-import com.joypeg.scamandrill.models.MKey
-import scala.util.Failure
 import com.joypeg.scamandrill.models.MRejectAdd
-import scala.util.Success
 import com.joypeg.scamandrill.models.MRejectAddResponse
 
-class RejectCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class RejectCallsTest extends MandrillSpec {
 
   "RejectAdd" should "work getting a valid MRejectAdd (async client)" in {
     val res = Await.result(mandrillAsyncClient.rejectAdd(MRejectAdd(email = "add@example.com")), DefaultConfig.defaultTimeout)

diff --git a/src/test/scala/com/joypeg/scamandrill/client/SendersCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/SendersCallsTest.scala
@@ -1,20 +1,16 @@
 package com.joypeg.scamandrill.client
 
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
+import com.joypeg.scamandrill.MandrillSpec
 
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
 
-import scala.util.{Failure, Success, Try}
+import scala.util.{Failure, Try}
 import com.joypeg.scamandrill.MandrillTestUtils._
-import com.joypeg.scamandrill.models.MRejectAdd
 
 import scala.util.Success
-import com.joypeg.scamandrill.models.MRejectAddResponse
-import com.joypeg.scamandrill.utils.SimpleLogger
 
-class SendersCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class SendersCallsTest extends MandrillSpec {
 
   "SendersList" should "work getting a valid List[MSendersListResp] (async client)" in {
     val res = Await.result(mandrillAsyncClient.sendersList(MKey()), DefaultConfig.defaultTimeout)

diff --git a/src/test/scala/com/joypeg/scamandrill/client/SubaccountCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/SubaccountCallsTest.scala
@@ -1,19 +1,20 @@
 package com.joypeg.scamandrill.client
 
-import com.joypeg.scamandrill.client.UnsuccessfulResponseException
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
+import com.joypeg.scamandrill.MandrillSpec
+
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
 import com.joypeg.scamandrill.MandrillTestUtils._
-import com.joypeg.scamandrill.utils.SimpleLogger
+
 import scala.util.Failure
 import scala.util.Success
 
+import org.scalatest.tagobjects.Retryable
+
 
-class SubaccountCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class SubaccountCallsTest extends MandrillSpec {
 
-  "SubaccountAdd" should "work getting a valid MSubaccountsResponse (async client)" in {
+  "SubaccountAdd" should "work getting a valid MSubaccountsResponse (async client)" taggedAs(Retryable) in {
     val res: MSubaccountsResponse = Await.result(
       mandrillAsyncClient.subaccountAdd(validSubaccount), DefaultConfig.defaultTimeout
     )

diff --git a/src/test/scala/com/joypeg/scamandrill/client/TagsCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/TagsCallsTest.scala
@@ -1,20 +1,16 @@
 package com.joypeg.scamandrill.client
 
-import com.joypeg.scamandrill
-import com.joypeg.scamandrill.client.UnsuccessfulResponseException
-import org.scalatest.{FlatSpec, Matchers}
-import com.joypeg.scamandrill.utils.SimpleLogger
+import com.joypeg.scamandrill.MandrillSpec
 
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
 
-import scala.util._
 import com.joypeg.scamandrill.MandrillTestUtils._
 import com.joypeg.scamandrill.models.MTagResponse
 
-import scala.util.{Try, Success, Failure}
+import scala.util.{Failure, Success, Try}
 
-class TagsCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class TagsCallsTest extends MandrillSpec {
 
   "TagList" should "work getting a valid List[MTagResponse] (async client)" in {
     val res: List[MTagResponse] = Await.result(mandrillAsyncClient.tagList(MKey()), DefaultConfig.defaultTimeout)

diff --git a/src/test/scala/com/joypeg/scamandrill/client/TemplateCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/TemplateCallsTest.scala
@@ -1,18 +1,15 @@
 package com.joypeg.scamandrill.client
 
-import com.joypeg.scamandrill
-import com.joypeg.scamandrill.client.UnsuccessfulResponseException
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
+import com.joypeg.scamandrill.MandrillSpec
 
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
-import com.joypeg.scamandrill.utils._
 import com.joypeg.scamandrill.MandrillTestUtils._
 
 import scala.util.{Failure, Success, Try}
+import org.scalatest.tagobjects.Retryable
 
-class TemplateCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class TemplateCallsTest extends MandrillSpec {
 
   "TemplateAdd" should "work getting a valid MTemplateAddResponses (async client)" in {
     val res: MTemplateAddResponses = Await.result(
@@ -22,7 +19,7 @@ class TemplateCallsTest extends FlatSpec with Matchers with SimpleLogger {
     res.slug shouldBe "templatetest2"
     res.publish_name shouldBe  "templatetest2"
   }
-  it should "work getting a valid MTemplateAddResponses (blocking client)" in {
+  it should "work getting a valid MTemplateAddResponses (blocking client)" taggedAs(Retryable) in {
     val res: Try[MTemplateAddResponses] = mandrillBlockingClient.templateAdd(validNonPublidhedTemplate)
     res match {
       case Success(resTemplate) =>

diff --git a/src/test/scala/com/joypeg/scamandrill/client/UrlCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/UrlCallsTest.scala
@@ -1,20 +1,16 @@
 package com.joypeg.scamandrill.client
 
-import com.joypeg.scamandrill
-import com.joypeg.scamandrill.client.UnsuccessfulResponseException
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
+import com.joypeg.scamandrill.MandrillSpec
+
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
-import scala.util.Success
-import com.joypeg.scamandrill.utils._
+
 import com.joypeg.scamandrill.MandrillTestUtils._
+
 import scala.util.Failure
-import com.joypeg.scamandrill.models.MRejectAdd
 import scala.util.Success
-import com.joypeg.scamandrill.models.MRejectAddResponse
 
-class UrlCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class UrlCallsTest extends MandrillSpec {
 
   "UrlsList" should "work getting a valid List[MUrlResponse] (async client)" in {
     val res = Await.result(mandrillAsyncClient.urlsList(MKey()), DefaultConfig.defaultTimeout)

diff --git a/src/test/scala/com/joypeg/scamandrill/client/UserCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/UserCallsTest.scala
@@ -1,19 +1,18 @@
 package com.joypeg.scamandrill.client
 
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
-import org.scalatest.BeforeAndAfterAll
+import com.joypeg.scamandrill.MandrillSpec
+
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
-import scala.util.{Try, Failure, Success}
-import com.joypeg.scamandrill.utils._
+
+import scala.util.{Failure, Success, Try}
 import com.joypeg.scamandrill.MandrillTestUtils._
 import com.joypeg.scamandrill.models.MSenderDataResponse
 import com.joypeg.scamandrill.models.MInfoResponse
 import com.joypeg.scamandrill.models.MKey
 import com.joypeg.scamandrill.models.MPingResponse
 
-class UserCallsTest extends FlatSpec with Matchers with SimpleLogger{
+class UserCallsTest extends MandrillSpec {
 
   "Ping" should "work getting a valid MPingResponse (async client)" in {
     val res = Await.result(mandrillAsyncClient.usersPing(MKey()), DefaultConfig.defaultTimeout)

diff --git a/src/test/scala/com/joypeg/scamandrill/client/WebhookCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/WebhookCallsTest.scala
@@ -1,17 +1,15 @@
 package com.joypeg.scamandrill.client
 
-import com.joypeg.scamandrill
-import com.joypeg.scamandrill.client.UnsuccessfulResponseException
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
+import com.joypeg.scamandrill.MandrillSpec
+
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
 import com.joypeg.scamandrill.MandrillTestUtils._
-import com.joypeg.scamandrill.utils.SimpleLogger
+
 import scala.util.Failure
 import scala.util.Success
 
-class WebhookCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class WebhookCallsTest extends MandrillSpec {
 
   "WebhookList" should "work getting a valid List[MWebhooksResponse] (async client)" in {
     val res = Await.result(mandrillAsyncClient.webhookList(MKey()), DefaultConfig.defaultTimeout)

diff --git a/src/test/scala/com/joypeg/scamandrill/client/WhitelistCallsTest.scala b/src/test/scala/com/joypeg/scamandrill/client/WhitelistCallsTest.scala
@@ -1,18 +1,15 @@
 package com.joypeg.scamandrill.client
 
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
+import com.joypeg.scamandrill.MandrillSpec
+
 import scala.concurrent.Await
 import com.joypeg.scamandrill.models._
-import scala.util.Success
-import com.joypeg.scamandrill.utils._
 import com.joypeg.scamandrill.MandrillTestUtils._
+
 import scala.util.Failure
-import com.joypeg.scamandrill.models.MRejectAdd
 import scala.util.Success
-import com.joypeg.scamandrill.models.MRejectAddResponse
 
-class WhitelistCallsTest extends FlatSpec with Matchers with SimpleLogger {
+class WhitelistCallsTest extends MandrillSpec {
 
   "WhitelistAdd" should "work getting a valid MWhitelistAddResponse (async client)" in {
     val res = Await.result(mandrillAsyncClient.whitelistAdd(MWhitelist(email = "whitelist@example.com")), DefaultConfig.defaultTimeout)

diff --git a/src/test/scala/com/joypeg/scamandrill/models/DefaultConfigTest.scala b/src/test/scala/com/joypeg/scamandrill/models/DefaultConfigTest.scala
@@ -1,10 +1,12 @@
 package com.joypeg.scamandrill.models
 
-import org.scalatest.{Matchers, FlatSpec}
+import com.joypeg.scamandrill.MandrillSpec
+import org.scalatest.{FlatSpec, Matchers}
 import com.joypeg.scamandrill.utils.SimpleLogger
+
 import scala.concurrent.duration._
 
-class DefaultConfigTest extends FlatSpec with Matchers with SimpleLogger {
+class DefaultConfigTest extends MandrillSpec {
 
   "DefaultConfig" should "read the defaut key from the configuration" in {
     DefaultConfig.defaultKeyFromConfig.getClass shouldBe classOf[String]