diff --git a/plugin/src/test/groovy/com/novoda/staticanalysis/internal/findbugs/FindbugsIntegrationTest.groovy b/plugin/src/test/groovy/com/novoda/staticanalysis/internal/findbugs/FindbugsIntegrationTest.groovy
index c7025db..c4fe2b1 100644
--- a/plugin/src/test/groovy/com/novoda/staticanalysis/internal/findbugs/FindbugsIntegrationTest.groovy
+++ b/plugin/src/test/groovy/com/novoda/staticanalysis/internal/findbugs/FindbugsIntegrationTest.groovy
@@ -363,7 +363,12 @@ class FindbugsIntegrationTest {
     @Test
     void shouldBeUpToDateWhenCheckTaskRunsAgain() {
         def project = projectRule.newProject()
-                .withSourceSet('main', SOURCES_WITH_LOW_VIOLATION)
+                .withSourceSet('debug', SOURCES_WITH_LOW_VIOLATION, SOURCES_WITH_MEDIUM_VIOLATION)	                .withSourceSet('main', SOURCES_WITH_LOW_VIOLATION)
+                .withSourceSet('release', SOURCES_WITH_HIGH_VIOLATION)	
+                .withPenalty('''{	
+                    maxErrors = 10	
+                    maxWarnings = 10	
+                }''')
                 .withToolsConfig('findbugs {}')
 
         project.build('check')