Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add indexes to test names so that Specmatic always generates tests with unique names. #1082

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
40 changes: 29 additions & 11 deletions core/src/main/kotlin/in/specmatic/core/Feature.kt
Original file line number Diff line number Diff line change
Expand Up @@ -352,8 +352,8 @@ data class Feature(
flagsBased
else
flagsBased.withoutGenerativeTests()

originalScenario.generateTestScenarios(resolverStrategies, testVariables, testBaseURLs).map { Pair(originalScenario.copy(generativePrefix = flagsBased.positivePrefix), it) }
val testScenarios = originalScenario.generateTestScenarios(resolverStrategies, testVariables, testBaseURLs)
addIndexAndPrefixToDistinguishTests(originalScenario, testScenarios, flagsBased.positivePrefix)
}

fun negativeTestScenarios(): Sequence<Pair<Scenario, ReturnValue<Scenario>>> {
Expand All @@ -365,19 +365,13 @@ data class Feature(
val negativeTestScenarios =
negativeScenario.generateTestScenarios(flagsBased, testVariables, testBaseURLs)

negativeTestScenarios.filterNot { negativeTestScenarioR ->
negativeTestScenarioR.withDefault(false) { negativeTestScenario ->
val nonMatchingNegativeTestScenarios = negativeTestScenarios.filterNot { currentNegativeTestScenario ->
currentNegativeTestScenario.withDefault(false) { negativeTestScenario ->
val sampleRequest = negativeTestScenario.httpRequestPattern.generate(negativeTestScenario.resolver)
originalScenario.httpRequestPattern.matches(sampleRequest, originalScenario.resolver).isSuccess()
}
}.mapIndexed { index, negativeTestScenarioR ->
Pair(negativeScenario, negativeTestScenarioR.ifValue { negativeTestScenario ->
negativeTestScenario.copy(
generativePrefix = flagsBased.negativePrefix,
disambiguate = { "[${(index + 1)}] " }
)
})
}
addIndexAndPrefixToDistinguishTests(negativeScenario, nonMatchingNegativeTestScenarios, flagsBased.negativePrefix)
}
}

Expand All @@ -400,6 +394,30 @@ data class Feature(
serverState = emptyMap()
}

private fun createIndexLabel(sequenceHasZeroOrOne: Boolean, index: Int): () -> String =
if (sequenceHasZeroOrOne) {
{ "" }
} else {
{ "[${(index + 1)}] " }
}

private fun addIndexAndPrefixToDistinguishTests(
originalScenario: Scenario,
testScenarios: Sequence<ReturnValue<Scenario>>,
prefix: String
): Sequence<Pair<Scenario, ReturnValue<Scenario>>> {
val sequenceHasZeroOrOne = testScenarios.take(2).count() < 2

return testScenarios.mapIndexed { index, currentTestScenarioResult ->
val indexLabel = createIndexLabel(sequenceHasZeroOrOne, index)
Pair(
originalScenario.addIndexLabelAndPrefix(prefix, indexLabel),
currentTestScenarioResult.ifValue { scenario ->
scenario.addIndexLabelAndPrefix( prefix, indexLabel)
})
}
}

private fun combine(baseScenario: Scenario, newScenario: Scenario): Scenario {
return convergeURLMatcher(baseScenario, newScenario).let { convergedScenario ->
convergeHeaders(convergedScenario, newScenario)
Expand Down
12 changes: 10 additions & 2 deletions core/src/main/kotlin/in/specmatic/core/Scenario.kt
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ data class Scenario(
val serviceType:String? = null,
val generativePrefix: String = "",
val statusInDescription: String = httpResponsePattern.status.toString(),
val disambiguate: () -> String = { "" }
val indexLabel: () -> String = { "" }
): ScenarioDetailsForResult {
constructor(scenarioInfo: ScenarioInfo) : this(
scenarioInfo.scenarioName,
Expand Down Expand Up @@ -430,7 +430,7 @@ data class Scenario(

val generativePrefix = this.generativePrefix

return "$generativePrefix Scenario: $method $path ${disambiguate()}-> $statusInDescription$exampleIdentifier"
return "$generativePrefix Scenario: $method $path -> $statusInDescription$exampleIdentifier ${indexLabel()}"
}

fun newBasedOn(scenario: Scenario): Scenario {
Expand Down Expand Up @@ -485,6 +485,14 @@ data class Scenario(
&& operationId.responseStatus == status
&& httpRequestPattern.matchesPath(operationId.requestPath, resolver).isSuccess()
}

fun addIndexLabelAndPrefix(
prefix: String,
indexLabel: () -> String
) = this.copy(
generativePrefix = prefix,
indexLabel = indexLabel
)
}

fun newExpectedServerStateBasedOn(
Expand Down
164 changes: 159 additions & 5 deletions core/src/test/kotlin/in/specmatic/core/FeatureTest.kt
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
package `in`.specmatic.core

import `in`.specmatic.conversions.OpenApiSpecification
import `in`.specmatic.core.pattern.NumberPattern
import `in`.specmatic.core.pattern.StringPattern
import `in`.specmatic.core.value.*
import `in`.specmatic.test.ContractTest
import `in`.specmatic.test.ScenarioTestGenerationException
import `in`.specmatic.test.ScenarioTestGenerationFailure
import `in`.specmatic.test.TestExecutor
Expand Down Expand Up @@ -1476,8 +1474,8 @@ paths:
assertThat(it).contains("-> 4xx")
}

negativeTestScenarios.zip((1..negativeTestScenarios.size).toList()).forEach { (scenario, index) ->
assertThat(scenario.testDescription()).contains("[$index] -> 4xx")
negativeTestScenarios.zip((1..negativeTestScenarios.size).toList()).forEach { (scenario,index) ->
assertThat(scenario.testDescription()).contains(" -> 4xx | EX:SUCCESS [$index]")
}
}

Expand Down Expand Up @@ -1556,14 +1554,170 @@ paths:
}
}

override fun setServerState(serverState: Map<String, Value>) {
override fun setServerState(serverState: Map<String, Value>) {
}
})

assertThat(results.success()).isTrue()
assertThat(results.failureCount).isZero()
}

@Test
fun `multiple positive tests with same names should have an index suffix`() {
val specification = OpenApiSpecification.fromYAML("""
openapi: 3.0.0
info:
title: Sample Product API
description: Optional multiline or single-line description in [CommonMark](http://commonmark.org/help/) or HTML.
version: 0.1.9
servers:
- url: http://localhost:8080
description: Local
paths:
/products:
get:
summary: Search for products
description: get-products
parameters:
- name: type
in: query
schema:
type: string
enum:
- gadget
- book
- food
- other

responses:
'200':
description: List of products in the response
content:
text/plain:
schema:
type: string
""".trimIndent(), "").toFeature()

val contractTests = specification.generateContractTestScenarios(emptyList())
val testNames = contractTests.map { it.first.testDescription().trim() }.toList()
assertThat(testNames).isEqualTo(
listOf(
"Scenario: GET /products -> 200 [1]",
"Scenario: GET /products -> 200 [2]",
"Scenario: GET /products -> 200 [3]",
"Scenario: GET /products -> 200 [4]",
"Scenario: GET /products -> 200 [5]",
)
)
}

@Test
fun `positive tests with distinct names should not have an index suffix`() {
val specification = OpenApiSpecification.fromYAML("""
openapi: 3.0.0
info:
title: Sample Product API
description: Optional multiline or single-line description in [CommonMark](http://commonmark.org/help/) or HTML.
version: 0.1.9
servers:
- url: http://localhost:8080
description: Local
paths:
/products:
get:
summary: Search for products
description: get-products
parameters:
- name: type
in: query
schema:
type: string
required: true
responses:
'200':
description: List of products in the response
content:
text/plain:
schema:
type: string
/orders:
get:
summary: Search for orders
description: get-orders
parameters:
- name: status
in: query
schema:
type: string
required: true

responses:
'200':
description: List of orders in the response
content:
text/plain:
schema:
type: string
""".trimIndent(), ""
).toFeature()

val contractTests = specification.generateContractTestScenarios(emptyList())
val testNames = contractTests.map { it.first.testDescription().trim() }.toList()
assertThat(testNames).isEqualTo(
listOf(
"Scenario: GET /products -> 200",
"Scenario: GET /orders -> 200"
)
)
}

@Test
fun `multiple negative tests with same names should have an index suffix`() {
val specification = OpenApiSpecification.fromYAML("""
openapi: 3.0.0
info:
title: Sample Product API
description: Optional multiline or single-line description in [CommonMark](http://commonmark.org/help/) or HTML.
version: 0.1.9
servers:
- url: http://localhost:8080
description: Local
paths:
/products:
get:
summary: Search for products
description: get-products
parameters:
- name: type
in: query
schema:
type: string
enum:
- gadget
- book
required: true
responses:
'200':
description: List of products in the response
content:
text/plain:
schema:
type: string
""".trimIndent(), "").toFeature()

val contractTests = specification.enableGenerativeTesting().generateContractTestScenarios(emptyList())
val testNames = contractTests.map { it.first.testDescription().trim() }.toList()

assertThat(testNames).isEqualTo(
listOf(
"+ve Scenario: GET /products -> 200 [1]",
"+ve Scenario: GET /products -> 200 [2]",
"-ve Scenario: GET /products -> 4xx [1]",
"-ve Scenario: GET /products -> 4xx [2]"
)
)
}

@Test
fun `errors during test realisation should bubble up via results`() {
val feature = OpenApiSpecification.fromYAML("""
Expand Down