From 009438da446feb3e3da3dc67404b527cbd97a177 Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Tue, 13 Dec 2022 15:42:44 +0300 Subject: [PATCH 1/4] Add smoke tests for fuzzer platform --- .../src/main/kotlin/org/utbot/fuzzing/Api.kt | 11 +- .../org/utbot/fuzzing/FuzzerSmokeTest.kt | 209 ++++++++++++++++++ 2 files changed, 213 insertions(+), 7 deletions(-) create mode 100644 utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt index a22f1b9592..7ecde64338 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt @@ -1,6 +1,7 @@ @file:JvmName("FuzzingApi") package org.utbot.fuzzing +import kotlinx.coroutines.yield import mu.KotlinLogging import org.utbot.fuzzing.seeds.KnownValue import org.utbot.fuzzing.utils.chooseOne @@ -259,7 +260,7 @@ suspend fun , F : Feedback> Fuzzing.f val seeds = Statistics() run breaking@ { sequence { - while (true) { + while (description.parameters.isNotEmpty()) { if (dynamicallyGenerated.isNotEmpty()) { yield(dynamicallyGenerated.removeFirst()) } else { @@ -280,15 +281,11 @@ suspend fun , F : Feedback> Fuzzing.f } } }.forEach execution@ { values -> + yield() check(values.parameters.size == values.result.size) { "Cannot create value for ${values.parameters}" } val valuesCache = mutableMapOf, R>() val result = values.result.map { valuesCache.computeIfAbsent(it) { r -> create(r) } } - val feedback = try { - fuzzing.handle(description, result) - } catch (t: Throwable) { - logger.error(t) { "Error when running fuzzing with $values" } - return@execution - } + val feedback = fuzzing.handle(description, result) when (feedback.control) { Control.CONTINUE -> { seeds.put(random, configuration, feedback, values) diff --git a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt new file mode 100644 index 0000000000..2d51635b9d --- /dev/null +++ b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt @@ -0,0 +1,209 @@ +package org.utbot.fuzzing + +import kotlinx.coroutines.* +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import kotlin.reflect.KClass + +class FuzzerSmokeTest { + + @Test + fun `fuzzing doesn't run with empty parameters`() { + runBlocking { + var count = 0 + runFuzzing, BaseFeedback>( + { _, _ -> sequenceOf() }, + Description(emptyList()) + ) { _, _ -> + count += 1 + BaseFeedback(Unit, Control.STOP) + } + Assertions.assertEquals(0, count) + } + } + + @Test + fun `fuzzing throws an exception if no values generated for some type`() { + assertThrows { + runBlocking { + var count = 0 + runFuzzing, BaseFeedback>( + { _, _ -> sequenceOf() }, + Description(listOf(Unit)) + ) { _, _ -> + count += 1 + BaseFeedback(Unit, Control.STOP) + } + Assertions.assertEquals(0, count) + } + } + } + + @Test + fun `fuzzing stops on Control$STOP signal after one execution`() { + runBlocking { + var count = 0 + runFuzzing( + { _, _ -> sequenceOf(Seed.Simple(Unit)) }, + Description(listOf(Unit)) + ) { _, _ -> + count += 1 + BaseFeedback(Unit, Control.STOP) + } + Assertions.assertEquals(1, count) + } + } + + @Test + fun `fuzzing stops on Control$STOP signal after 3 execution`() { + runBlocking { + var count = 3 + var executions = 0 + runFuzzing( + { _, _ -> sequenceOf(Seed.Simple(Unit)) }, + Description(listOf(Unit)) + ) { _, _ -> + executions++ + BaseFeedback(Unit, if (--count == 0) Control.STOP else Control.CONTINUE) + } + Assertions.assertEquals(0, count) + Assertions.assertEquals(3, executions) + } + } + + @Test + fun `fuzzing runs value generation once per type by default`() { + runBlocking { + var count = 10 + var generations = 0 + runFuzzing( + { _, _ -> + generations++ + sequenceOf(Seed.Simple(Unit)) + }, + Description(listOf(Unit)) + ) { _, _ -> + BaseFeedback(Unit, if (--count == 0) Control.STOP else Control.CONTINUE) + } + Assertions.assertEquals(0, count) + Assertions.assertEquals(1, generations) + } + } + + @Test + fun `fuzzing runs value generation every type when cache is being reset`() { + runBlocking { + var count = 10 + var generations = 0 + runFuzzing( + { _, _ -> + generations++ + sequenceOf(Seed.Simple(Unit)) + }, + Description(listOf(Unit)) + ) { _, _ -> + BaseFeedback(Unit, if (--count == 0) Control.STOP else Control.RESET_TYPE_CACHE_AND_CONTINUE) + } + Assertions.assertEquals(0, count) + // fuzzing swaps generated and modified values, + // therefore it can call generation only once, + // but at the moment it should be called 5 times + Assertions.assertEquals(5, generations) + } + } + + @Test + fun `fuzzer rethrow exception from execution block`() { + class SpecialException : Exception() + runBlocking { + assertThrows { + withTimeout(1000) { + runFuzzing( + { _, _ -> sequenceOf(Seed.Simple(Unit)) }, + Description(listOf(Unit)) + ) { _, _ -> + throw SpecialException() + } + } + } + } + } + + @Test + fun `fuzzer generates recursive data with correct depth`() { + data class Node(val left: Node?, val right: Node?) + + runBlocking { + val configuration = Configuration( + recursionTreeDepth = 10 + ) + var depth = 0 + var count = 0 + runFuzzing( + ValueProvider, Node?, Description>> { _, _ -> + sequenceOf(Seed.Recursive( + construct = Routine.Create(listOf(Node::class, Node::class)) { v -> + Node(v[0], v[1]) + }, + empty = Routine.Empty { null } + )) + }, + Description(listOf(Node::class)), + configuration = configuration + ) { _, v -> + fun traverse(n: Node?, l: Int = 1) { + n ?: return + depth = maxOf(depth, l) + count++ + traverse(n.left, l + 1) + traverse(n.right, l + 1) + } + traverse(v.first()) + Assertions.assertEquals(configuration.recursionTreeDepth, depth) + Assertions.assertEquals((1 shl configuration.recursionTreeDepth) - 1, count) + BaseFeedback(this, Control.STOP) + } + } + } + + @Test + fun `fuzzer can be cancelled by timeout`() { + val start = System.currentTimeMillis() + runBlocking { + assertThrows { + withTimeout(1000) { + runFuzzing( + { _, _ -> sequenceOf(Seed.Simple(Unit)) }, + Description(listOf(Unit)) + ) { _, _ -> + if (System.currentTimeMillis() - start > 10_000) { + error("Fuzzer didn't stopped in 10 000 ms") + } + BaseFeedback(Unit, Control.CONTINUE) + } + } + } + } + } + + @Test + fun `fuzzer can be cancelled by coroutine`() { + runBlocking { + val deferred = async { + val start = System.currentTimeMillis() + runFuzzing( + { _, _ -> sequenceOf(Seed.Simple(Unit)) }, + Description(listOf(Unit)) + ) { _, _ -> + if (System.currentTimeMillis() - start > 10_000) { + error("Fuzzer didn't stopped in 10_000 ms") + } + BaseFeedback(Unit, Control.CONTINUE) + } + } + delay(1000) + deferred.cancel() + } + } +} \ No newline at end of file From 7f556d5a6f1615b4e0902bf209a93fdbe6830080 Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Wed, 14 Dec 2022 10:47:00 +0300 Subject: [PATCH 2/4] Add tests for ValueProvider --- .../kotlin/org/utbot/fuzzing/Providers.kt | 21 +-- .../kotlin/org/utbot/fuzzing/ProvidersTest.kt | 172 ++++++++++++++++++ 2 files changed, 178 insertions(+), 15 deletions(-) create mode 100644 utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt index c6aac53178..f2b1e9ca1c 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt @@ -70,7 +70,7 @@ fun interface ValueProvider> { * * This model provider is called before `anotherValueProviders`. */ - fun with(anotherValueProvider: ValueProvider): ValueProvider { + infix fun with(anotherValueProvider: ValueProvider): ValueProvider { fun toList(m: ValueProvider) = if (m is Combined) m.providers else listOf(m) return Combined(toList(this) + toList(anotherValueProvider)) } @@ -122,15 +122,8 @@ fun interface ValueProvider> { * Creates new value provider that creates default value if no values are generated by this provider. */ fun withFallback(fallbackSupplier: (T) -> Seed) : ValueProvider { - val thisProvider = this - return ValueProvider { description, type -> - if (accept(type)) { - thisProvider.generate(description, type) - .takeIf { it.iterator().hasNext() } - ?: sequenceOf(fallbackSupplier(type)) - } else { - emptySequence() - } + return withFallback { _, type -> + sequenceOf(fallbackSupplier(type)) } } @@ -170,10 +163,6 @@ fun interface ValueProvider> { } } -fun> List>.withFallback(fallbackSupplier: (T) -> Seed) : List> { - return listOf(ValueProvider.of(this).withFallback(fallbackSupplier)) -} - /** * Simple value provider for a concrete type. * @@ -186,6 +175,8 @@ class TypeProvider>( ) : ValueProvider { override fun accept(type: T) = this.type == type override fun generate(description: D, type: T) = sequence { - this.generate(description, type) + if (accept(type)) { + this.generate(description, type) + } } } diff --git a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt new file mode 100644 index 0000000000..9d306b9ad7 --- /dev/null +++ b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt @@ -0,0 +1,172 @@ +package org.utbot.fuzzing + +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class ProvidersTest { + + private fun p(supplier: () -> T) : ValueProvider> { + return ValueProvider { _, _ -> sequenceOf(Seed.Simple(supplier())) } + } + + private fun p( + accept: () -> Boolean, + supplier: () -> T + ) : ValueProvider> { + return object : ValueProvider> { + override fun accept(type: Unit) = accept() + override fun generate(description: Description, type: Unit) = sequence> { + yield(Seed.Simple(supplier())) + } + } + } + + private val description = Description(listOf(Unit)) + + @Test + fun `test common provider API`() { + val provider = ValueProvider.of(listOf(p { 1 })) + (1..3).forEach { _ -> + val attempt = provider.generate(description, Unit).first() as Seed.Simple + Assertions.assertEquals(1, attempt.value) + } + } + + @Test + fun `test merging of providers`() { + var count = 0 + ValueProvider.of(listOf( + p { 1 }, p { 2 }, p { 3 } + )).generate(description, Unit) + .map { (it as Seed.Simple).value } + .forEachIndexed { index, result -> + count++ + Assertions.assertEquals(index + 1, result) + } + Assertions.assertEquals(3, count) + } + + @Test + fun `test merging of providers several times`() { + val p1 = p { 1 } + val p2 = p { 2 } + val p3 = p { 3 } + val p4 = p { 4 } + val m1 = ValueProvider.of(listOf(p1, p2)) + val m2 = p3 with p4 + val m3 = m1 with m2 + val m4 = m3 with ValueProvider.of(emptyList()) + + Assertions.assertEquals(1, p1.generate(description, Unit).count()) + Assertions.assertEquals(1, (p1.generate(description, Unit).first() as Seed.Simple).value) + + Assertions.assertEquals(1, p2.generate(description, Unit).count()) + Assertions.assertEquals(2, (p2.generate(description, Unit).first() as Seed.Simple).value) + + Assertions.assertEquals(1, p3.generate(description, Unit).count()) + Assertions.assertEquals(3, (p3.generate(description, Unit).first() as Seed.Simple).value) + + Assertions.assertEquals(1, p4.generate(description, Unit).count()) + Assertions.assertEquals(4, (p4.generate(description, Unit).first() as Seed.Simple).value) + + Assertions.assertEquals(2, m1.generate(description, Unit).count()) + Assertions.assertEquals(1, (m1.generate(description, Unit).first() as Seed.Simple).value) + Assertions.assertEquals(2, (m1.generate(description, Unit).drop(1).first() as Seed.Simple).value) + + Assertions.assertEquals(2, m2.generate(description, Unit).count()) + Assertions.assertEquals(3, (m2.generate(description, Unit).first() as Seed.Simple).value) + Assertions.assertEquals(4, (m2.generate(description, Unit).drop(1).first() as Seed.Simple).value) + + Assertions.assertEquals(4, m3.generate(description, Unit).count()) + Assertions.assertEquals(1, (m3.generate(description, Unit).first() as Seed.Simple).value) + Assertions.assertEquals(2, (m3.generate(description, Unit).drop(1).first() as Seed.Simple).value) + Assertions.assertEquals(3, (m3.generate(description, Unit).drop(2).first() as Seed.Simple).value) + Assertions.assertEquals(4, (m3.generate(description, Unit).drop(3).first() as Seed.Simple).value) + + Assertions.assertEquals(4, m4.generate(description, Unit).count()) + } + + @Test + fun `test merging of providers by with-method`() { + var count = 0 + p { 1 }.with(p { 2 }).with(p { 3 }).generate(description, Unit) + .map { (it as Seed.Simple).value } + .forEachIndexed { index, result -> + count++ + Assertions.assertEquals(index + 1, result) + } + Assertions.assertEquals(3, count) + } + + @Test + fun `test excepting from providers`() { + val provider = p { 2 } + val seq = ValueProvider.of(listOf( + p { 1 }, provider, p { 3 } + )).except { + it === provider + }.generate(description, Unit) + Assertions.assertEquals(2, seq.count()) + Assertions.assertEquals(1, (seq.first() as Seed.Simple).value) + Assertions.assertEquals(3, (seq.drop(1).first() as Seed.Simple).value) + } + + @Test + fun `provider is not called when accept-method returns false`() { + val seq = ValueProvider.of(listOf( + p({ true }, { 1 }), p({ false }, { 2 }), p({ true }, { 3 }), + )).generate(description, Unit) + Assertions.assertEquals(2, seq.count()) + Assertions.assertEquals(1, (seq.first() as Seed.Simple).value) + Assertions.assertEquals(3, (seq.drop(1).first() as Seed.Simple).value) + } + + @Test + fun `provider doesnt call fallback when values is generated`() { + val seq = ValueProvider.of(listOf( + p({ true }, { 1 }), p({ false }, { 2 }), p({ true }, { 3 }), + )).withFallback { + Seed.Simple(4) + }.generate(description, Unit) + Assertions.assertEquals(2, seq.count()) + Assertions.assertEquals(1, (seq.first() as Seed.Simple).value) + Assertions.assertEquals(3, (seq.drop(1).first() as Seed.Simple).value) + } + + @Test + fun `provider calls fallback when values are not generated`() { + val seq = ValueProvider.of(listOf( + p({ false }, { 1 }), p({ false }, { 2 }), p({ false }, { 3 }), + )).withFallback { + Seed.Simple(4) + }.generate(description, Unit) + Assertions.assertEquals(1, seq.count()) + Assertions.assertEquals(4, (seq.first() as Seed.Simple).value) + } + + @Test + fun `provider generates no values when fallback cannot accept value`() { + val seq = ValueProvider.of(listOf( + p({ false }, { 1 }), p({ false }, { 2 }), p({ false }, { 3 }), + )).withFallback( + object : ValueProvider> { + override fun accept(type: Unit) = false + override fun generate(description: Description, type: Unit) = emptySequence>() + } + ).generate(description, Unit) + Assertions.assertEquals(0, seq.count()) + } + + @Test + fun `type providers check exactly the type`() { + val seq1 = TypeProvider>('A') { _, _ -> + yield(Seed.Simple(2)) + }.generate(Description(listOf('A')), 'A') + Assertions.assertEquals(1, seq1.count()) + + val seq2 = TypeProvider>('A') { _, _ -> + yield(Seed.Simple(2)) + }.generate(Description(listOf('A')), 'B') + Assertions.assertEquals(0, seq2.count()) + } +} \ No newline at end of file From 8b7ea00989a76ea9a2023b65da4c93ad77c5082e Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Wed, 14 Dec 2022 11:36:26 +0300 Subject: [PATCH 3/4] Test that random generates same values by default for every run --- .../org/utbot/fuzzing/FuzzerSmokeTest.kt | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt index 2d51635b9d..57d5ddd092 100644 --- a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt +++ b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt @@ -4,6 +4,8 @@ import kotlinx.coroutines.* import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.utbot.fuzzing.seeds.BitVectorValue +import org.utbot.fuzzing.seeds.Signed import kotlin.reflect.KClass class FuzzerSmokeTest { @@ -206,4 +208,46 @@ class FuzzerSmokeTest { deferred.cancel() } } + + @Test + fun `fuzzer generate same result when random is seeded`() { + data class B(var a: Int) + val provider = ValueProvider> { _, _ -> + sequenceOf( + Seed.Simple(B(0)) { p, r -> B(p.a + r.nextInt()) }, + Seed.Known(BitVectorValue(32, Signed.POSITIVE)) { B(it.toInt()) }, + Seed.Recursive( + construct = Routine.Create(listOf(Unit)) { _ -> B(2) }, + modify = sequenceOf( + Routine.Call(listOf(Unit)) { self, _ -> self.a = 3 }, + Routine.Call(listOf(Unit)) { self, _ -> self.a = 4 }, + Routine.Call(listOf(Unit)) { self, _ -> self.a = 5 }, + Routine.Call(listOf(Unit)) { self, _ -> self.a = 6 }, + ), + empty = Routine.Empty { B(7) } + ), + Seed.Collection( + construct = Routine.Collection { size -> B(size) }, + modify = Routine.ForEach(listOf(Unit)) { self, ind, v -> self.a = ind * self.a * v.first().a } + ) + ) + } + fun createValues(): MutableList { + val result = mutableListOf() + val probes = 1_000 + runBlocking { + runFuzzing(provider, Description(listOf(Unit))) { _, v -> + result.add(v.first().a) + BaseFeedback(Unit, if (result.size >= probes) Control.STOP else Control.CONTINUE) + } + } + return result + } + val firstRun = createValues() + val secondRun = createValues() + val thirdRun = createValues() + Assertions.assertEquals(firstRun, secondRun) + Assertions.assertEquals(firstRun, thirdRun) + Assertions.assertEquals(secondRun, thirdRun) + } } \ No newline at end of file From 6b09a94d926c50f3a3ca0f022a26e8f3e103cd71 Mon Sep 17 00:00:00 2001 From: Maksim Pelevin Date: Wed, 14 Dec 2022 14:42:22 +0300 Subject: [PATCH 4/4] Result of fuzzing should be reproducible --- .../org/utbot/engine/UtBotSymbolicEngine.kt | 1 - .../kotlin/org/utbot/fuzzing/JavaLanguage.kt | 3 +- .../java/org/utbot/fuzzing/samples/Stubs.java | 8 +++ .../org/utbot/fuzzing/JavaFuzzingTest.kt | 53 +++++++++++++++++++ .../src/main/kotlin/org/utbot/fuzzing/Api.kt | 4 +- .../org/utbot/fuzzing/demo/JsonFuzzing.kt | 3 +- .../kotlin/org/utbot/fuzzing/utils/Trie.kt | 6 --- 7 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 utbot-fuzzers/src/test/java/org/utbot/fuzzing/samples/Stubs.java create mode 100644 utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index c85b7e84f8..9ceab10d33 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -344,7 +344,6 @@ class UtBotSymbolicEngine( methodUnderTest, collectConstantsForFuzzer(graph), names, - { mockStrategy.eligibleToMock(it, classUnderTest) }, listOf(transform(ValueProvider.of(defaultValueProviders(defaultIdGenerator)))) ) { thisInstance, descr, values -> if (controller.job?.isActive == false || System.currentTimeMillis() >= until) { diff --git a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt index 4bc4cd4349..4ff6df412f 100644 --- a/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt +++ b/utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt @@ -44,7 +44,6 @@ suspend fun runJavaFuzzing( methodUnderTest: ExecutableId, constants: Collection, names: List, - mock: (ClassId) -> Boolean = { false }, providers: List> = defaultValueProviders(idGenerator), exec: suspend (thisInstance: FuzzedValue?, description: FuzzedDescription, values: List) -> BaseFeedback, FuzzedType, FuzzedValue> ) { @@ -83,7 +82,7 @@ suspend fun runJavaFuzzing( null } } - shouldMock = mock + shouldMock = { false } } val thisInstance = with(methodUnderTest) { diff --git a/utbot-fuzzers/src/test/java/org/utbot/fuzzing/samples/Stubs.java b/utbot-fuzzers/src/test/java/org/utbot/fuzzing/samples/Stubs.java new file mode 100644 index 0000000000..16c2748681 --- /dev/null +++ b/utbot-fuzzers/src/test/java/org/utbot/fuzzing/samples/Stubs.java @@ -0,0 +1,8 @@ +package org.utbot.fuzzing.samples; + +@SuppressWarnings("unused") +public class Stubs { + + public static void name(String value) {} + +} diff --git a/utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt b/utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt new file mode 100644 index 0000000000..6561ff6f99 --- /dev/null +++ b/utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt @@ -0,0 +1,53 @@ +package org.utbot.fuzzing + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.utbot.framework.plugin.api.MethodId +import org.utbot.framework.plugin.api.TestIdentityPreservingIdGenerator +import org.utbot.framework.plugin.api.UtPrimitiveModel +import org.utbot.framework.plugin.api.util.* +import org.utbot.fuzzer.FuzzedConcreteValue +import org.utbot.fuzzing.samples.Stubs +import org.utbot.fuzzing.utils.Trie + +class JavaFuzzingTest { + + @Test + fun `string generates same values`() { + fun collect(): List { + return runBlockingWithContext { + val results = mutableListOf() + var count = 0 + val probes = 10000 + runJavaFuzzing( + TestIdentityPreservingIdGenerator, + methodUnderTest = MethodId(Stubs::class.id, "name", voidClassId, listOf(stringClassId)), + constants = listOf( + FuzzedConcreteValue(stringClassId, "Hello"), + FuzzedConcreteValue(stringClassId, "World"), + FuzzedConcreteValue(stringClassId, "!"), + ), + names = emptyList() + ) { _, _, values -> + results += (values.first().model as UtPrimitiveModel).value as String + BaseFeedback(Trie.emptyNode(), if (++count < probes) Control.CONTINUE else Control.STOP) + } + Assertions.assertEquals(count, results.size) + results + } + } + + val probe1 = collect() + val probe2 = collect() + Assertions.assertEquals(probe1, probe2) + } + + private fun runBlockingWithContext(block: suspend () -> T) : T { + return withUtContext(UtContext(this::class.java.classLoader)) { + runBlocking { + block() + } + } + } +} \ No newline at end of file diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt index 7ecde64338..e89eeedc67 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt @@ -631,8 +631,8 @@ private class Node( private class Statistics> { - private val seeds = hashMapOf>() - private val count = hashMapOf() + private val seeds = linkedMapOf>() + private val count = linkedMapOf() fun put(random: Random, configuration: Configuration, feedback: FEEDBACK, seed: Node) { if (random.flipCoin(configuration.probUpdateSeedInsteadOfKeepOld)) { diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JsonFuzzing.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JsonFuzzing.kt index fff760d395..2c7eb0b3e5 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JsonFuzzing.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JsonFuzzing.kt @@ -31,6 +31,7 @@ private class JsonBuilder( */ @Suppress("RemoveExplicitTypeArguments") suspend fun main() { + var count = 0 BaseFuzzing, Feedback>( TypeProvider(CustomType.INT) { _, _ -> for (b in Signed.values()) { @@ -79,7 +80,7 @@ suspend fun main() { }, ) { _, values -> println(values) - emptyFeedback() + if (++count < 1000) emptyFeedback() else error("") }.fuzz( Description(listOf(CustomType.LST, CustomType.OBJ)), Random(0), diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt index 16ba6b0c1e..7bc4190f64 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt @@ -175,12 +175,6 @@ open class Trie( get() = error("empty node has no data") override val count: Int get() = 0 - override fun equals(other: Any?): Boolean { - return false - } - override fun hashCode(): Int { - return 0 - } } companion object {