From 5c3e88cab66da6bced45c2c7df9b31dc0f896554 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 22 May 2023 16:39:31 +0300 Subject: [PATCH 01/35] switching to develop --- .github/workflows/build-and-test.yml | 1 + .github/workflows/github-packages.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index cb75d4810..541a0f81f 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -11,6 +11,7 @@ on: push: branches: - main + - develop - 'feature-**' - '*/feature-**' pull_request: diff --git a/.github/workflows/github-packages.yml b/.github/workflows/github-packages.yml index 038883397..498be8115 100644 --- a/.github/workflows/github-packages.yml +++ b/.github/workflows/github-packages.yml @@ -3,7 +3,7 @@ name: Publish to GitHub packages on: push: branches: - - main + - develop jobs: publish: From 0ce1a3c49b20dbe33e42315fa296dc160bfef1c5 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Wed, 24 May 2023 10:58:22 +0300 Subject: [PATCH 02/35] make AbstractFullExprSetCollector api be Java friendly --- .../jacodb/api/cfg/FullExprSetCollector.kt | 117 ++++++++++-------- .../features/classpaths/KotlinMetadata.kt | 2 +- .../java/org/jacodb/testing/JavaApiTest.java | 16 ++- 3 files changed, 80 insertions(+), 55 deletions(-) diff --git a/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/FullExprSetCollector.kt b/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/FullExprSetCollector.kt index d2effbe0c..f26cc0253 100644 --- a/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/FullExprSetCollector.kt +++ b/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/FullExprSetCollector.kt @@ -130,9 +130,9 @@ abstract class AbstractFullRawExprSetCollector : JcRawExprVisitor, Default abstract fun ifMatches(expr: JcRawExpr) } -abstract class AbstractFullExprSetCollector : JcExprVisitor, DefaultJcInstVisitor { +abstract class AbstractFullExprSetCollector : JcExprVisitor, DefaultJcInstVisitor { - override val defaultInstHandler: (JcInst) -> Unit + override val defaultInstHandler: (JcInst) -> Any get() = { it.operands.forEach { ifMatches(it) @@ -154,103 +154,114 @@ abstract class AbstractFullExprSetCollector : JcExprVisitor, DefaultJcInst expr.args.forEach { it.accept(this) } } - override fun visitJcAddExpr(expr: JcAddExpr) = visitBinaryExpr(expr) - override fun visitJcAndExpr(expr: JcAndExpr) = visitBinaryExpr(expr) - override fun visitJcCmpExpr(expr: JcCmpExpr) = visitBinaryExpr(expr) - override fun visitJcCmpgExpr(expr: JcCmpgExpr) = visitBinaryExpr(expr) - override fun visitJcCmplExpr(expr: JcCmplExpr) = visitBinaryExpr(expr) - override fun visitJcDivExpr(expr: JcDivExpr) = visitBinaryExpr(expr) - override fun visitJcMulExpr(expr: JcMulExpr) = visitBinaryExpr(expr) - override fun visitJcEqExpr(expr: JcEqExpr) = visitBinaryExpr(expr) - override fun visitJcNeqExpr(expr: JcNeqExpr) = visitBinaryExpr(expr) - override fun visitJcGeExpr(expr: JcGeExpr) = visitBinaryExpr(expr) - override fun visitJcGtExpr(expr: JcGtExpr) = visitBinaryExpr(expr) - override fun visitJcLeExpr(expr: JcLeExpr) = visitBinaryExpr(expr) - override fun visitJcLtExpr(expr: JcLtExpr) = visitBinaryExpr(expr) - override fun visitJcOrExpr(expr: JcOrExpr) = visitBinaryExpr(expr) - override fun visitJcRemExpr(expr: JcRemExpr) = visitBinaryExpr(expr) - override fun visitJcShlExpr(expr: JcShlExpr) = visitBinaryExpr(expr) - override fun visitJcShrExpr(expr: JcShrExpr) = visitBinaryExpr(expr) - override fun visitJcSubExpr(expr: JcSubExpr) = visitBinaryExpr(expr) - override fun visitJcUshrExpr(expr: JcUshrExpr) = visitBinaryExpr(expr) - override fun visitJcXorExpr(expr: JcXorExpr) = visitBinaryExpr(expr) - - override fun visitJcLambdaExpr(expr: JcLambdaExpr) { + override fun visitJcAddExpr(expr: JcAddExpr): Any = visitBinaryExpr(expr) + override fun visitJcAndExpr(expr: JcAndExpr): Any = visitBinaryExpr(expr) + override fun visitJcCmpExpr(expr: JcCmpExpr): Any = visitBinaryExpr(expr) + override fun visitJcCmpgExpr(expr: JcCmpgExpr): Any = visitBinaryExpr(expr) + override fun visitJcCmplExpr(expr: JcCmplExpr): Any = visitBinaryExpr(expr) + override fun visitJcDivExpr(expr: JcDivExpr): Any = visitBinaryExpr(expr) + override fun visitJcMulExpr(expr: JcMulExpr): Any = visitBinaryExpr(expr) + override fun visitJcEqExpr(expr: JcEqExpr): Any = visitBinaryExpr(expr) + override fun visitJcNeqExpr(expr: JcNeqExpr): Any = visitBinaryExpr(expr) + override fun visitJcGeExpr(expr: JcGeExpr): Any = visitBinaryExpr(expr) + override fun visitJcGtExpr(expr: JcGtExpr): Any = visitBinaryExpr(expr) + override fun visitJcLeExpr(expr: JcLeExpr): Any = visitBinaryExpr(expr) + override fun visitJcLtExpr(expr: JcLtExpr): Any = visitBinaryExpr(expr) + override fun visitJcOrExpr(expr: JcOrExpr): Any = visitBinaryExpr(expr) + override fun visitJcRemExpr(expr: JcRemExpr): Any = visitBinaryExpr(expr) + override fun visitJcShlExpr(expr: JcShlExpr): Any = visitBinaryExpr(expr) + override fun visitJcShrExpr(expr: JcShrExpr): Any = visitBinaryExpr(expr) + override fun visitJcSubExpr(expr: JcSubExpr): Any = visitBinaryExpr(expr) + override fun visitJcUshrExpr(expr: JcUshrExpr): Any = visitBinaryExpr(expr) + override fun visitJcXorExpr(expr: JcXorExpr): Any = visitBinaryExpr(expr) + + override fun visitJcLambdaExpr(expr: JcLambdaExpr): Any { ifMatches(expr) expr.args.forEach { it.accept(this) } + return Unit } - override fun visitJcLengthExpr(expr: JcLengthExpr) { + override fun visitJcLengthExpr(expr: JcLengthExpr): Any { ifMatches(expr) expr.array.accept(this) + return Unit } - override fun visitJcNegExpr(expr: JcNegExpr) { + override fun visitJcNegExpr(expr: JcNegExpr): Any { ifMatches(expr) expr.operand.accept(this) + return Unit } - override fun visitJcCastExpr(expr: JcCastExpr) { + override fun visitJcCastExpr(expr: JcCastExpr): Any { ifMatches(expr) expr.operand.accept(this) + return Unit } - override fun visitJcNewExpr(expr: JcNewExpr) { + override fun visitJcNewExpr(expr: JcNewExpr): Any { ifMatches(expr) + return Unit } - override fun visitJcNewArrayExpr(expr: JcNewArrayExpr) { + override fun visitJcNewArrayExpr(expr: JcNewArrayExpr): Any { ifMatches(expr) expr.dimensions.forEach { it.accept(this) } + return Unit } - override fun visitJcInstanceOfExpr(expr: JcInstanceOfExpr) { + override fun visitJcInstanceOfExpr(expr: JcInstanceOfExpr): Any { ifMatches(expr) expr.operand.accept(this) + return Unit } - override fun visitJcDynamicCallExpr(expr: JcDynamicCallExpr) { + override fun visitJcDynamicCallExpr(expr: JcDynamicCallExpr): Any { ifMatches(expr) expr.args.forEach { it.accept(this) } + return Unit } - override fun visitJcVirtualCallExpr(expr: JcVirtualCallExpr) = visitCallExpr(expr) - override fun visitJcStaticCallExpr(expr: JcStaticCallExpr) = visitCallExpr(expr) - override fun visitJcSpecialCallExpr(expr: JcSpecialCallExpr) = visitCallExpr(expr) - override fun visitJcThis(value: JcThis) = ifMatches(value) - override fun visitJcArgument(value: JcArgument) = ifMatches(value) - override fun visitJcLocalVar(value: JcLocalVar) = ifMatches(value) + override fun visitJcVirtualCallExpr(expr: JcVirtualCallExpr): Any = visitCallExpr(expr) + override fun visitJcStaticCallExpr(expr: JcStaticCallExpr): Any = visitCallExpr(expr) + override fun visitJcSpecialCallExpr(expr: JcSpecialCallExpr): Any = visitCallExpr(expr) + override fun visitJcThis(value: JcThis): Any = ifMatches(value) + override fun visitJcArgument(value: JcArgument): Any = ifMatches(value) + override fun visitJcLocalVar(value: JcLocalVar): Any = ifMatches(value) - override fun visitJcFieldRef(value: JcFieldRef) { + override fun visitJcFieldRef(value: JcFieldRef): Any { ifMatches(value) value.instance?.accept(this) + return Unit } - override fun visitJcArrayAccess(value: JcArrayAccess) { + override fun visitJcArrayAccess(value: JcArrayAccess) : Any{ ifMatches(value) value.array.accept(this) value.index.accept(this) + return Unit } - override fun visitJcPhiExpr(expr: JcPhiExpr) { + override fun visitJcPhiExpr(expr: JcPhiExpr) : Any{ ifMatches(expr) expr.args.forEach { it.accept(this) } expr.values.forEach { it.accept(this) } + return Unit } - override fun visitExternalJcExpr(expr: JcExpr) = ifMatches(expr) - override fun visitJcBool(value: JcBool) = ifMatches(value) - override fun visitJcByte(value: JcByte) = ifMatches(value) - override fun visitJcChar(value: JcChar) = ifMatches(value) - override fun visitJcShort(value: JcShort) = ifMatches(value) - override fun visitJcInt(value: JcInt) = ifMatches(value) - override fun visitJcLong(value: JcLong) = ifMatches(value) - override fun visitJcFloat(value: JcFloat) = ifMatches(value) - override fun visitJcDouble(value: JcDouble) = ifMatches(value) - override fun visitJcNullConstant(value: JcNullConstant) = ifMatches(value) - override fun visitJcStringConstant(value: JcStringConstant) = ifMatches(value) - override fun visitJcClassConstant(value: JcClassConstant) = ifMatches(value) - override fun visitJcMethodConstant(value: JcMethodConstant) = ifMatches(value) + override fun visitExternalJcExpr(expr: JcExpr): Any = ifMatches(expr) + override fun visitJcBool(value: JcBool) : Any= ifMatches(value) + override fun visitJcByte(value: JcByte): Any = ifMatches(value) + override fun visitJcChar(value: JcChar) : Any= ifMatches(value) + override fun visitJcShort(value: JcShort): Any = ifMatches(value) + override fun visitJcInt(value: JcInt): Any = ifMatches(value) + override fun visitJcLong(value: JcLong): Any = ifMatches(value) + override fun visitJcFloat(value: JcFloat): Any = ifMatches(value) + override fun visitJcDouble(value: JcDouble): Any = ifMatches(value) + override fun visitJcNullConstant(value: JcNullConstant): Any = ifMatches(value) + override fun visitJcStringConstant(value: JcStringConstant) : Any= ifMatches(value) + override fun visitJcClassConstant(value: JcClassConstant) : Any= ifMatches(value) + override fun visitJcMethodConstant(value: JcMethodConstant) : Any= ifMatches(value) abstract fun ifMatches(expr: JcExpr) } \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/KotlinMetadata.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/KotlinMetadata.kt index 89eceb551..01abd49cb 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/KotlinMetadata.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/KotlinMetadata.kt @@ -70,7 +70,7 @@ object KotlinMetadata : JcClassExtFeature { } -class KotlinMetadataHolder(val meta: KotlinClassMetadata) { +class KotlinMetadataHolder(meta: KotlinClassMetadata) { val functions: List = when (meta) { is KotlinClassMetadata.Class -> meta.toKmClass().functions diff --git a/jacodb-core/src/test/java/org/jacodb/testing/JavaApiTest.java b/jacodb-core/src/test/java/org/jacodb/testing/JavaApiTest.java index ff11c7668..92eacb9ad 100644 --- a/jacodb-core/src/test/java/org/jacodb/testing/JavaApiTest.java +++ b/jacodb-core/src/test/java/org/jacodb/testing/JavaApiTest.java @@ -19,11 +19,14 @@ import com.google.common.collect.Lists; import org.jacodb.api.JcClassOrInterface; import org.jacodb.api.JcClasspath; -import org.jacodb.api.JcClasspathFeature; import org.jacodb.api.JcDatabase; +import org.jacodb.api.cfg.JcArgument; +import org.jacodb.api.cfg.JcExpr; +import org.jacodb.api.cfg.TypedExprResolver; import org.jacodb.impl.JacoDB; import org.jacodb.impl.JcSettings; import org.jacodb.impl.features.Usages; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -34,6 +37,17 @@ public class JavaApiTest { + private static class ArgumentResolver extends TypedExprResolver { + + @Override + public void ifMatches(@NotNull JcExpr jcExpr) { + if (jcExpr instanceof JcArgument) { + getResult().add((JcArgument) jcExpr); + } + } + + } + @Test public void createJcdb() throws ExecutionException, InterruptedException, IOException { System.out.println("Creating database"); From 9631fefd993752e6a89e854cf52d3f234bcf3771 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Wed, 24 May 2023 17:25:27 +0300 Subject: [PATCH 03/35] working on caching settings --- jacodb-core/build.gradle.kts | 2 + .../main/kotlin/org/jacodb/impl/JcSettings.kt | 38 +++++++++++++----- .../org/jacodb/impl/cfg/TypedMethodRefImpl.kt | 2 +- .../features/classpaths/ClasspathCache.kt | 40 ++++++++++++++----- .../kotlin/org/jacodb/testing/cfg/IRTest.kt | 4 +- 5 files changed, 61 insertions(+), 25 deletions(-) diff --git a/jacodb-core/build.gradle.kts b/jacodb-core/build.gradle.kts index 5d59a4449..298593c23 100644 --- a/jacodb-core/build.gradle.kts +++ b/jacodb-core/build.gradle.kts @@ -61,6 +61,8 @@ dependencies { testFixturesImplementation(group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version = coroutinesVersion) testFixturesImplementation(group = "org.mockito", name = "mockito-core", version = "4.8.0") testFixturesImplementation(group = "org.jetbrains", name = "annotations", version = "20.1.0") + testImplementation(group = "org.slf4j", name = "slf4j-simple", version = "1.6.1") + } tasks.register("generateSqlScheme") { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt index 42192726e..b3ddc10ba 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt @@ -198,22 +198,40 @@ enum class PredefinedPersistenceType : JcPersistenceType { class JcByteCodeCache(val prefixes: List = persistentListOf("java.", "javax.", "kotlinx.", "kotlin.")) +data class JcCacheSegmentSettings( + val valueStoreType: ValueStoreType = ValueStoreType.STRONG, + val maxSize: Long = 10_000, + val expiration: Duration = Duration.ofMinutes(1) +) + +enum class ValueStoreType { WEAK, SOFT, STRONG } + + class JcCacheSettings { - var classes: Pair = 10_000L to Duration.ofMinutes(1) - var types: Pair = 10_000L to Duration.ofMinutes(1) - var graphs: Pair = 10_000L to Duration.ofMinutes(1) + var classes: JcCacheSegmentSettings = JcCacheSegmentSettings() + var types: JcCacheSegmentSettings = JcCacheSegmentSettings() + var rawInstLists: JcCacheSegmentSettings = JcCacheSegmentSettings() + var instLists: JcCacheSegmentSettings = JcCacheSegmentSettings() + var flowGraphs: JcCacheSegmentSettings = JcCacheSegmentSettings() - var byteCodeCache: JcByteCodeCache = JcByteCodeCache() + fun classes(maxSize: Long, expiration: Duration, valueStoreType: ValueStoreType = ValueStoreType.STRONG) = apply { + classes = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) + } - fun classes(maxSize: Long, expiration: Duration) = apply { - classes = maxSize to expiration + fun types(maxSize: Long, expiration: Duration, valueStoreType: ValueStoreType = ValueStoreType.STRONG) = apply { + types = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) } - fun types(maxSize: Long, expiration: Duration) = apply { - types = maxSize to expiration + fun rawInstLists(maxSize: Long, expiration: Duration, valueStoreType: ValueStoreType = ValueStoreType.STRONG) = apply { + rawInstLists = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) } - fun graphs(maxSize: Long, expiration: Duration) = apply { - graphs = maxSize to expiration + fun instLists(maxSize: Long, expiration: Duration, valueStoreType: ValueStoreType = ValueStoreType.STRONG) = apply { + instLists = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) } + + fun flowGraphs(maxSize: Long, expiration: Duration, valueStoreType: ValueStoreType = ValueStoreType.STRONG) = apply { + flowGraphs = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) + } + } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt index ddeef3d26..e0d659e02 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt @@ -132,7 +132,7 @@ data class TypedMethodRefImpl( raw.returnType ) - override val method: JcTypedMethod by weakLazy { + override val method: JcTypedMethod by softLazy { type.getMethod(name, argTypes, returnType) } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt index 913aed1b6..544ed268f 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt @@ -19,6 +19,7 @@ package org.jacodb.impl.features.classpaths import com.google.common.cache.CacheBuilder import com.google.common.cache.CacheLoader import com.google.common.cache.CacheStats +import mu.KLogging import org.jacodb.api.JcClassFoundEvent import org.jacodb.api.JcClassNotFound import org.jacodb.api.JcClassOrInterface @@ -34,11 +35,13 @@ import org.jacodb.api.cfg.JcGraph import org.jacodb.api.cfg.JcInst import org.jacodb.api.cfg.JcInstList import org.jacodb.api.cfg.JcRawInst +import org.jacodb.impl.JcCacheSegmentSettings import org.jacodb.impl.JcCacheSettings +import org.jacodb.impl.ValueStoreType import org.jacodb.impl.cfg.nonCachedFlowGraph import org.jacodb.impl.cfg.nonCachedInstList import org.jacodb.impl.cfg.nonCachedRawInstList -import java.time.Duration +import java.text.NumberFormat import java.util.* @@ -46,6 +49,9 @@ import java.util.* * any class cache should extend this class */ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, JcMethodExtFeature { + + companion object : KLogging() + /** * */ @@ -55,21 +61,21 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc private val typesCache = segmentBuilder(settings.types) .build>() - private val rawInstCache = segmentBuilder(settings.graphs) + private val rawInstCache = segmentBuilder(settings.rawInstLists) .build(object : CacheLoader>() { override fun load(key: JcMethod): JcInstList { return nonCachedRawInstList(key) } }); - private val instCache = segmentBuilder(settings.graphs, weakValues = true) + private val instCache = segmentBuilder(settings.instLists) .build(object : CacheLoader>() { override fun load(key: JcMethod): JcInstList { return nonCachedInstList(key) } }); - private val cfgCache = segmentBuilder(settings.graphs, weakValues = true) + private val cfgCache = segmentBuilder(settings.flowGraphs) .build(object : CacheLoader() { override fun load(key: JcMethod): JcGraph { return nonCachedFlowGraph(key) @@ -102,18 +108,18 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc } } - protected fun segmentBuilder(settings: Pair, weakValues: Boolean = false): CacheBuilder { - val maxSize = settings.first - val expiration = settings.second + protected fun segmentBuilder(settings: JcCacheSegmentSettings): CacheBuilder { + val maxSize = settings.maxSize + val expiration = settings.expiration return CacheBuilder.newBuilder() .expireAfterAccess(expiration) .recordStats() .maximumSize(maxSize).let { - if (weakValues) { - it.weakValues() - } else { - it.softValues() + when (settings.valueStoreType) { + ValueStoreType.WEAK -> it.weakValues() + ValueStoreType.SOFT -> it.softValues() + else -> it } } } @@ -125,4 +131,16 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc this["raw-instructions"] = rawInstCache.stats() this["instructions"] = instCache.stats() } + + open fun dumpStats() { + stats().entries.toList() + .sortedBy { it.key } + .forEach { (key, stat) -> + logger.info("$key cache hit rate: ${stat.hitRate().forPercentages()}, total count ${stat.requestCount()}") + } + } + + protected fun Double.forPercentages(): String { + return NumberFormat.getPercentInstance().format(this) + } } \ No newline at end of file diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt index 9ac7a7400..e447562f9 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt @@ -348,9 +348,7 @@ class IRTest : BaseTest() { @AfterEach fun printStats() { cp.features!!.filterIsInstance().forEach { - it.stats().forEach { - println(it.key + " : " + it.value) - } + it.dumpStats() } } From 40475699801dbcef67dcfc7d3e43f15fa7df9b58 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Wed, 24 May 2023 17:26:39 +0300 Subject: [PATCH 04/35] cleanup --- .../jacodb/impl/features/classpaths/ClasspathCache.kt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt index 544ed268f..c088dcf57 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt @@ -52,9 +52,6 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc companion object : KLogging() - /** - * - */ private val classesCache = segmentBuilder(settings.classes) .build>() @@ -66,21 +63,21 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc override fun load(key: JcMethod): JcInstList { return nonCachedRawInstList(key) } - }); + }) private val instCache = segmentBuilder(settings.instLists) .build(object : CacheLoader>() { override fun load(key: JcMethod): JcInstList { return nonCachedInstList(key) } - }); + }) private val cfgCache = segmentBuilder(settings.flowGraphs) .build(object : CacheLoader() { override fun load(key: JcMethod): JcGraph { return nonCachedFlowGraph(key) } - }); + }) override fun tryFindClass(classpath: JcClasspath, name: String): Optional? { From 0a089d457023fa839cc0518883d906a6641544e2 Mon Sep 17 00:00:00 2001 From: Ivan Volkov Date: Mon, 22 May 2023 13:21:50 +0300 Subject: [PATCH 05/35] [jacodb-ifds] Fix usage of locals criteria --- .../org/jacodb/analysis/analyzers/UnusedVariableAnalyzer.kt | 6 +++++- .../kotlin/org/jacodb/analysis/impl/UnusedVariableTest.kt | 1 - jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcGraphs.kt | 6 ------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/analyzers/UnusedVariableAnalyzer.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/analyzers/UnusedVariableAnalyzer.kt index 2b1930c93..aaee78e62 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/analyzers/UnusedVariableAnalyzer.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/analyzers/UnusedVariableAnalyzer.kt @@ -61,8 +61,12 @@ class UnusedVariableAnalyzer( private fun AccessPath.isUsedAt(inst: JcInst): Boolean { val callExpr = inst.callExpr - // TODO: currently we use here that `this` is not in `operands` of JcSpecialCallExpr, this may be wrong if (callExpr != null) { + // Don't count constructor calls as usages + if (callExpr.method.method.isConstructor && isUsedAt((callExpr as JcSpecialCallExpr).instance)) { + return false + } + if (graph.callees(inst).none()) { return isUsedAt(callExpr) } diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/UnusedVariableTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/UnusedVariableTest.kt index 6718d295b..5680858be 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/UnusedVariableTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/UnusedVariableTest.kt @@ -38,7 +38,6 @@ import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.MethodSource -import java.io.File import java.util.* import java.util.stream.Stream import kotlin.streams.asStream diff --git a/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcGraphs.kt b/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcGraphs.kt index 495f7540a..5e6335193 100644 --- a/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcGraphs.kt +++ b/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcGraphs.kt @@ -39,12 +39,6 @@ class ValueResolver : TypedExprResolver() { result.add(expr) } } - - override fun visitJcSpecialCallExpr(expr: JcSpecialCallExpr) { - ifMatches(expr) - expr.args.forEach { it.accept(this) } - } - } fun JcGraph.apply(visitor: JcInstVisitor): JcGraph { From d4141105d9712ef5846633ea380b656c167ce1c0 Mon Sep 17 00:00:00 2001 From: Ivan Volkov Date: Wed, 24 May 2023 14:05:28 +0300 Subject: [PATCH 06/35] [jacodb-ifds] Add JodaTime tests --- jacodb-analysis/build.gradle.kts | 2 + .../analyzers/UnusedVariableAnalyzer.kt | 17 +++++- .../jacodb/analysis/engine/IFDSInstance.kt | 2 +- .../analysis/impl/JodaDateTimeAnalysisTest.kt | 60 +++++++++++++++++++ .../jacodb/analysis/impl/NpeAnalysisTest.kt | 4 +- .../org/jacodb/impl/cfg/TypedMethodRefImpl.kt | 4 ++ 6 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt diff --git a/jacodb-analysis/build.gradle.kts b/jacodb-analysis/build.gradle.kts index 37b7521c9..af1919b90 100644 --- a/jacodb-analysis/build.gradle.kts +++ b/jacodb-analysis/build.gradle.kts @@ -15,6 +15,8 @@ dependencies { testImplementation("org.junit.jupiter:junit-jupiter-params:5.9.2") testImplementation(files("src/testFixtures/resources/juliet.jar")) testImplementation(files("src/testFixtures/resources/pointerbench.jar")) + testImplementation("joda-time:joda-time:2.12.5") + implementation("org.jetbrains.kotlinx:kotlinx-cli:0.3.5") implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1") diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/analyzers/UnusedVariableAnalyzer.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/analyzers/UnusedVariableAnalyzer.kt index aaee78e62..f5975a130 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/analyzers/UnusedVariableAnalyzer.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/analyzers/UnusedVariableAnalyzer.kt @@ -32,13 +32,16 @@ import org.jacodb.api.JcClasspath import org.jacodb.api.JcMethod import org.jacodb.api.analysis.JcApplicationGraph import org.jacodb.api.cfg.JcArgument +import org.jacodb.api.cfg.JcArrayAccess import org.jacodb.api.cfg.JcAssignInst +import org.jacodb.api.cfg.JcBranchingInst import org.jacodb.api.cfg.JcExpr import org.jacodb.api.cfg.JcInst import org.jacodb.api.cfg.JcInstanceCallExpr import org.jacodb.api.cfg.JcLocal import org.jacodb.api.cfg.JcSpecialCallExpr import org.jacodb.api.cfg.JcStaticCallExpr +import org.jacodb.api.cfg.JcTerminatingInst import org.jacodb.api.cfg.values import org.jacodb.api.ext.cfg.callExpr @@ -77,8 +80,14 @@ class UnusedVariableAnalyzer( return false } if (inst is JcAssignInst) { + if (inst.lhv is JcArrayAccess && isUsedAt((inst.lhv as JcArrayAccess).index)) { + return true + } return isUsedAt(inst.rhv) && (inst.lhv !is JcLocal || inst.rhv !is JcLocal) } + if (inst is JcTerminatingInst || inst is JcBranchingInst) { + return inst.operands.any { isUsedAt(it) } + } return false } @@ -96,7 +105,7 @@ class UnusedVariableAnalyzer( } } val vulnerabilities = used.filterValues { !it }.keys.map { - VulnerabilityInstance(value, listOf(it.toString()), it.toString(), emptyList()) + VulnerabilityInstance(value, listOf(it.location.toString()), it.location.toString() + "cmd = " + it.toString(), emptyList()) } return DumpableAnalysisResult(vulnerabilities) } @@ -122,7 +131,11 @@ private class UnusedVariableForwardFunctions( if (fact == ZEROFact) { val toPath = current.lhv.toPathOrNull() ?: return listOf(ZEROFact) - return listOf(ZEROFact, UnusedVariableNode(toPath, current)) + return if (!toPath.isOnHeap) { + listOf(ZEROFact, UnusedVariableNode(toPath, current)) + } else { + listOf(ZEROFact) + } } if (fact !is UnusedVariableNode) { diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/engine/IFDSInstance.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/engine/IFDSInstance.kt index e1964a233..fd999cd4c 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/engine/IFDSInstance.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/engine/IFDSInstance.kt @@ -100,7 +100,7 @@ class IFDSInstance( if (e !in pathEdges) { pathEdges.add(e) workList.add(e) - val isNew = pred.kind != SEQUENT || pred.predEdge.v.domainFact != e.v.domainFact + val isNew = pred.kind != SEQUENT && pred.kind != UNKNOWN || pred.predEdge.v.domainFact != e.v.domainFact val predInst = pred.predEdge.v.statement.takeIf { it != e.v.statement && it.location.method == e.v.statement.location.method } listeners.forEach { it.onPropagate(e, predInst, isNew) } return true diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt new file mode 100644 index 000000000..3ec9dfc3c --- /dev/null +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.analysis.impl + +import org.jacodb.analysis.AnalysisEngineFactory +import org.jacodb.analysis.JcNaivePoints2EngineFactory +import org.jacodb.analysis.JcSimplifiedGraphFactory +import org.jacodb.analysis.NPEAnalysisFactory +import org.jacodb.analysis.UnusedVariableAnalysisFactory +import org.jacodb.api.ext.findClass +import org.jacodb.impl.features.InMemoryHierarchy +import org.jacodb.impl.features.Usages +import org.jacodb.testing.BaseTest +import org.jacodb.testing.WithDB +import org.joda.time.DateTime +import org.junit.jupiter.api.Test + +class JodaDateTimeAnalysisTest : BaseTest() { + companion object : WithDB(Usages, InMemoryHierarchy) + + fun testOneFactory(factory: AnalysisEngineFactory) { + val clazz = cp.findClass() + + val graph = JcSimplifiedGraphFactory().createGraph(cp) + val points2Engine = JcNaivePoints2EngineFactory().createPoints2Engine(graph) + val ifds = factory.createAnalysisEngine(graph, points2Engine) + clazz.declaredMethods + .forEach { ifds.addStart(it) } + val result = ifds.analyze().foundVulnerabilities + + result.forEachIndexed { ind, vulnerability -> + println("VULNERABILITY $ind:") + println(vulnerability) + } + } + + @Test + fun `test Unused variable analysis`() { + testOneFactory(UnusedVariableAnalysisFactory()) + } + + @Test + fun `test NPE analysis`() { + testOneFactory(NPEAnalysisFactory()) + } +} \ No newline at end of file diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt index 3010b5c24..6c508bad8 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt @@ -266,8 +266,8 @@ class NpeAnalysisTest : BaseTest() { @Test fun `analyse something`() { - val testingMethod = cp.findClass().declaredMethods.single { it.name == "npeOnLength" } - val results = findNpeSources(testingMethod) + val testingMethod = cp.findClass().declaredMethods.single { it.name == "kek" } + val results = testingMethod.flowGraph() print(results) } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt index e0d659e02..d57c05f8a 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt @@ -213,6 +213,10 @@ class JcInstLocationImpl( override val lineNumber: Int ) : JcInstLocation { + override fun toString(): String { + return "${methodRef.method.enclosingClass.name}#${methodRef.method.name}:$lineNumber" + } + override val method: JcMethod by softLazy { methodRef.method } From fa8384bcf7fd6791fcaa79a088f1eec41d152d6d Mon Sep 17 00:00:00 2001 From: Ivan Volkov Date: Wed, 24 May 2023 18:01:07 +0300 Subject: [PATCH 07/35] Fix cfg build of virtual calls --- jacodb-analysis/build.gradle.kts | 1 - .../org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt | 1 + jacodb-core/build.gradle.kts | 1 + .../main/kotlin/org/jacodb/impl/bytecode/JcMethodImpl.kt | 2 -- .../src/main/kotlin/org/jacodb/impl/cfg/JcGraphBuilder.kt | 4 ++-- .../main/kotlin/org/jacodb/impl/cfg/RawInstListBuilder.kt | 2 +- .../main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt | 8 -------- .../kotlin/org/jacodb/testing/cfg/InstructionsTest.kt | 6 ++---- 8 files changed, 7 insertions(+), 18 deletions(-) diff --git a/jacodb-analysis/build.gradle.kts b/jacodb-analysis/build.gradle.kts index af1919b90..c4b104b97 100644 --- a/jacodb-analysis/build.gradle.kts +++ b/jacodb-analysis/build.gradle.kts @@ -17,7 +17,6 @@ dependencies { testImplementation(files("src/testFixtures/resources/pointerbench.jar")) testImplementation("joda-time:joda-time:2.12.5") - implementation("org.jetbrains.kotlinx:kotlinx-cli:0.3.5") implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1") implementation(group = "io.github.microutils", name = "kotlin-logging", version = "1.8.3") diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt index 3ec9dfc3c..ebfc9d270 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt @@ -29,6 +29,7 @@ import org.jacodb.testing.WithDB import org.joda.time.DateTime import org.junit.jupiter.api.Test +//@Disabled("Running time is more then 10 minutes") class JodaDateTimeAnalysisTest : BaseTest() { companion object : WithDB(Usages, InMemoryHierarchy) diff --git a/jacodb-core/build.gradle.kts b/jacodb-core/build.gradle.kts index 298593c23..8b9df032e 100644 --- a/jacodb-core/build.gradle.kts +++ b/jacodb-core/build.gradle.kts @@ -51,6 +51,7 @@ dependencies { implementation(group = "org.jetbrains.kotlinx", name = "kotlinx-metadata-jvm", version = kmetadataVersion) testImplementation(group = "javax.activation", name = "activation", version = "1.1") testImplementation(group = "javax.mail", name = "mail", version = "1.4.7") + testImplementation(group = "joda-time", name = "joda-time", version = "2.12.5") testFixturesImplementation(project(":jacodb-api")) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcMethodImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcMethodImpl.kt index 0ad781f7d..45879c342 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcMethodImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcMethodImpl.kt @@ -25,10 +25,8 @@ import org.jacodb.api.TypeName import org.jacodb.api.cfg.JcInst import org.jacodb.api.cfg.JcInstList import org.jacodb.api.cfg.JcRawInst -import org.jacodb.impl.softLazy import org.jacodb.impl.types.MethodInfo import org.jacodb.impl.types.TypeNameImpl -import org.jacodb.impl.weakLazy import org.objectweb.asm.tree.MethodNode class JcMethodImpl( diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcGraphBuilder.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcGraphBuilder.kt index efd3ccab8..3c01b83ee 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcGraphBuilder.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcGraphBuilder.kt @@ -434,7 +434,7 @@ class JcGraphBuilder( val instance = expr.instance.accept(this) as JcValue val args = expr.args.map { it.accept(this) as JcValue } return JcVirtualCallExpr( - instance.type.methodRef(expr), instance, args + classpath.methodRef(expr), instance, args ) } @@ -442,7 +442,7 @@ class JcGraphBuilder( val instance = expr.instance.accept(this) as JcValue val args = expr.args.map { it.accept(this) as JcValue } return JcVirtualCallExpr( - instance.type.methodRef(expr), instance, args + classpath.methodRef(expr), instance, args ) } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/RawInstListBuilder.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/RawInstListBuilder.kt index a0e1cdef0..9883387ae 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/RawInstListBuilder.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/RawInstListBuilder.kt @@ -445,7 +445,7 @@ class RawInstListBuilder( private fun local(variable: Int, expr: JcRawValue, insn: AbstractInsnNode): JcRawAssignInst? { val oldVar = currentFrame.locals[variable] return if (oldVar != null) { - if (oldVar.typeName == expr.typeName || expr is JcRawNullConstant) { + if (oldVar.typeName == expr.typeName || (expr is JcRawNullConstant && !oldVar.typeName.isPrimitive)) { JcRawAssignInst(method, oldVar, expr) } else if (expr is JcRawSimpleValue) { currentFrame = currentFrame.put(variable, expr) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt index d57c05f8a..eca1a8b04 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt @@ -160,14 +160,6 @@ fun JcClasspath.methodRef(expr: JcRawCallExpr): TypedMethodRef { return TypedMethodRefImpl(this, expr) } -fun JcType.methodRef(expr: JcRawCallExpr): TypedMethodRef { - return when (expr) { - is JcRawStaticCallExpr -> TypedStaticMethodRefImpl((this as JcClassType).classpath, expr) - is JcRawSpecialCallExpr -> TypedSpecialMethodRefImpl((this as JcClassType).classpath, expr) - else -> TypedMethodRefImpl(this, expr) - } -} - fun JcTypedMethod.methodRef(): TypedMethodRef { return TypedMethodRefImpl( enclosingType as JcClassType, diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt index 8f26b8e62..c04f69e18 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt @@ -24,8 +24,6 @@ import org.jacodb.api.JcMethod import org.jacodb.api.RegisteredLocation import org.jacodb.api.cfg.JcAssignInst import org.jacodb.api.cfg.JcLocalVar -import org.jacodb.api.cfg.locals -import org.jacodb.api.cfg.values import org.jacodb.api.ext.cfg.callExpr import org.jacodb.api.ext.cfg.locals import org.jacodb.api.ext.cfg.values @@ -119,8 +117,8 @@ class InstructionsTest : BaseTest() { @Test fun `java 5 bytecode processed correctly`() { val jars = cp.registeredLocations.map { it.path } - .filter { it.contains("mail-1.4.7.jar") || it.contains("activation-1.1.jar") } - assertEquals(2, jars.size) + .filter { it.contains("mail-1.4.7.jar") || it.contains("activation-1.1.jar") || it.contains("joda-time-2.12.5.jar") } + assertEquals(3, jars.size) val list = ConcurrentHashMap.newKeySet() runBlocking { cp.execute(object : JcClassProcessingTask { From 801c1d21ea6529f80c8fd495f6d15a95deb6bc32 Mon Sep 17 00:00:00 2001 From: Ivan Volkov Date: Wed, 24 May 2023 18:34:44 +0300 Subject: [PATCH 08/35] Disabled long-running test --- .../org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt index ebfc9d270..48bb5f2fd 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt @@ -27,9 +27,10 @@ import org.jacodb.impl.features.Usages import org.jacodb.testing.BaseTest import org.jacodb.testing.WithDB import org.joda.time.DateTime +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test -//@Disabled("Running time is more then 10 minutes") +@Disabled("Running time is more then 10 minutes") class JodaDateTimeAnalysisTest : BaseTest() { companion object : WithDB(Usages, InMemoryHierarchy) From 9c021f3f0af9a75796ebe41e779f62d4a05912bf Mon Sep 17 00:00:00 2001 From: Ivan Volkov <65076429+volivan239@users.noreply.github.com> Date: Wed, 24 May 2023 18:39:25 +0300 Subject: [PATCH 09/35] Fix swapped rhv and lhv in JcRawConditionExprs (#82) --- .../org/jacodb/impl/cfg/RawInstListBuilder.kt | 12 +++---- .../jacodb/testing/cfg/InstructionsTest.kt | 26 ++++++++++++++ .../org/jacodb/testing/cfg/Conditionals.java | 34 +++++++++++++++++++ 3 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 jacodb-core/src/testFixtures/java/org/jacodb/testing/cfg/Conditionals.java diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/RawInstListBuilder.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/RawInstListBuilder.kt index 9883387ae..fff166c0a 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/RawInstListBuilder.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/RawInstListBuilder.kt @@ -1123,12 +1123,12 @@ class RawInstListBuilder( val expr = when (opcode) { Opcodes.IFNULL -> JcRawEqExpr(boolTypeName, rhv, JcRawNull()) Opcodes.IFNONNULL -> JcRawNeqExpr(boolTypeName, rhv, JcRawNull()) - Opcodes.IFEQ -> JcRawEqExpr(boolTypeName, JcRawZero(rhv.typeName), rhv) - Opcodes.IFNE -> JcRawNeqExpr(boolTypeName, JcRawZero(rhv.typeName), rhv) - Opcodes.IFLT -> JcRawLtExpr(boolTypeName, JcRawZero(rhv.typeName), rhv) - Opcodes.IFGE -> JcRawGeExpr(boolTypeName, JcRawZero(rhv.typeName), rhv) - Opcodes.IFGT -> JcRawGtExpr(boolTypeName, JcRawZero(rhv.typeName), rhv) - Opcodes.IFLE -> JcRawLeExpr(boolTypeName, JcRawZero(rhv.typeName), rhv) + Opcodes.IFEQ -> JcRawEqExpr(boolTypeName, rhv, JcRawZero(rhv.typeName)) + Opcodes.IFNE -> JcRawNeqExpr(boolTypeName, rhv, JcRawZero(rhv.typeName)) + Opcodes.IFLT -> JcRawLtExpr(boolTypeName, rhv, JcRawZero(rhv.typeName)) + Opcodes.IFGE -> JcRawGeExpr(boolTypeName, rhv, JcRawZero(rhv.typeName)) + Opcodes.IFGT -> JcRawGtExpr(boolTypeName, rhv, JcRawZero(rhv.typeName)) + Opcodes.IFLE -> JcRawLeExpr(boolTypeName, rhv, JcRawZero(rhv.typeName)) Opcodes.IF_ICMPEQ -> JcRawEqExpr(boolTypeName, pop(), rhv) Opcodes.IF_ICMPNE -> JcRawNeqExpr(boolTypeName, pop(), rhv) Opcodes.IF_ICMPLT -> JcRawLtExpr(boolTypeName, pop(), rhv) diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt index c04f69e18..8709c579b 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt @@ -22,7 +22,12 @@ import org.jacodb.api.JcClassOrInterface import org.jacodb.api.JcClassProcessingTask import org.jacodb.api.JcMethod import org.jacodb.api.RegisteredLocation +import org.jacodb.api.cfg.JcArgument import org.jacodb.api.cfg.JcAssignInst +import org.jacodb.api.cfg.JcGeExpr +import org.jacodb.api.cfg.JcGtExpr +import org.jacodb.api.cfg.JcIfInst +import org.jacodb.api.cfg.JcInt import org.jacodb.api.cfg.JcLocalVar import org.jacodb.api.ext.cfg.callExpr import org.jacodb.api.ext.cfg.locals @@ -58,6 +63,27 @@ class InstructionsTest : BaseTest() { assertEquals("%1", (assign.rhv as JcLocalVar).name) } + @Test + fun `cmp insts`() { + val clazz = cp.findClass() + val method = clazz.declaredMethods.first { it.name == "main" } + val instructions = method.instList.instructions + val cmpExprs = instructions.filterIsInstance().map { it.condition } + assertEquals(4, cmpExprs.size) + + val geZero = cmpExprs[0] as JcGeExpr + assertEquals(0 to 0, (geZero.lhv as JcArgument).index to (geZero.rhv as JcInt).value) + + val gtZero = cmpExprs[1] as JcGtExpr + assertEquals(0 to 0, (gtZero.lhv as JcArgument).index to (gtZero.rhv as JcInt).value) + + val geOther = cmpExprs[2] as JcGeExpr + assertEquals(0 to 1, (geOther.lhv as JcArgument).index to (geOther.rhv as JcArgument).index) + + val gtOther = cmpExprs[3] as JcGtExpr + assertEquals(0 to 1, (gtOther.lhv as JcArgument).index to (gtOther.rhv as JcArgument).index) + } + @Test fun `null ref test`() { val clazz = cp.findClass() diff --git a/jacodb-core/src/testFixtures/java/org/jacodb/testing/cfg/Conditionals.java b/jacodb-core/src/testFixtures/java/org/jacodb/testing/cfg/Conditionals.java new file mode 100644 index 000000000..a8166af1a --- /dev/null +++ b/jacodb-core/src/testFixtures/java/org/jacodb/testing/cfg/Conditionals.java @@ -0,0 +1,34 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.testing.cfg; + +public class Conditionals { + void main(int x, int y) { + if (x < 0) { + System.out.println("< 0"); + } + if (x <= 0) { + System.out.println("<= 0"); + } + if (x < y) { + System.out.println("<"); + } + if (x <= y) { + System.out.println("<="); + } + } +} From 7178361a8d712e0819931a690f5015d6583daf54 Mon Sep 17 00:00:00 2001 From: Ivan Volkov Date: Wed, 24 May 2023 19:42:16 +0300 Subject: [PATCH 10/35] [jacodb-ifds] Filter inappropriate candidates in virtual calls devirtualizer * Fix tests --- .../analysis/points2/AllOverridesDevirtualizer.kt | 15 ++++++++++++--- .../org/jacodb/analysis/impl/NpeAnalysisTest.kt | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/points2/AllOverridesDevirtualizer.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/points2/AllOverridesDevirtualizer.kt index 9e05036dc..04b8ddce4 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/points2/AllOverridesDevirtualizer.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/points2/AllOverridesDevirtualizer.kt @@ -19,12 +19,14 @@ package org.jacodb.analysis.points2 import kotlinx.coroutines.runBlocking import org.jacodb.analysis.Points2Engine import org.jacodb.analysis.points2.AllOverridesDevirtualizer.Companion.bannedPackagePrefixes +import org.jacodb.api.JcClassType import org.jacodb.api.JcClasspath import org.jacodb.api.JcMethod import org.jacodb.api.analysis.JcApplicationGraph import org.jacodb.api.cfg.JcInst import org.jacodb.api.cfg.JcVirtualCallExpr import org.jacodb.api.ext.cfg.callExpr +import org.jacodb.api.ext.isSubClassOf import org.jacodb.impl.features.hierarchyExt /** @@ -42,14 +44,21 @@ class AllOverridesDevirtualizer( override fun findPossibleCallees(sink: JcInst): Collection { val methods = initialGraph.callees(sink).toList() - if (sink.callExpr !is JcVirtualCallExpr) - return methods + val callExpr = sink.callExpr as? JcVirtualCallExpr ?: return methods + val instanceClass = (callExpr.instance.type as JcClassType).jcClass + return methods .flatMap { method -> if (bannedPackagePrefixes.any { method.enclosingClass.name.startsWith(it) }) listOf(method) else { - val allOverrides = hierarchyExtension.findOverrides(method) + val allOverrides = hierarchyExtension + .findOverrides(method) + .filter { + it.enclosingClass isSubClassOf instanceClass || + // TODO: use only down-most override here + instanceClass isSubClassOf it.enclosingClass + } // TODO: maybe filter inaccessible methods here? return if (limit != null) { diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt index 6c508bad8..80d37b5b3 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt @@ -266,7 +266,7 @@ class NpeAnalysisTest : BaseTest() { @Test fun `analyse something`() { - val testingMethod = cp.findClass().declaredMethods.single { it.name == "kek" } + val testingMethod = cp.findClass().declaredMethods.single { it.name == "id" } val results = testingMethod.flowGraph() print(results) } From 91d8f0a2409fe266d44e94b8a2e7a45a57c8aa21 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 25 May 2023 11:26:59 +0300 Subject: [PATCH 11/35] build snapshots from develop branch --- .github/workflows/nightly-builds.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/nightly-builds.yml b/.github/workflows/nightly-builds.yml index 85ca1f476..9448058ac 100644 --- a/.github/workflows/nightly-builds.yml +++ b/.github/workflows/nightly-builds.yml @@ -12,6 +12,8 @@ jobs: packages: write steps: - uses: actions/checkout@v3 + with: + ref: develop - name: Set up JDK 11 uses: actions/setup-java@v3 with: From a5f87c800187bba1b886079cd93606d05f8bb293 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 25 May 2023 11:28:24 +0300 Subject: [PATCH 12/35] bring methodRef separation (special, static etc) back --- .../main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt index eca1a8b04..248c1cb26 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt @@ -157,7 +157,11 @@ data class TypedMethodRefImpl( } fun JcClasspath.methodRef(expr: JcRawCallExpr): TypedMethodRef { - return TypedMethodRefImpl(this, expr) + return when (expr) { + is JcRawStaticCallExpr -> TypedStaticMethodRefImpl((this as JcClassType).classpath, expr) + is JcRawSpecialCallExpr -> TypedSpecialMethodRefImpl((this as JcClassType).classpath, expr) + else -> TypedMethodRefImpl(this, expr) + } } fun JcTypedMethod.methodRef(): TypedMethodRef { From 57a56c1ba756d5bbbf13f0746c340966a6c5ffe1 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 25 May 2023 16:21:23 +0300 Subject: [PATCH 13/35] - review features api - fix special/static method resolutions --- .../main/kotlin/org/jacodb/api/JcClasspath.kt | 17 +--- .../performance/JcInstructionsBenchmark.kt | 16 +--- .../kotlin/org/jacodb/impl/JcClasspathImpl.kt | 67 +++++++-------- .../kotlin/org/jacodb/impl/JcDatabaseImpl.kt | 9 +- .../main/kotlin/org/jacodb/impl/JcSettings.kt | 5 ++ .../org/jacodb/impl/bytecode/Conversions.kt | 5 +- .../impl/bytecode/JcClassOrInterfaceImpl.kt | 21 ++--- .../org/jacodb/impl/bytecode/JcMethodImpl.kt | 21 ++++- .../kotlin/org/jacodb/impl/cfg/GraphExt.kt | 23 +---- ...JcGraphBuilder.kt => JcInstListBuilder.kt} | 22 +---- .../org/jacodb/impl/cfg/TypedMethodRefImpl.kt | 14 +-- .../kotlin/org/jacodb/impl/cfg/util/types.kt | 3 +- .../jacodb/impl/features/JcFeaturesChain.kt | 58 +++++++++++++ .../features/classpaths/ClasspathCache.kt | 86 +++++++++++-------- .../classpaths/MethodInstructionsFeature.kt | 54 ++++++++++++ .../classpaths/virtual/JcVirtualMethod.kt | 4 +- .../test/java/org/jacodb/testing/JavaApi.java | 44 ++++++++++ .../java/org/jacodb/testing/JavaApiTest.java | 11 --- .../kotlin/org/jacodb/testing/cfg/IRTest.kt | 8 +- 19 files changed, 305 insertions(+), 183 deletions(-) rename jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/{JcGraphBuilder.kt => JcInstListBuilder.kt} (97%) create mode 100644 jacodb-core/src/main/kotlin/org/jacodb/impl/features/JcFeaturesChain.kt create mode 100644 jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/MethodInstructionsFeature.kt create mode 100644 jacodb-core/src/test/java/org/jacodb/testing/JavaApi.java diff --git a/jacodb-api/src/main/kotlin/org/jacodb/api/JcClasspath.kt b/jacodb-api/src/main/kotlin/org/jacodb/api/JcClasspath.kt index 1ac1174af..b2674de1a 100644 --- a/jacodb-api/src/main/kotlin/org/jacodb/api/JcClasspath.kt +++ b/jacodb-api/src/main/kotlin/org/jacodb/api/JcClasspath.kt @@ -103,7 +103,7 @@ interface JcClassProcessingTask : JcClasspathTask { @JvmDefaultWithoutCompatibility interface JcClasspathFeature { - fun on(event: JcClasspathFeatureEvent) { + fun on(result: Any?, vararg input: Any) { } } @@ -145,17 +145,8 @@ interface JcInstExtFeature : JcClasspathFeature { interface JcMethodExtFeature : JcClasspathFeature { - fun flowGraph(method: JcMethod): JcGraph - fun instList(method: JcMethod): JcInstList - fun rawInstList(method: JcMethod): JcInstList + fun flowGraph(method: JcMethod): JcGraph? = null + fun instList(method: JcMethod): JcInstList? = null + fun rawInstList(method: JcMethod): JcInstList? = null } - - -fun JcClasspath.broadcast(event: JcClasspathFeatureEvent) = features?.forEach { it.on(event) } - -sealed interface JcClasspathFeatureEvent - -data class JcClassFoundEvent(val clazz: JcClassOrInterface) : JcClasspathFeatureEvent -data class JcTypeFoundEvent(val type: JcType) : JcClasspathFeatureEvent -data class JcClassNotFound(val name: String) : JcClasspathFeatureEvent \ No newline at end of file diff --git a/jacodb-benchmarks/src/test/kotlin/org/jacodb/testing/performance/JcInstructionsBenchmark.kt b/jacodb-benchmarks/src/test/kotlin/org/jacodb/testing/performance/JcInstructionsBenchmark.kt index 06fa66e04..efc7e1363 100644 --- a/jacodb-benchmarks/src/test/kotlin/org/jacodb/testing/performance/JcInstructionsBenchmark.kt +++ b/jacodb-benchmarks/src/test/kotlin/org/jacodb/testing/performance/JcInstructionsBenchmark.kt @@ -27,10 +27,8 @@ import org.jacodb.api.cfg.JcRawInst import org.jacodb.api.ext.findClass import org.jacodb.impl.JcCacheSettings import org.jacodb.impl.JcClasspathImpl -import org.jacodb.impl.cfg.nonCachedFlowGraph -import org.jacodb.impl.cfg.nonCachedInstList -import org.jacodb.impl.cfg.nonCachedRawInstList import org.jacodb.impl.features.classpaths.ClasspathCache +import org.jacodb.impl.features.classpaths.MethodInstructionsFeature import org.jacodb.impl.jacodb import org.jacodb.testing.allClasspath import org.openjdk.jmh.annotations.Benchmark @@ -57,17 +55,11 @@ class JcInstructionsBenchmark { object NoInstructionsCache : ClasspathCache(JcCacheSettings()) { - override fun instList(method: JcMethod): JcInstList { - return nonCachedInstList(method) - } + override fun instList(method: JcMethod): JcInstList? = null - override fun rawInstList(method: JcMethod): JcInstList { - return nonCachedRawInstList(method) - } + override fun rawInstList(method: JcMethod): JcInstList? = null - override fun flowGraph(method: JcMethod): JcGraph { - return nonCachedFlowGraph(method) - } + override fun flowGraph(method: JcMethod): JcGraph? = null } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt index 3419b3a3d..8d0704357 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt @@ -24,8 +24,6 @@ import kotlinx.coroutines.withContext import org.jacodb.api.ClassSource import org.jacodb.api.JcArrayType import org.jacodb.api.JcByteCodeLocation -import org.jacodb.api.JcClassFoundEvent -import org.jacodb.api.JcClassNotFound import org.jacodb.api.JcClassOrInterface import org.jacodb.api.JcClasspath import org.jacodb.api.JcClasspathExtFeature @@ -33,33 +31,32 @@ import org.jacodb.api.JcClasspathFeature import org.jacodb.api.JcClasspathTask import org.jacodb.api.JcRefType import org.jacodb.api.JcType -import org.jacodb.api.JcTypeFoundEvent import org.jacodb.api.PredefinedPrimitives import org.jacodb.api.RegisteredLocation -import org.jacodb.api.broadcast import org.jacodb.api.ext.toType import org.jacodb.api.throwClassNotFound import org.jacodb.impl.bytecode.JcClassOrInterfaceImpl +import org.jacodb.impl.features.JcFeaturesChain import org.jacodb.impl.fs.ClassSourceImpl import org.jacodb.impl.types.JcArrayTypeImpl import org.jacodb.impl.types.JcClassTypeImpl import org.jacodb.impl.types.substition.JcSubstitutor import org.jacodb.impl.vfs.ClasspathVfs import org.jacodb.impl.vfs.GlobalClassesVfs +import java.util.* class JcClasspathImpl( private val locationsRegistrySnapshot: LocationsRegistrySnapshot, override val db: JcDatabaseImpl, override val features: List, globalClassVFS: GlobalClassesVfs -) : JcClasspath { +) : JcClasspath, JcClasspathExtFeature { override val locations: List = locationsRegistrySnapshot.locations.mapNotNull { it.jcLocation } override val registeredLocations: List = locationsRegistrySnapshot.locations private val classpathVfs = ClasspathVfs(globalClassVFS, locationsRegistrySnapshot) - - private val classpathExtFeature = features.filterIsInstance() + private val featuresChain = JcFeaturesChain(features + this) override suspend fun refreshed(closeOld: Boolean): JcClasspath { return db.new(this).also { @@ -69,20 +66,34 @@ class JcClasspathImpl( } } - override fun findClassOrNull(name: String): JcClassOrInterface? { - val result = classpathExtFeature.firstNotNullOfOrNull { it.tryFindClass(this, name) } - if (result != null) { - return result.orElse(null) - } + override fun tryFindClass(classpath: JcClasspath, name: String): Optional { val source = classpathVfs.firstClassOrNull(name) val jcClass = source?.let { toJcClass(it.source) } ?: db.persistence.findClassSourceByName(this, locationsRegistrySnapshot.locations, name)?.let { toJcClass(it) } - if (jcClass == null) { - broadcast(JcClassNotFound(name)) + return Optional.ofNullable(jcClass) + } + + override fun tryFindType(classpath: JcClasspath, name: String): Optional? { + if (name.endsWith("[]")) { + val targetName = name.removeSuffix("[]") + return findTypeOrNull(targetName)?.let { + Optional.of(JcArrayTypeImpl(it, true)) + } + } + val predefined = PredefinedPrimitives.of(name, this) + if (predefined != null) { + return Optional.of(predefined) } - return jcClass + val clazz = findClassOrNull(name) ?: return Optional.empty() + return Optional.of(typeOf(clazz)) + } + + override fun findClassOrNull(name: String): JcClassOrInterface? { + return featuresChain.newRequest(name).call> { + it.tryFindClass(this, name) + }?.orElse(null) } override fun typeOf(jcClass: JcClassOrInterface): JcRefType { @@ -92,9 +103,7 @@ class JcClasspathImpl( jcClass.outerClass?.toType() as? JcClassTypeImpl, JcSubstitutor.empty, nullable = null - ).also { - broadcast(JcTypeFoundEvent(it)) - } + ) } override fun arrayTypeOf(elementType: JcType): JcArrayType { @@ -102,27 +111,13 @@ class JcClasspathImpl( } override fun toJcClass(source: ClassSource): JcClassOrInterface { - return JcClassOrInterfaceImpl(this, source, features).also { - broadcast(JcClassFoundEvent(it)) - } + return JcClassOrInterfaceImpl(this, source, featuresChain) } override fun findTypeOrNull(name: String): JcType? { - val result = classpathExtFeature.firstNotNullOfOrNull { it.tryFindType(this, name) } - if (result != null) { - return result.orElse(null) - } - if (name.endsWith("[]")) { - val targetName = name.removeSuffix("[]") - return findTypeOrNull(targetName)?.let { - JcArrayTypeImpl(it, true) - } ?: targetName.throwClassNotFound() - } - val predefined = PredefinedPrimitives.of(name, this) - if (predefined != null) { - return predefined - } - return typeOf(findClassOrNull(name) ?: return null) + return featuresChain.newRequest(name).call> { + it.tryFindType(this, name) + }?.orElse(null) } override suspend fun execute(task: T): T { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt index a06899964..61ae2d1d9 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt @@ -38,6 +38,7 @@ import org.jacodb.api.JcFeature import org.jacodb.api.RegisteredLocation import org.jacodb.impl.features.classpaths.ClasspathCache import org.jacodb.impl.features.classpaths.KotlinMetadata +import org.jacodb.impl.features.classpaths.MethodInstructionsFeature import org.jacodb.impl.fs.JavaRuntime import org.jacodb.impl.fs.asByteCodeLocation import org.jacodb.impl.fs.filterExisted @@ -88,11 +89,11 @@ class JcDatabaseImpl( ).new.process(true) } - private fun List?.appendCaching(): List { + private fun List?.appendBuiltInFeatures(): List { if (this != null && any { it is ClasspathCache }) { - return listOf(KotlinMetadata) + this + return listOf(KotlinMetadata, MethodInstructionsFeature) + this } - return listOf(ClasspathCache(settings.cacheSettings), KotlinMetadata) + this.orEmpty() + return listOf(ClasspathCache(settings.cacheSettings), KotlinMetadata, MethodInstructionsFeature) + orEmpty() } override suspend fun classpath(dirOrJars: List, features: List?): JcClasspath { @@ -107,7 +108,7 @@ class JcDatabaseImpl( return JcClasspathImpl( locationsRegistry.newSnapshot(locations), this, - features.appendCaching(), + features.appendBuiltInFeatures(), classesVfs ) } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt index b3ddc10ba..10e4cb34f 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt @@ -214,22 +214,27 @@ class JcCacheSettings { var instLists: JcCacheSegmentSettings = JcCacheSegmentSettings() var flowGraphs: JcCacheSegmentSettings = JcCacheSegmentSettings() + @JvmOverloads fun classes(maxSize: Long, expiration: Duration, valueStoreType: ValueStoreType = ValueStoreType.STRONG) = apply { classes = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) } + @JvmOverloads fun types(maxSize: Long, expiration: Duration, valueStoreType: ValueStoreType = ValueStoreType.STRONG) = apply { types = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) } + @JvmOverloads fun rawInstLists(maxSize: Long, expiration: Duration, valueStoreType: ValueStoreType = ValueStoreType.STRONG) = apply { rawInstLists = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) } + @JvmOverloads fun instLists(maxSize: Long, expiration: Duration, valueStoreType: ValueStoreType = ValueStoreType.STRONG) = apply { instLists = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) } + @JvmOverloads fun flowGraphs(maxSize: Long, expiration: Duration, valueStoreType: ValueStoreType = ValueStoreType.STRONG) = apply { flowGraphs = JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType) } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/Conversions.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/Conversions.kt index afac87761..c7f8ba97a 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/Conversions.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/Conversions.kt @@ -19,11 +19,12 @@ package org.jacodb.impl.bytecode import org.jacodb.api.JcClassOrInterface import org.jacodb.api.JcMethod import org.jacodb.api.JcMethodExtFeature +import org.jacodb.impl.features.JcFeaturesChain import org.jacodb.impl.types.MethodInfo fun JcClassOrInterface.toJcMethod( methodInfo: MethodInfo, - cache: JcMethodExtFeature + featuresChain: JcFeaturesChain ): JcMethod { - return JcMethodImpl(methodInfo, cache, this) + return JcMethodImpl(methodInfo, featuresChain, this) } \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcClassOrInterfaceImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcClassOrInterfaceImpl.kt index 55e3de09c..104b7b456 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcClassOrInterfaceImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcClassOrInterfaceImpl.kt @@ -21,12 +21,11 @@ import org.jacodb.api.JcAnnotation import org.jacodb.api.JcClassExtFeature import org.jacodb.api.JcClassOrInterface import org.jacodb.api.JcClasspath -import org.jacodb.api.JcClasspathFeature import org.jacodb.api.JcField import org.jacodb.api.JcMethod -import org.jacodb.api.JcMethodExtFeature import org.jacodb.api.ext.findClass import org.jacodb.api.ext.findMethodOrNull +import org.jacodb.impl.features.JcFeaturesChain import org.jacodb.impl.fs.ClassSourceImpl import org.jacodb.impl.fs.LazyClassSourceImpl import org.jacodb.impl.fs.fullAsmNode @@ -39,10 +38,10 @@ import kotlin.LazyThreadSafetyMode.PUBLICATION class JcClassOrInterfaceImpl( override val classpath: JcClasspath, private val classSource: ClassSource, - features: List, + private val featuresChain: JcFeaturesChain, ) : JcClassOrInterface { - private val cache = features.filterIsInstance().first() + private val hasClassFeatures = featuresChain.features.any { it is JcClassExtFeature } private val cachedInfo: ClassInfo? = when (classSource) { is LazyClassSourceImpl -> classSource.info // that means that we are loading bytecode. It can be removed let's cache info @@ -50,11 +49,9 @@ class JcClassOrInterfaceImpl( else -> null // maybe we do not need to do right now } - private val classFeatures = features.filterIsInstance() - private val extensionData by lazy(PUBLICATION) { HashMap().also { map -> - classFeatures.forEach { + featuresChain.newRequest().run { map.putAll(it.extensionValuesOf(this).orEmpty()) } } @@ -134,9 +131,9 @@ class JcClassOrInterfaceImpl( get() { val result: List = info.fields.map { JcFieldImpl(this, it) } return when { - classFeatures.isNotEmpty() -> { + hasClassFeatures -> { val modifiedFields = result.toMutableList() - classFeatures.forEach { + featuresChain.newRequest().run { it.fieldsOf(this)?.let { modifiedFields.addAll(it) } @@ -149,11 +146,11 @@ class JcClassOrInterfaceImpl( } override val declaredMethods: List by lazy(PUBLICATION) { - val result: List = info.methods.map { toJcMethod(it, cache) } + val result: List = info.methods.map { toJcMethod(it, featuresChain) } when { - classFeatures.isNotEmpty() -> { + hasClassFeatures -> { val modifiedMethods = result.toMutableList() - classFeatures.forEach { + featuresChain.newRequest().run { it.methodsOf(this)?.let { modifiedMethods.addAll(it) } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcMethodImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcMethodImpl.kt index 45879c342..02b11b26b 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcMethodImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcMethodImpl.kt @@ -22,16 +22,18 @@ import org.jacodb.api.JcMethod import org.jacodb.api.JcMethodExtFeature import org.jacodb.api.JcParameter import org.jacodb.api.TypeName +import org.jacodb.api.cfg.JcGraph import org.jacodb.api.cfg.JcInst import org.jacodb.api.cfg.JcInstList import org.jacodb.api.cfg.JcRawInst +import org.jacodb.impl.features.JcFeaturesChain import org.jacodb.impl.types.MethodInfo import org.jacodb.impl.types.TypeNameImpl import org.objectweb.asm.tree.MethodNode class JcMethodImpl( private val methodInfo: MethodInfo, - private val classpathCache: JcMethodExtFeature, + private val featuresChain: JcFeaturesChain, override val enclosingClass: JcClassOrInterface ) : JcMethod { @@ -59,11 +61,22 @@ class JcMethodImpl( return enclosingClass.asmNode().methods.first { it.name == name && it.desc == methodInfo.desc }.jsrInlined } - override val rawInstList: JcInstList get() = classpathCache.rawInstList(this) + override val rawInstList: JcInstList + get() { + return featuresChain.newRequest(this) + .call> { it.rawInstList(this) }!! + } - override fun flowGraph() = classpathCache.flowGraph(this) + override fun flowGraph(): JcGraph { + return featuresChain.newRequest(this) + .call { it.flowGraph(this) }!! + } - override val instList: JcInstList get() = classpathCache.instList(this) + + override val instList: JcInstList get() { + return featuresChain.newRequest(this) + .call> { it.instList(this) }!! + } override fun equals(other: Any?): Boolean { if (other == null || other !is JcMethodImpl) { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/GraphExt.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/GraphExt.kt index cd6219f33..ee949291d 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/GraphExt.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/GraphExt.kt @@ -338,25 +338,4 @@ open class JcExceptionResolver(val classpath: JcClasspath) : DefaultJcExprVisito } } -} - -private val JcMethod.methodFeatures - get() = enclosingClass.classpath.features?.filterIsInstance().orEmpty() - -fun nonCachedRawInstList(method: JcMethod): JcInstList { - val list: JcInstList = RawInstListBuilder(method, method.asmNode()).build() - return method.methodFeatures.fold(list) { value, feature -> - feature.transformRawInstList(method, value) - } -} - -fun nonCachedFlowGraph(method: JcMethod): JcGraph { - return JcGraphBuilder(method, method.rawInstList).buildFlowGraph() -} - -fun nonCachedInstList(method: JcMethod): JcInstList { - val list: JcInstList = JcGraphBuilder(method, method.rawInstList).buildInstList() - return method.methodFeatures.fold(list) { value, feature -> - feature.transformInstList(method, value) - } -} +} \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcGraphBuilder.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcInstListBuilder.kt similarity index 97% rename from jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcGraphBuilder.kt rename to jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcInstListBuilder.kt index 3c01b83ee..f11d0a00b 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcGraphBuilder.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcInstListBuilder.kt @@ -169,10 +169,7 @@ import org.jacodb.api.ext.short import org.jacodb.api.ext.toType /** This class stores state and is NOT THREAD SAFE. Use it carefully */ -class JcGraphBuilder( - val method: JcMethod, - val instList: JcInstList -) : JcRawInstVisitor, JcRawExprVisitor { +class JcInstListBuilder(val method: JcMethod,val instList: JcInstList) : JcRawInstVisitor, JcRawExprVisitor { val classpath: JcClasspath = method.enclosingClass.classpath private val methodRef = JcMethodRefImpl(method) @@ -194,12 +191,6 @@ class JcGraphBuilder( index = 0 } - fun buildFlowGraph(): JcGraph { - return JcGraphImpl(method, instList.mapNotNull { convertRawInst(it) }).also { - reset() - } - } - fun buildInstList(): JcInstList { return JcInstListImpl(instList.mapNotNull { convertRawInst(it) }).also { reset() @@ -433,17 +424,13 @@ class JcGraphBuilder( override fun visitJcRawVirtualCallExpr(expr: JcRawVirtualCallExpr): JcExpr { val instance = expr.instance.accept(this) as JcValue val args = expr.args.map { it.accept(this) as JcValue } - return JcVirtualCallExpr( - classpath.methodRef(expr), instance, args - ) + return JcVirtualCallExpr(classpath.methodRef(expr), instance, args) } override fun visitJcRawInterfaceCallExpr(expr: JcRawInterfaceCallExpr): JcExpr { val instance = expr.instance.accept(this) as JcValue val args = expr.args.map { it.accept(this) as JcValue } - return JcVirtualCallExpr( - classpath.methodRef(expr), instance, args - ) + return JcVirtualCallExpr(classpath.methodRef(expr), instance, args) } override fun visitJcRawStaticCallExpr(expr: JcRawStaticCallExpr): JcExpr { @@ -457,8 +444,7 @@ class JcGraphBuilder( return JcSpecialCallExpr(classpath.methodRef(expr), instance, args) } - override fun visitJcRawThis(value: JcRawThis): JcExpr = - JcThis(method.enclosingClass.toType()) + override fun visitJcRawThis(value: JcRawThis): JcExpr = JcThis(method.enclosingClass.toType()) override fun visitJcRawArgument(value: JcRawArgument): JcExpr = method.parameters[value.index].let { JcArgument.of(it.index, value.name, it.type.asType()) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt index 248c1cb26..64dfec194 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/TypedMethodRefImpl.kt @@ -51,7 +51,7 @@ interface MethodSignatureRef : TypedMethodRef { fun JcClassType.findDeclaredMethod(filter: (JcTypedMethod) -> Boolean): JcTypedMethod? { val types = argTypes.joinToString { it.typeName } - return this.declaredMethods.first { it.name == name && filter(it) && it.method.parameters.joinToString { it.type.typeName } == types } + return this.declaredMethods.firstOrNull { it.name == name && filter(it) && it.method.parameters.joinToString { it.type.typeName } == types } } val methodNotFoundMessage: String @@ -86,7 +86,9 @@ data class TypedStaticMethodRefImpl( ) override val method: JcTypedMethod by weakLazy { - findDeclaredMethod { it.isStatic } ?: throw IllegalStateException(methodNotFoundMessage) + findDeclaredMethod { it.isStatic } ?: type.superType?.let { + TypedStaticMethodRefImpl(it, name, argTypes, returnType).method + } ?: throw IllegalStateException(methodNotFoundMessage) } } @@ -105,7 +107,9 @@ data class TypedSpecialMethodRefImpl( ) override val method: JcTypedMethod by weakLazy { - findDeclaredMethod() ?: throw IllegalStateException(methodNotFoundMessage) + findDeclaredMethod() ?: type.superType?.let { + TypedSpecialMethodRefImpl(it, name, argTypes, returnType).method + } ?: throw IllegalStateException(methodNotFoundMessage) } } @@ -158,8 +162,8 @@ data class TypedMethodRefImpl( fun JcClasspath.methodRef(expr: JcRawCallExpr): TypedMethodRef { return when (expr) { - is JcRawStaticCallExpr -> TypedStaticMethodRefImpl((this as JcClassType).classpath, expr) - is JcRawSpecialCallExpr -> TypedSpecialMethodRefImpl((this as JcClassType).classpath, expr) + is JcRawStaticCallExpr -> TypedStaticMethodRefImpl(this, expr) + is JcRawSpecialCallExpr -> TypedSpecialMethodRefImpl(this, expr) else -> TypedMethodRefImpl(this, expr) } } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/util/types.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/util/types.kt index 247778220..814635f62 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/util/types.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/util/types.kt @@ -60,8 +60,7 @@ internal val TypeName.isDWord internal fun String.typeName(): TypeName = TypeNameImpl(this.jcdbName()) internal fun TypeName.asArray(dimensions: Int = 1) = "$typeName${"[]".repeat(dimensions)}".typeName() -internal fun TypeName.elementType() = elementTypeOrNull() - ?: error("Attempting to get element type of non-array type $this") +internal fun TypeName.elementType() = elementTypeOrNull() ?: this internal fun TypeName.elementTypeOrNull() = when { this == NULL -> NULL diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/JcFeaturesChain.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/JcFeaturesChain.kt new file mode 100644 index 000000000..406503705 --- /dev/null +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/JcFeaturesChain.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.impl.features + +import org.jacodb.api.JcClasspathFeature + +class JcFeaturesChain(val features: List) { + + fun newRequest(vararg input: Any) = JcFeaturesRequest(features, *input) + +} + +class JcFeaturesRequest(val features: List, vararg i: Any) { + + val input = i + + inline fun call(call: (T) -> W?): W? { + var result: W? = null + for (feature in features) { + if (feature is T) { + result = call(feature) + if (result != null) { + break + } + } + } + if (result != null) { + for (feature in features) { + feature.on(result, *input) + } + } + return result + } + + inline fun run(call: (T) -> Unit) { + for (feature in features) { + if (feature is T) { + call(feature) + } + } + + } + +} \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt index c088dcf57..a4919abea 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt @@ -17,20 +17,15 @@ package org.jacodb.impl.features.classpaths import com.google.common.cache.CacheBuilder -import com.google.common.cache.CacheLoader import com.google.common.cache.CacheStats import mu.KLogging -import org.jacodb.api.JcClassFoundEvent -import org.jacodb.api.JcClassNotFound import org.jacodb.api.JcClassOrInterface import org.jacodb.api.JcClassType import org.jacodb.api.JcClasspath import org.jacodb.api.JcClasspathExtFeature -import org.jacodb.api.JcClasspathFeatureEvent import org.jacodb.api.JcMethod import org.jacodb.api.JcMethodExtFeature import org.jacodb.api.JcType -import org.jacodb.api.JcTypeFoundEvent import org.jacodb.api.cfg.JcGraph import org.jacodb.api.cfg.JcInst import org.jacodb.api.cfg.JcInstList @@ -38,9 +33,6 @@ import org.jacodb.api.cfg.JcRawInst import org.jacodb.impl.JcCacheSegmentSettings import org.jacodb.impl.JcCacheSettings import org.jacodb.impl.ValueStoreType -import org.jacodb.impl.cfg.nonCachedFlowGraph -import org.jacodb.impl.cfg.nonCachedInstList -import org.jacodb.impl.cfg.nonCachedRawInstList import java.text.NumberFormat import java.util.* @@ -59,25 +51,13 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc .build>() private val rawInstCache = segmentBuilder(settings.rawInstLists) - .build(object : CacheLoader>() { - override fun load(key: JcMethod): JcInstList { - return nonCachedRawInstList(key) - } - }) + .build>() private val instCache = segmentBuilder(settings.instLists) - .build(object : CacheLoader>() { - override fun load(key: JcMethod): JcInstList { - return nonCachedInstList(key) - } - }) + .build>() private val cfgCache = segmentBuilder(settings.flowGraphs) - .build(object : CacheLoader() { - override fun load(key: JcMethod): JcGraph { - return nonCachedFlowGraph(key) - } - }) + .build() override fun tryFindClass(classpath: JcClasspath, name: String): Optional? { @@ -88,24 +68,52 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc return typesCache.getIfPresent(name) } - override fun flowGraph(method: JcMethod) = cfgCache.getUnchecked(method) - override fun instList(method: JcMethod) = instCache.getUnchecked(method) - override fun rawInstList(method: JcMethod) = rawInstCache.getUnchecked(method) - - override fun on(event: JcClasspathFeatureEvent) { - when (event) { - is JcClassFoundEvent -> classesCache.put(event.clazz.name, Optional.of(event.clazz)) - is JcClassNotFound -> classesCache.put(event.name, Optional.empty()) - is JcTypeFoundEvent -> { - val type = event.type - if (type is JcClassType && type.typeParameters.isEmpty()) { - typesCache.put(event.type.typeName, Optional.of(event.type)) + override fun flowGraph(method: JcMethod) = cfgCache.getIfPresent(method) + override fun instList(method: JcMethod) = instCache.getIfPresent(method) + override fun rawInstList(method: JcMethod) = rawInstCache.getIfPresent(method) + + override fun on(result: Any?, vararg input: Any) { + if (result == null) { + return + } + when (result) { + is Optional<*> -> { + if (result.isPresent) { + val found = result.get() + if (found is JcClassOrInterface) { + classesCache.put(found.name, Optional.of(found)) + } else if (found is JcClassType && found.typeParameters.isEmpty()) { + typesCache.put(found.typeName, Optional.of(found)) + } + } else { + val name = input[0] as String + classesCache.put(name, Optional.empty()) + typesCache.put(name, Optional.empty()) + } + } + + is JcGraph -> { + val method = input[0] as JcMethod + cfgCache.put(method, result) + } + + is JcInstList<*> -> { + val method = input[0] as JcMethod + if (result.instructions.isEmpty()) { + instCache.put(method, result as JcInstList) + rawInstCache.put(method, result as JcInstList) + } + if (result.instructions.first() is JcInst) { + instCache.put(method, result as JcInstList) + } else { + rawInstCache.put(method, result as JcInstList) } } } } - protected fun segmentBuilder(settings: JcCacheSegmentSettings): CacheBuilder { + protected fun segmentBuilder(settings: JcCacheSegmentSettings) + : CacheBuilder { val maxSize = settings.maxSize val expiration = settings.expiration @@ -133,7 +141,11 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc stats().entries.toList() .sortedBy { it.key } .forEach { (key, stat) -> - logger.info("$key cache hit rate: ${stat.hitRate().forPercentages()}, total count ${stat.requestCount()}") + logger.info( + "$key cache hit rate: ${ + stat.hitRate().forPercentages() + }, total count ${stat.requestCount()}" + ) } } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/MethodInstructionsFeature.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/MethodInstructionsFeature.kt new file mode 100644 index 000000000..28c2117df --- /dev/null +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/MethodInstructionsFeature.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.impl.features.classpaths + +import org.jacodb.api.JcInstExtFeature +import org.jacodb.api.JcMethod +import org.jacodb.api.JcMethodExtFeature +import org.jacodb.api.cfg.JcGraph +import org.jacodb.api.cfg.JcInst +import org.jacodb.api.cfg.JcInstList +import org.jacodb.api.cfg.JcRawInst +import org.jacodb.impl.cfg.JcGraphImpl +import org.jacodb.impl.cfg.JcInstListBuilder +import org.jacodb.impl.cfg.RawInstListBuilder + +object MethodInstructionsFeature : JcMethodExtFeature { + + private val JcMethod.methodFeatures + get() = enclosingClass.classpath.features?.filterIsInstance().orEmpty() + + + override fun flowGraph(method: JcMethod): JcGraph { + return JcGraphImpl(method, method.instList.instructions) + } + + override fun instList(method: JcMethod): JcInstList { + val list: JcInstList = JcInstListBuilder(method, method.rawInstList).buildInstList() + return method.methodFeatures.fold(list) { value, feature -> + feature.transformInstList(method, value) + } + } + + override fun rawInstList(method: JcMethod): JcInstList { + val list: JcInstList = RawInstListBuilder(method, method.asmNode()).build() + return method.methodFeatures.fold(list) { value, feature -> + feature.transformRawInstList(method, value) + } + } + +} \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/virtual/JcVirtualMethod.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/virtual/JcVirtualMethod.kt index 62a947a2e..51797d0a9 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/virtual/JcVirtualMethod.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/virtual/JcVirtualMethod.kt @@ -27,8 +27,8 @@ import org.jacodb.api.cfg.JcInst import org.jacodb.api.cfg.JcInstList import org.jacodb.api.cfg.JcRawInst import org.jacodb.impl.bytecode.JcDeclarationImpl -import org.jacodb.impl.cfg.JcGraphBuilder import org.jacodb.impl.cfg.JcInstListImpl +import org.jacodb.impl.features.classpaths.MethodInstructionsFeature import org.objectweb.asm.Opcodes import org.objectweb.asm.tree.MethodNode @@ -44,7 +44,7 @@ interface JcVirtualMethod : JcMethod { get() = JcInstListImpl(emptyList()) override fun flowGraph(): JcGraph { - return JcGraphBuilder(this, rawInstList).buildFlowGraph() + return MethodInstructionsFeature.flowGraph(this) } } diff --git a/jacodb-core/src/test/java/org/jacodb/testing/JavaApi.java b/jacodb-core/src/test/java/org/jacodb/testing/JavaApi.java new file mode 100644 index 000000000..4fa7efc94 --- /dev/null +++ b/jacodb-core/src/test/java/org/jacodb/testing/JavaApi.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.testing; + +import org.jacodb.api.cfg.JcArgument; +import org.jacodb.api.cfg.JcExpr; +import org.jacodb.api.cfg.TypedExprResolver; +import org.jacodb.impl.JcCacheSettings; +import org.jetbrains.annotations.NotNull; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; + +public class JavaApi { + private static class ArgumentResolver extends TypedExprResolver { + + @Override + public void ifMatches(@NotNull JcExpr jcExpr) { + if (jcExpr instanceof JcArgument) { + getResult().add((JcArgument) jcExpr); + } + } + + } + + public static void cacheSettings() { + new JcCacheSettings().types(10, Duration.of(1, ChronoUnit.MINUTES)); + } + +} diff --git a/jacodb-core/src/test/java/org/jacodb/testing/JavaApiTest.java b/jacodb-core/src/test/java/org/jacodb/testing/JavaApiTest.java index 92eacb9ad..049fe3ab1 100644 --- a/jacodb-core/src/test/java/org/jacodb/testing/JavaApiTest.java +++ b/jacodb-core/src/test/java/org/jacodb/testing/JavaApiTest.java @@ -37,17 +37,6 @@ public class JavaApiTest { - private static class ArgumentResolver extends TypedExprResolver { - - @Override - public void ifMatches(@NotNull JcExpr jcExpr) { - if (jcExpr instanceof JcArgument) { - getResult().add((JcArgument) jcExpr); - } - } - - } - @Test public void createJcdb() throws ExecutionException, InterruptedException, IOException { System.out.println("Creating database"); diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt index e447562f9..bb4e608de 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt @@ -47,6 +47,7 @@ import org.jacodb.api.cfg.JcVirtualCallExpr import org.jacodb.api.cfg.applyAndGet import org.jacodb.api.ext.HierarchyExtension import org.jacodb.api.ext.findClass +import org.jacodb.api.ext.findMethodOrNull import org.jacodb.api.ext.isKotlin import org.jacodb.api.ext.packageName import org.jacodb.api.ext.toType @@ -56,7 +57,7 @@ import org.jacodb.impl.bytecode.JcClassOrInterfaceImpl import org.jacodb.impl.bytecode.JcDatabaseClassWriter import org.jacodb.impl.bytecode.JcMethodImpl import org.jacodb.impl.cfg.JcBlockGraphImpl -import org.jacodb.impl.cfg.JcGraphBuilder +import org.jacodb.impl.cfg.JcInstListBuilder import org.jacodb.impl.cfg.MethodNodeBuilder import org.jacodb.impl.cfg.RawInstListBuilder import org.jacodb.impl.cfg.Simplifier @@ -319,7 +320,8 @@ class IRTest : BaseTest() { testClass(cp.findClass.BinarySearchTreeIterator>()) } - @Test + + @Test fun `get ir of self`() { testClass(cp.findClass()) testClass(cp.findClass()) @@ -328,7 +330,7 @@ class IRTest : BaseTest() { testClass(cp.findClass()) testClass(cp.findClass()) testClass(cp.findClass()) - testClass(cp.findClass()) + testClass(cp.findClass()) testClass(cp.findClass()) } From 32ce228ba5b1a243cfb30de74e103cd2ebb4b3b3 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 25 May 2023 16:29:09 +0300 Subject: [PATCH 14/35] fix tests --- .../kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt index a4919abea..3fc52db90 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt @@ -102,6 +102,7 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc if (result.instructions.isEmpty()) { instCache.put(method, result as JcInstList) rawInstCache.put(method, result as JcInstList) + return } if (result.instructions.first() is JcInst) { instCache.put(method, result as JcInstList) From 11c889e85f5c5b2e1304587e9db932bd77fdf10e Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 25 May 2023 16:45:54 +0300 Subject: [PATCH 15/35] tests for caching --- .../jacodb/testing/tests/DatabaseEnvTest.kt | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/tests/DatabaseEnvTest.kt b/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/tests/DatabaseEnvTest.kt index 9027a87d5..007a333d8 100644 --- a/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/tests/DatabaseEnvTest.kt +++ b/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/tests/DatabaseEnvTest.kt @@ -39,6 +39,7 @@ import org.jacodb.api.ext.isNullable import org.jacodb.api.ext.jcdbSignature import org.jacodb.api.ext.jvmSignature import org.jacodb.api.ext.methods +import org.jacodb.impl.features.classpaths.ClasspathCache import org.jacodb.impl.features.classpaths.VirtualClassContent import org.jacodb.impl.features.classpaths.VirtualClasses import org.jacodb.impl.features.classpaths.virtual.JcVirtualClass @@ -483,7 +484,28 @@ abstract class DatabaseEnvTest { method.parameters.forEach { assertNotNull(it.method) } + } + @Test + fun `class caching feature works for not existed class`() { + val notExistedClass = "xxx.Xxx" + val clazz = cp.findClassOrNull(notExistedClass) + assertNull(clazz) + val cache = cp.features?.first { it is ClasspathCache } as ClasspathCache + val optional = cache.tryFindClass(cp, notExistedClass) + assertNotNull(optional) + assertFalse(optional!!.isPresent) + } + + @Test + fun `class caching feature works for existed class`() { + val existedClass = "java.lang.String" + val clazz = cp.findClassOrNull(existedClass) + assertNotNull(clazz) + val cache = cp.features?.first { it is ClasspathCache } as ClasspathCache + val optional = cache.tryFindClass(cp, existedClass) + assertNotNull(optional) + assertTrue(optional!!.isPresent) } private inline fun findSubClasses(allHierarchy: Boolean = false): Sequence { From f8be865b51e6a656fd7388506dd06bf774af05ab Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 25 May 2023 18:30:07 +0300 Subject: [PATCH 16/35] fix tests --- .../jacodb/testing/cfg/InstructionsTest.kt | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt index 8709c579b..20556795f 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/InstructionsTest.kt @@ -18,6 +18,7 @@ package org.jacodb.testing.cfg import com.sun.mail.imap.IMAPMessage import kotlinx.coroutines.runBlocking +import mu.KLogging import org.jacodb.api.JcClassOrInterface import org.jacodb.api.JcClassProcessingTask import org.jacodb.api.JcMethod @@ -38,6 +39,7 @@ import org.jacodb.testing.WithDB import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.DisabledOnJre import org.junit.jupiter.api.condition.EnabledOnJre import org.junit.jupiter.api.condition.JRE import org.objectweb.asm.util.Textifier @@ -123,7 +125,7 @@ class InstructionsTest : BaseTest() { val clazz = cp.findClass() with(clazz.declaredMethods.first { it.name == "sortTimes" }) { assertEquals(9, instList.locals.size) - assertEquals(13, instList.values .size) + assertEquals(13, instList.values.size) } with(clazz.declaredMethods.first { it.name == "test" }) { @@ -141,10 +143,21 @@ class InstructionsTest : BaseTest() { } @Test - fun `java 5 bytecode processed correctly`() { + @EnabledOnJre(JRE.JAVA_8) + fun `java 5 bytecode processed correctly on java 8`() { + runAlong("mail-1.4.7.jar", "joda-time-2.12.5.jar") + } + + @Test + @DisabledOnJre(JRE.JAVA_8) + fun `java 5 bytecode processed correctly on java 9+`() { + runAlong("mail-1.4.7.jar", "activation-1.1.jar", "joda-time-2.12.5.jar") + } + + private fun runAlong(vararg patters: String) { val jars = cp.registeredLocations.map { it.path } - .filter { it.contains("mail-1.4.7.jar") || it.contains("activation-1.1.jar") || it.contains("joda-time-2.12.5.jar") } - assertEquals(3, jars.size) + .filter { patters.any { pattern -> it.contains(pattern) } } + assertEquals(patters.size, jars.size) val list = ConcurrentHashMap.newKeySet() runBlocking { cp.execute(object : JcClassProcessingTask { @@ -163,6 +176,7 @@ class InstructionsTest : BaseTest() { try { it.flowGraph() } catch (e: Exception) { + KLogging().logger.error(e) { "can't process $it" } failed.add(it) } } @@ -172,7 +186,6 @@ class InstructionsTest : BaseTest() { "Failed to process methods: \n${failed.joinToString("\n") { it.enclosingClass.name + "#" + it.name }}" ) } - } fun JcMethod.dumpInstructions(): String { @@ -191,4 +204,4 @@ private fun List<*>.printList(builder: StringBuilder) { builder.append(it.toString()) } } -} +} \ No newline at end of file From d740896fd6ccdffb470557d116576c1a0e1b8c7a Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 25 May 2023 20:27:06 +0300 Subject: [PATCH 17/35] - fix tests - fix situation when there are not unique locations in classpath get method --- .../src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt | 2 +- .../org/jacodb/impl/storage/PersistentLocationRegistry.kt | 5 +++-- .../org/jacodb/testing/persistence/RestoredDBTest.kt | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt index 61ae2d1d9..7620c2dad 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt @@ -91,7 +91,7 @@ class JcDatabaseImpl( private fun List?.appendBuiltInFeatures(): List { if (this != null && any { it is ClasspathCache }) { - return listOf(KotlinMetadata, MethodInstructionsFeature) + this + return this + listOf(KotlinMetadata, MethodInstructionsFeature) } return listOf(ClasspathCache(settings.cacheSettings), KotlinMetadata, MethodInstructionsFeature) + orEmpty() } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/PersistentLocationRegistry.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/PersistentLocationRegistry.kt index 401750cca..045a3f86d 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/PersistentLocationRegistry.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/PersistentLocationRegistry.kt @@ -89,15 +89,16 @@ class PersistentLocationRegistry(private val jcdb: JcDatabase, private val featu } override fun registerIfNeeded(locations: List): RegistrationResult { + val uniqueLocations = locations.toSet() return persistence.write { val result = arrayListOf() val toAdd = arrayListOf() - val fsId = locations.map { it.fileSystemId } + val fsId = uniqueLocations.map { it.fileSystemId } val existed = it.selectFrom(BYTECODELOCATIONS) .where(BYTECODELOCATIONS.UNIQUEID.`in`(fsId)) .fetch().associateBy { it.uniqueid } - locations.forEach { + uniqueLocations.forEach { val found = existed[it.fileSystemId] if (found == null) { toAdd += it diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/persistence/RestoredDBTest.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/persistence/RestoredDBTest.kt index 50d598fb6..0e088e68c 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/persistence/RestoredDBTest.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/persistence/RestoredDBTest.kt @@ -29,14 +29,14 @@ class RestoredDBTest : DatabaseEnvTest() { companion object : WithRestoredDB() - override val cp: JcClasspath - get() = runBlocking { + override val cp: JcClasspath by lazy { + runBlocking { val withDB = this@RestoredDBTest.javaClass.withDB withDB.db.classpath(allClasspath) } + } - override val hierarchyExt: HierarchyExtension - get() = runBlocking { cp.hierarchyExt() } + override val hierarchyExt: HierarchyExtension by lazy { runBlocking { cp.hierarchyExt() } } } From 78e7049ba9633663a69eb469db6b07d6de13d3bc Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Fri, 26 May 2023 10:52:57 +0300 Subject: [PATCH 18/35] fix too many puts in cache --- .../main/kotlin/org/jacodb/api/JcClasspath.kt | 10 ++- .../kotlin/org/jacodb/impl/JcClasspathImpl.kt | 66 +++++++++++-------- .../jacodb/impl/features/JcFeaturesChain.kt | 22 +++++-- .../features/classpaths/ClasspathCache.kt | 8 +-- 4 files changed, 67 insertions(+), 39 deletions(-) diff --git a/jacodb-api/src/main/kotlin/org/jacodb/api/JcClasspath.kt b/jacodb-api/src/main/kotlin/org/jacodb/api/JcClasspath.kt index b2674de1a..5cfcf640d 100644 --- a/jacodb-api/src/main/kotlin/org/jacodb/api/JcClasspath.kt +++ b/jacodb-api/src/main/kotlin/org/jacodb/api/JcClasspath.kt @@ -103,9 +103,17 @@ interface JcClassProcessingTask : JcClasspathTask { @JvmDefaultWithoutCompatibility interface JcClasspathFeature { - fun on(result: Any?, vararg input: Any) { + fun on(event: JcFeatureEvent) { } + fun event(result: Any, input: Array): JcFeatureEvent? = null + +} + +interface JcFeatureEvent { + val feature: JcClasspathFeature + val result: Any + val input: Array } @JvmDefaultWithoutCompatibility diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt index 8d0704357..bd8132070 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt @@ -29,6 +29,7 @@ import org.jacodb.api.JcClasspath import org.jacodb.api.JcClasspathExtFeature import org.jacodb.api.JcClasspathFeature import org.jacodb.api.JcClasspathTask +import org.jacodb.api.JcFeatureEvent import org.jacodb.api.JcRefType import org.jacodb.api.JcType import org.jacodb.api.PredefinedPrimitives @@ -36,6 +37,7 @@ import org.jacodb.api.RegisteredLocation import org.jacodb.api.ext.toType import org.jacodb.api.throwClassNotFound import org.jacodb.impl.bytecode.JcClassOrInterfaceImpl +import org.jacodb.impl.features.JcFeatureEventImpl import org.jacodb.impl.features.JcFeaturesChain import org.jacodb.impl.fs.ClassSourceImpl import org.jacodb.impl.types.JcArrayTypeImpl @@ -50,13 +52,13 @@ class JcClasspathImpl( override val db: JcDatabaseImpl, override val features: List, globalClassVFS: GlobalClassesVfs -) : JcClasspath, JcClasspathExtFeature { +) : JcClasspath { override val locations: List = locationsRegistrySnapshot.locations.mapNotNull { it.jcLocation } override val registeredLocations: List = locationsRegistrySnapshot.locations private val classpathVfs = ClasspathVfs(globalClassVFS, locationsRegistrySnapshot) - private val featuresChain = JcFeaturesChain(features + this) + private val featuresChain = JcFeaturesChain(features + JcClasspathFeatureImpl()) override suspend fun refreshed(closeOld: Boolean): JcClasspath { return db.new(this).also { @@ -66,30 +68,6 @@ class JcClasspathImpl( } } - override fun tryFindClass(classpath: JcClasspath, name: String): Optional { - val source = classpathVfs.firstClassOrNull(name) - val jcClass = source?.let { toJcClass(it.source) } - ?: db.persistence.findClassSourceByName(this, locationsRegistrySnapshot.locations, name)?.let { - toJcClass(it) - } - return Optional.ofNullable(jcClass) - } - - override fun tryFindType(classpath: JcClasspath, name: String): Optional? { - if (name.endsWith("[]")) { - val targetName = name.removeSuffix("[]") - return findTypeOrNull(targetName)?.let { - Optional.of(JcArrayTypeImpl(it, true)) - } - } - val predefined = PredefinedPrimitives.of(name, this) - if (predefined != null) { - return Optional.of(predefined) - } - val clazz = findClassOrNull(name) ?: return Optional.empty() - return Optional.of(typeOf(clazz)) - } - override fun findClassOrNull(name: String): JcClassOrInterface? { return featuresChain.newRequest(name).call> { it.tryFindClass(this, name) @@ -148,4 +126,38 @@ class JcClasspathImpl( locationsRegistrySnapshot.close() } -} \ No newline at end of file + private inner class JcClasspathFeatureImpl: JcClasspathExtFeature{ + + override fun tryFindClass(classpath: JcClasspath, name: String): Optional { + val source = classpathVfs.firstClassOrNull(name) + val jcClass = source?.let { toJcClass(it.source) } + ?: db.persistence.findClassSourceByName(this@JcClasspathImpl, locationsRegistrySnapshot.locations, name)?.let { + toJcClass(it) + } + return Optional.ofNullable(jcClass) + } + + override fun tryFindType(classpath: JcClasspath, name: String): Optional? { + if (name.endsWith("[]")) { + val targetName = name.removeSuffix("[]") + return findTypeOrNull(targetName)?.let { + Optional.of(JcArrayTypeImpl(it, true)) + } + } + val predefined = PredefinedPrimitives.of(name, this@JcClasspathImpl) + if (predefined != null) { + return Optional.of(predefined) + } + val clazz = findClassOrNull(name) ?: return Optional.empty() + return Optional.of(typeOf(clazz)) + } + + override fun event(result: Any, input: Array): JcFeatureEvent { + return JcFeatureEventImpl(this, result, input) + } + + } + +} + + diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/JcFeaturesChain.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/JcFeaturesChain.kt index 406503705..b4a281967 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/JcFeaturesChain.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/JcFeaturesChain.kt @@ -17,30 +17,31 @@ package org.jacodb.impl.features import org.jacodb.api.JcClasspathFeature +import org.jacodb.api.JcFeatureEvent class JcFeaturesChain(val features: List) { - fun newRequest(vararg input: Any) = JcFeaturesRequest(features, *input) + fun newRequest(vararg input: Any) = JcFeaturesRequest(features, arrayOf(*input)) } -class JcFeaturesRequest(val features: List, vararg i: Any) { - - val input = i +class JcFeaturesRequest(val features: List, val input: Array) { inline fun call(call: (T) -> W?): W? { var result: W? = null + var event: JcFeatureEvent? = null for (feature in features) { if (feature is T) { result = call(feature) if (result != null) { + event = feature.event(result, input) break } } } - if (result != null) { + if (result != null && event != null) { for (feature in features) { - feature.on(result, *input) + feature.on(event) } } return result @@ -55,4 +56,11 @@ class JcFeaturesRequest(val features: List, vararg i: Any) { } -} \ No newline at end of file +} + + +class JcFeatureEventImpl( + override val feature: JcClasspathFeature, + override val result: Any, + override val input: Array +) : JcFeatureEvent \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt index 3fc52db90..cee2e169e 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/ClasspathCache.kt @@ -23,6 +23,7 @@ import org.jacodb.api.JcClassOrInterface import org.jacodb.api.JcClassType import org.jacodb.api.JcClasspath import org.jacodb.api.JcClasspathExtFeature +import org.jacodb.api.JcFeatureEvent import org.jacodb.api.JcMethod import org.jacodb.api.JcMethodExtFeature import org.jacodb.api.JcType @@ -72,10 +73,9 @@ open class ClasspathCache(settings: JcCacheSettings) : JcClasspathExtFeature, Jc override fun instList(method: JcMethod) = instCache.getIfPresent(method) override fun rawInstList(method: JcMethod) = rawInstCache.getIfPresent(method) - override fun on(result: Any?, vararg input: Any) { - if (result == null) { - return - } + override fun on(event: JcFeatureEvent) { + val result = event.result + val input = event.input when (result) { is Optional<*> -> { if (result.isPresent) { From 59291954baea04ed48e6a1bd1c659cbf05519c1d Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Fri, 26 May 2023 10:59:12 +0300 Subject: [PATCH 19/35] fix missed cfg/instructions caching --- .../impl/features/classpaths/MethodInstructionsFeature.kt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/MethodInstructionsFeature.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/MethodInstructionsFeature.kt index 28c2117df..e675f3558 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/MethodInstructionsFeature.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/classpaths/MethodInstructionsFeature.kt @@ -16,6 +16,7 @@ package org.jacodb.impl.features.classpaths +import org.jacodb.api.JcFeatureEvent import org.jacodb.api.JcInstExtFeature import org.jacodb.api.JcMethod import org.jacodb.api.JcMethodExtFeature @@ -26,6 +27,7 @@ import org.jacodb.api.cfg.JcRawInst import org.jacodb.impl.cfg.JcGraphImpl import org.jacodb.impl.cfg.JcInstListBuilder import org.jacodb.impl.cfg.RawInstListBuilder +import org.jacodb.impl.features.JcFeatureEventImpl object MethodInstructionsFeature : JcMethodExtFeature { @@ -51,4 +53,8 @@ object MethodInstructionsFeature : JcMethodExtFeature { } } + override fun event(result: Any, input: Array): JcFeatureEvent { + return JcFeatureEventImpl(this, result, input) + } + } \ No newline at end of file From 2b84a8697ab43b63a0f3aef90062382f994959f0 Mon Sep 17 00:00:00 2001 From: Ivan Volkov Date: Thu, 25 May 2023 23:46:28 +0300 Subject: [PATCH 20/35] [jacodb-ifds] Improve cli api + test for cli --- jacodb-analysis/build.gradle.kts | 1 + .../org/jacodb/analysis/AnalysisMain.kt | 130 ++++++++++++------ .../org/jacodb/analysis/impl/CliTest.kt | 32 +++++ .../jacodb/analysis/impl/NpeAnalysisTest.kt | 2 +- .../src/test/resources/config.json | 6 + .../jacodb/analysis/samples}/NPEExamples.java | 2 + 6 files changed, 129 insertions(+), 44 deletions(-) create mode 100644 jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/CliTest.kt create mode 100644 jacodb-analysis/src/test/resources/config.json rename jacodb-analysis/src/testFixtures/java/{ => org/jacodb/analysis/samples}/NPEExamples.java (99%) diff --git a/jacodb-analysis/build.gradle.kts b/jacodb-analysis/build.gradle.kts index c4b104b97..094ccc467 100644 --- a/jacodb-analysis/build.gradle.kts +++ b/jacodb-analysis/build.gradle.kts @@ -3,6 +3,7 @@ val coroutinesVersion: String by rootProject plugins { `java-test-fixtures` + kotlin("plugin.serialization") version "1.7.20" } dependencies { diff --git a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/AnalysisMain.kt b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/AnalysisMain.kt index b21054cdd..4bf9b09c6 100644 --- a/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/AnalysisMain.kt +++ b/jacodb-analysis/src/main/kotlin/org/jacodb/analysis/AnalysisMain.kt @@ -17,9 +17,11 @@ package org.jacodb.analysis import kotlinx.cli.ArgParser import kotlinx.cli.ArgType +import kotlinx.cli.default import kotlinx.cli.required import kotlinx.coroutines.runBlocking import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString import kotlinx.serialization.json.Json import kotlinx.serialization.json.encodeToStream import mu.KLogging @@ -39,6 +41,7 @@ import org.jacodb.api.JcClasspath import org.jacodb.api.JcMethod import org.jacodb.api.analysis.JcApplicationGraph import org.jacodb.api.cfg.JcInst +import org.jacodb.api.ext.findClass import org.jacodb.impl.features.InMemoryHierarchy import org.jacodb.impl.features.Usages import org.jacodb.impl.features.usagesExt @@ -57,6 +60,11 @@ data class VulnerabilityInstance( @Serializable data class DumpableAnalysisResult(val foundVulnerabilities: List) +typealias AnalysesOptions = Map + +@Serializable +data class AnalysisConfig(val analyses: Map) + interface AnalysisEngine { fun analyze(): DumpableAnalysisResult fun addStart(method: JcMethod) @@ -127,19 +135,6 @@ interface Points2EngineFactory : Factory { interface GraphFactory : Factory { fun createGraph(classpath: JcClasspath): JcApplicationGraph - - fun createGraph(classpath: List, cacheDir: File): JcApplicationGraph = runBlocking { - val classpathHash = classpath.toString().hashCode() - val persistentPath = cacheDir.resolve("jacodb-for-$classpathHash") - - val jcdb = jacodb { - loadByteCode(classpath) - persistent(persistentPath.absolutePath) - installFeatures(InMemoryHierarchy, Usages) - } - val cp = jcdb.classpath(classpath) - createGraph(cp) - } } class JcSimplifiedGraphFactory( @@ -191,44 +186,63 @@ class AnalysisMain { fun run(args: List) = main(args.toTypedArray()) } +fun loadAnalysisEngineFactoriesByConfig(config: AnalysisConfig): List { + return config.analyses.mapNotNull { (analysis, _) -> + when (analysis) { + "NPE" -> NPEAnalysisFactory() + "Unused" -> UnusedVariableAnalysisFactory() + else -> { + logger.error { "Unknown analysis type: $analysis" } + null + } + } + } +} + fun main(args: Array) { val parser = ArgParser("taint-analysis") - val classpath by parser.option( + val configFilePath by parser.option( ArgType.String, - fullName = "classpath", - shortName = "cp", - description = "Classpath for analysis. Used by JacoDB." + fullName = "analysisConf", + shortName = "a", + description = "File with analysis configuration in JSON format" + ).required() + val cacheDirPath by parser.option( + ArgType.String, + fullName = "cachedir", + shortName = "c", + description = "Directory with caches for analysis. All parent directories will be created if not exists. Directory will be created if not exists. Directory must be empty." + ).required() + val startClasses by parser.option( + ArgType.String, + fullName = "start", + shortName = "s", + description = "classes from which to start the analysis" ).required() + val outputPath by parser.option( + ArgType.String, + fullName = "output", + shortName = "o", + description = "File where analysis report will be written. All parent directories will be created if not exists. File will be created if not exists. Existing file will be overwritten." + ).default("report.txt") // TODO: create SARIF here val graphFactory by parser.option( factoryChoice(), fullName = "graph-type", shortName = "g", description = "Type of code graph to be used by analysis." - ).required() - val engineFactory by parser.option( - factoryChoice(), - fullName = "engine", - shortName = "e", - description = "Type of IFDS engine." - ).required() + ).default(JcSimplifiedGraphFactory()) val points2Factory by parser.option( factoryChoice(), fullName = "points2", shortName = "p2", description = "Type of points-to engine." - ).required() - val cacheDirPath by parser.option( - ArgType.String, - fullName = "cache-directory", - shortName = "c", - description = "Directory with caches for analysis. All parent directories will be created if not exists. Directory will be created if not exists. Directory must be empty." - ).required() - val outputPath by parser.option( + ).default(JcNaivePoints2EngineFactory()) + val classpath by parser.option( ArgType.String, - fullName = "output", - shortName = "o", - description = "File where analysis report will be written. All parent directories will be created if not exists. File will be created if not exists. Existing file will be overwritten." - ).required() + fullName = "classpath", + shortName = "cp", + description = "Classpath for analysis. Used by JacoDB." + ).default(System.getProperty("java.class.path")) parser.parse(args) @@ -238,8 +252,6 @@ fun main(args: Array) { throw IllegalArgumentException("Provided path for output file is directory, please provide correct path") } else if (outputFile.exists()) { logger.info { "Output file $outputFile already exists, results will be overwritten" } - } else { - outputFile.parentFile.mkdirs() } val cacheDir = File(cacheDirPath) @@ -252,14 +264,46 @@ fun main(args: Array) { throw IllegalArgumentException("Provided path to cache directory is not directory") } + val configFile = File(configFilePath) + if (!configFile.isFile) { + throw IllegalArgumentException("Can't find provided config file $configFilePath") + } + val config = Json.decodeFromString(configFile.readText()) + val classpathAsFiles = classpath.split(File.pathSeparatorChar).sorted().map { File(it) } - val graph = graphFactory.createGraph(classpathAsFiles, cacheDir) + val classpathHash = classpath.hashCode() + val persistentPath = cacheDir.resolve("jacodb-for-$classpathHash") + + val cp = runBlocking { + val jacodb = jacodb { + loadByteCode(classpathAsFiles) + persistent(persistentPath.absolutePath) + installFeatures(InMemoryHierarchy, Usages) + } + jacodb.classpath(classpathAsFiles) + } + + val graph = graphFactory.createGraph(cp) val points2Engine = points2Factory.createPoints2Engine(graph) - val analysisEngine = engineFactory.createAnalysisEngine(graph, points2Engine) - val analysisResult = analysisEngine.analyze() - val json = Json { prettyPrint = true } + val startJcClasses = startClasses.split(";").map { cp.findClass(it) } + val analysisEngines = loadAnalysisEngineFactoriesByConfig(config).map { + it.createAnalysisEngine(graph, points2Engine) + } + + val analysisResults = analysisEngines.map { engine -> + startJcClasses.forEach { clazz -> + clazz.declaredMethods.forEach { + engine.addStart(it) + } + } + engine.analyze() + } + + val mergedResult = DumpableAnalysisResult(analysisResults.flatMap { it.foundVulnerabilities }) + + val json = Json { prettyPrint = true } outputFile.outputStream().use { fileOutputStream -> - json.encodeToStream(analysisResult, fileOutputStream) + json.encodeToStream(mergedResult, fileOutputStream) } } \ No newline at end of file diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/CliTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/CliTest.kt new file mode 100644 index 000000000..ccb166879 --- /dev/null +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/CliTest.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.analysis.impl + +import org.jacodb.analysis.AnalysisMain +import org.junit.jupiter.api.Test + +class CliTest { + @Test + fun `test basic analysis cli api`() { + val args = listOf( + "-a", CliTest::class.java.getResource("/config.json")?.file ?: error("Can't find file with config"), + "-c", "tmp-analysis-db", + "-s", "org.jacodb.analysis.samples.NPEExamples" + ) + AnalysisMain().run(args) + } +} \ No newline at end of file diff --git a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt index 80d37b5b3..40738de7f 100644 --- a/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt +++ b/jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NpeAnalysisTest.kt @@ -16,7 +16,6 @@ package org.jacodb.analysis.impl -import NPEExamples import juliet.testcasesupport.AbstractTestCase import kotlinx.coroutines.runBlocking import org.jacodb.analysis.JcNaivePoints2EngineFactory @@ -25,6 +24,7 @@ import org.jacodb.analysis.NPEAnalysisFactory import org.jacodb.analysis.VulnerabilityInstance import org.jacodb.analysis.analyzers.NpeAnalyzer import org.jacodb.analysis.graph.JcApplicationGraphImpl +import org.jacodb.analysis.samples.NPEExamples import org.jacodb.api.JcClassOrInterface import org.jacodb.api.JcMethod import org.jacodb.api.ext.constructors diff --git a/jacodb-analysis/src/test/resources/config.json b/jacodb-analysis/src/test/resources/config.json new file mode 100644 index 000000000..f56166da3 --- /dev/null +++ b/jacodb-analysis/src/test/resources/config.json @@ -0,0 +1,6 @@ +{ + "analyses": { + "NPE": {}, + "Unused": {} + } +} \ No newline at end of file diff --git a/jacodb-analysis/src/testFixtures/java/NPEExamples.java b/jacodb-analysis/src/testFixtures/java/org/jacodb/analysis/samples/NPEExamples.java similarity index 99% rename from jacodb-analysis/src/testFixtures/java/NPEExamples.java rename to jacodb-analysis/src/testFixtures/java/org/jacodb/analysis/samples/NPEExamples.java index 84a10e4ee..1ded46c2e 100644 --- a/jacodb-analysis/src/testFixtures/java/NPEExamples.java +++ b/jacodb-analysis/src/testFixtures/java/org/jacodb/analysis/samples/NPEExamples.java @@ -14,6 +14,8 @@ * limitations under the License. */ +package org.jacodb.analysis.samples; + import org.jetbrains.annotations.NotNull; @SuppressWarnings("ALL") From 24339b0ce6fac8dcac630610e7567cef2562b08c Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Fri, 26 May 2023 13:23:48 +0300 Subject: [PATCH 21/35] better equality of JcClassTypes --- .../org/jacodb/impl/types/JcClassTypeImpl.kt | 7 ++--- .../types/substition/JcSubstitutorImpl.kt | 14 ++++++++++ .../org/jacodb/testing/types/TypesTest.kt | 28 +++++++++++++++++-- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/types/JcClassTypeImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/types/JcClassTypeImpl.kt index b9d573f1d..e63b149e9 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/types/JcClassTypeImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/types/JcClassTypeImpl.kt @@ -160,14 +160,13 @@ open class JcClassTypeImpl( other as JcClassTypeImpl if (nullable != other.nullable) return false - if (typeName != other.typeName) return false - - return true + if (jcClass != other.jcClass) return false + return substitutor == other.substitutor } override fun hashCode(): Int { val result = nullable.hashCode() - return 31 * result + typeName.hashCode() + return 31 * result + jcClass.hashCode() } private fun typedMethods( diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/types/substition/JcSubstitutorImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/types/substition/JcSubstitutorImpl.kt index 7b33d6915..520a54062 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/types/substition/JcSubstitutorImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/types/substition/JcSubstitutorImpl.kt @@ -142,4 +142,18 @@ class JcSubstitutorImpl( } } + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as JcSubstitutorImpl + + return substitutions == other.substitutions + } + + override fun hashCode(): Int { + return substitutions.hashCode() + } + + } diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/types/TypesTest.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/types/TypesTest.kt index d26de1dd4..d72e8187b 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/types/TypesTest.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/types/TypesTest.kt @@ -16,14 +16,15 @@ package org.jacodb.testing.types -import com.zaxxer.hikari.pool.HikariPool -import com.zaxxer.hikari.util.ConcurrentBag import org.jacodb.api.JcArrayType import org.jacodb.api.JcPrimitiveType import org.jacodb.api.JcTypeVariable import org.jacodb.api.ext.findClass import org.jacodb.api.ext.toType +import org.jacodb.impl.types.JcClassTypeImpl +import org.jacodb.impl.types.substition.JcSubstitutor import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -100,4 +101,27 @@ class TypesTest : BaseTypesTest() { assertEquals(secondParam.jcClass, cp.findClass("sun.security.ssl.SSLSessionImpl")) } + private val listClass = List::class.java.name + + @Test + fun `raw types equality`() { + val rawType1 = JcClassTypeImpl(cp, listClass, null, JcSubstitutor.empty, false) + val rawType2 = JcClassTypeImpl(cp, listClass, null, JcSubstitutor.empty, false) + assertEquals(rawType1, rawType2) + } + + interface X : List + interface Y : List + + @Test + fun `parametrized types equality`() { + val rawType = JcClassTypeImpl(cp, listClass, null, JcSubstitutor.empty, false) + val type1 = cp.findClass().toType().interfaces.first() + val type2 = cp.findClass().toType().interfaces.first() + assertNotEquals(rawType, type1) + assertNotEquals(rawType, type2) + + assertNotEquals(type1, type2) + } + } \ No newline at end of file From fa6f811f639d9d8cb398901df590016cf8062537 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Fri, 26 May 2023 15:43:38 +0300 Subject: [PATCH 22/35] Add ability to override real method closes #85 --- .../impl/bytecode/JcClassOrInterfaceImpl.kt | 40 ++++++++--------- .../jacodb/testing/tests/DatabaseEnvTest.kt | 43 +++++++++++++++++++ 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcClassOrInterfaceImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcClassOrInterfaceImpl.kt index 104b7b456..e6da3d751 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcClassOrInterfaceImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/JcClassOrInterfaceImpl.kt @@ -33,6 +33,7 @@ import org.jacodb.impl.fs.info import org.jacodb.impl.types.ClassInfo import org.jacodb.impl.weakLazy import org.objectweb.asm.tree.ClassNode +import java.util.* import kotlin.LazyThreadSafetyMode.PUBLICATION class JcClassOrInterfaceImpl( @@ -129,36 +130,31 @@ class JcClassOrInterfaceImpl( override val declaredFields: List get() { - val result: List = info.fields.map { JcFieldImpl(this, it) } - return when { - hasClassFeatures -> { - val modifiedFields = result.toMutableList() - featuresChain.newRequest().run { - it.fieldsOf(this)?.let { - modifiedFields.addAll(it) - } + val default = info.fields.map { JcFieldImpl(this, it) } + if (hasClassFeatures) { + val result = TreeSet { o1, o2 -> o1.name.compareTo(o2.name) } + featuresChain.newRequest().run { + it.fieldsOf(this)?.let { + result.addAll(it) } - modifiedFields } - - else -> result + return (result + default).toList() } + return default } override val declaredMethods: List by lazy(PUBLICATION) { - val result: List = info.methods.map { toJcMethod(it, featuresChain) } - when { - hasClassFeatures -> { - val modifiedMethods = result.toMutableList() - featuresChain.newRequest().run { - it.methodsOf(this)?.let { - modifiedMethods.addAll(it) - } + val default = info.methods.map { toJcMethod(it, featuresChain) } + if (hasClassFeatures) { + val result = TreeSet { o1, o2 -> (o1.name + o1.description).compareTo(o2.name + o2.description) } + featuresChain.newRequest().run { + it.methodsOf(this)?.let { + result.addAll(it) } - modifiedMethods } - - else -> result + (result + default).toList() + } else { + default } } diff --git a/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/tests/DatabaseEnvTest.kt b/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/tests/DatabaseEnvTest.kt index 007a333d8..96981ad41 100644 --- a/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/tests/DatabaseEnvTest.kt +++ b/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/tests/DatabaseEnvTest.kt @@ -486,6 +486,49 @@ abstract class DatabaseEnvTest { } } + @Test + fun `override existed field and method`() { + val fieldName = "byteArray" + val methodName = "smth" + val byteArrayTypeName = PredefinedPrimitives.Byte + "[]" + + val cp = runBlocking { + cp.db.classpath( + allClasspath, listOf( + VirtualClassContent + .builder() + .content { + matcher { it.name == Bar::class.java.name } + field { builder, _ -> + builder.name(fieldName) + builder.type(PredefinedPrimitives.Int) + } + method { builder, _ -> + builder.name(methodName) + builder.returnType(byteArrayTypeName) + builder.params(byteArrayTypeName) + } + }.build() + ) + ) + } + val clazz = cp.findClass() + val field = clazz.findDeclaredFieldOrNull(fieldName) + assertTrue(field is JcVirtualField) + assertEquals(PredefinedPrimitives.Int, field!!.type.typeName) + assertNotNull(field.enclosingClass) + + val method = clazz.declaredMethods.first { it.name == methodName } + assertTrue(method is JcVirtualMethod) + assertEquals(byteArrayTypeName, method.returnType.typeName) + assertEquals(1, method.parameters.size) + assertEquals(byteArrayTypeName, method.parameters.first().type.typeName) + assertNotNull(method.enclosingClass) + method.parameters.forEach { + assertNotNull(it.method) + } + } + @Test fun `class caching feature works for not existed class`() { val notExistedClass = "xxx.Xxx" From cca11170b6e677f2883dc0dca493f9d0cfa063d7 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Fri, 26 May 2023 18:59:38 +0300 Subject: [PATCH 23/35] Typo in the JcRawDynamicCallExpr/JcDynamicCallExpr member names and JcLambdaExpr javadoc #83 --- .../src/main/kotlin/org/jacodb/api/cfg/JcInst.kt | 10 +++++----- .../src/main/kotlin/org/jacodb/api/cfg/JcRawInst.kt | 10 +++++----- .../impl/analysis/impl/StringConcatSimplifier.kt | 8 ++++---- .../kotlin/org/jacodb/impl/cfg/JcInstListBuilder.kt | 7 +++---- .../kotlin/org/jacodb/impl/cfg/MethodNodeBuilder.kt | 4 ++-- .../main/kotlin/org/jacodb/impl/cfg/util/ExprMapper.kt | 6 +++--- 6 files changed, 22 insertions(+), 23 deletions(-) diff --git a/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcInst.kt b/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcInst.kt index d6189f9a1..8cec4badb 100644 --- a/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcInst.kt +++ b/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcInst.kt @@ -680,14 +680,14 @@ data class JcLambdaExpr( data class JcDynamicCallExpr( private val bsmRef: TypedMethodRef, val bsmArgs: List, - val callCiteMethodName: String, - val callCiteArgTypes: List, - val callCiteReturnType: JcType, - val callCiteArgs: List + val callSiteMethodName: String, + val callSiteArgTypes: List, + val callSiteReturnType: JcType, + val callSiteArgs: List ) : JcCallExpr { override val method get() = bsmRef.method - override val args get() = callCiteArgs + override val args get() = callSiteArgs override fun accept(visitor: JcExprVisitor): T { return visitor.visitJcDynamicCallExpr(this) diff --git a/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcRawInst.kt b/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcRawInst.kt index cbc4a5dbc..aa8f4cd7c 100644 --- a/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcRawInst.kt +++ b/jacodb-api/src/main/kotlin/org/jacodb/api/cfg/JcRawInst.kt @@ -690,17 +690,17 @@ data class BsmHandle( data class JcRawDynamicCallExpr( val bsm: BsmHandle, val bsmArgs: List, - val callCiteMethodName: String, - val callCiteArgTypes: List, - val callCiteReturnType: TypeName, - val callCiteArgs: List + val callSiteMethodName: String, + val callSiteArgTypes: List, + val callSiteReturnType: TypeName, + val callSiteArgs: List ) : JcRawCallExpr { override val declaringClass get() = bsm.declaringClass override val methodName get() = bsm.name override val argumentTypes get() = bsm.argTypes override val returnType get() = bsm.returnType override val typeName get() = returnType - override val args get() = callCiteArgs + override val args get() = callSiteArgs override fun accept(visitor: JcRawExprVisitor): T { return visitor.visitJcRawDynamicCallExpr(this) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/analysis/impl/StringConcatSimplifier.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/analysis/impl/StringConcatSimplifier.kt index a12299168..7e3655d39 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/analysis/impl/StringConcatSimplifier.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/analysis/impl/StringConcatSimplifier.kt @@ -58,12 +58,12 @@ class StringConcatSimplifier(val jcGraph: JcGraph) : DefaultJcInstVisitor rhv.callCiteArgs - rhv.callCiteArgs.size == 1 && rhv.bsmArgs.size == 1 && rhv.bsmArgs[0] is BsmStringArg -> listOf( - rhv.callCiteArgs[0], + rhv.callSiteArgs.size == 2 -> rhv.callSiteArgs + rhv.callSiteArgs.size == 1 && rhv.bsmArgs.size == 1 && rhv.bsmArgs[0] is BsmStringArg -> listOf( + rhv.callSiteArgs[0], JcStringConstant((rhv.bsmArgs[0] as BsmStringArg).value, stringType) ) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcInstListBuilder.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcInstListBuilder.kt index f11d0a00b..45258409b 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcInstListBuilder.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/JcInstListBuilder.kt @@ -51,7 +51,6 @@ import org.jacodb.api.cfg.JcFieldRef import org.jacodb.api.cfg.JcFloat import org.jacodb.api.cfg.JcGeExpr import org.jacodb.api.cfg.JcGotoInst -import org.jacodb.api.cfg.JcGraph import org.jacodb.api.cfg.JcGtExpr import org.jacodb.api.cfg.JcIfInst import org.jacodb.api.cfg.JcInst @@ -412,9 +411,9 @@ class JcInstListBuilder(val method: JcMethod,val instList: JcInstList return JcDynamicCallExpr( classpath.methodRef(expr), expr.bsmArgs, - expr.callCiteMethodName, - expr.callCiteArgTypes.map { it.asType() }, - expr.callCiteReturnType.asType(), + expr.callSiteMethodName, + expr.callSiteArgTypes.map { it.asType() }, + expr.callSiteReturnType.asType(), expr.args.map { it.accept(this) as JcValue } ) } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/MethodNodeBuilder.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/MethodNodeBuilder.kt index 431d08f45..d0bfe95a7 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/MethodNodeBuilder.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/MethodNodeBuilder.kt @@ -660,8 +660,8 @@ class MethodNodeBuilder( expr.args.forEach { it.accept(this) } currentInsnList.add( InvokeDynamicInsnNode( - expr.callCiteMethodName, - "(${expr.callCiteArgTypes.joinToString("") { it.jvmTypeName }})${expr.callCiteReturnType.jvmTypeName}", + expr.callSiteMethodName, + "(${expr.callSiteArgTypes.joinToString("") { it.jvmTypeName }})${expr.callSiteReturnType.jvmTypeName}", expr.bsm.asAsmHandle, *expr.bsmArgs.map { when (it) { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/util/ExprMapper.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/util/ExprMapper.kt index 678cc864a..b598c4fdb 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/util/ExprMapper.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/cfg/util/ExprMapper.kt @@ -318,9 +318,9 @@ class ExprMapper(val mapping: Map) : JcRawInstVisitor JcRawDynamicCallExpr( expr.bsm, expr.bsmArgs, - expr.callCiteMethodName, - expr.callCiteArgTypes, - expr.callCiteReturnType, + expr.callSiteMethodName, + expr.callSiteArgTypes, + expr.callSiteReturnType, newArgs ) } From debf15b822afa9195b9cbb690268b3ac4152efbe Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 29 May 2023 11:12:06 +0300 Subject: [PATCH 24/35] add coverage infrastructure --- .github/workflows/coverage.yml | 77 ++++++++++++++++++++++++++++++++++ build.gradle.kts | 13 ++++++ 2 files changed, 90 insertions(+) create mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 000000000..eb7640d16 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,77 @@ +name: Update coverage + +on: + push: + branches: [ develop, main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Checkout badges branch to a badges directory nested inside first checkout + uses: actions/checkout@v3 + with: + ref: badges + path: badges + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + - uses: gradle/gradle-build-action@v2 + with: + gradle-version: 7.6.1 + - name: Build and run tests + run: | + gradle clean build --no-daemon --info + + - name: Generate JaCoCo badge + id: jacoco + uses: cicirello/jacoco-badge-generator@v2 + with: + badges-directory: badges + generate-branches-badge: true + generate-summary: true + + - name: Log coverage percentages to workflow output + run: | + echo "coverage = ${{ steps.jacoco.outputs.coverage }}" + echo "branches = ${{ steps.jacoco.outputs.branches }}" + - name: Upload JaCoCo coverage report as a workflow artifact + uses: actions/upload-artifact@v3 + with: + name: jacoco-report + path: build/reports/jacoco/ + + - name: Commit and push the coverage badges and summary file + if: ${{ github.event_name != 'pull_request' }} + run: | + cd badges + if [[ `git status --porcelain *.svg *.json` ]]; then + git config --global user.name 'github-actions' + git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add *.svg *.json + git commit -m "Autogenerated JaCoCo coverage badges" *.svg *.json + git push + fi + - name: Comment on PR with coverage percentages + if: ${{ github.event_name == 'pull_request' }} + run: | + REPORT=$( { useJUnitPlatform() jvmArgs = listOf("-Xmx2g", "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=heapdump.hprof") testLogging { events("passed", "skipped", "failed") } + finalizedBy(jacocoTestReport) // report is always generated after tests run } + val sourcesJar by creating(Jar::class) { archiveClassifier.set("sources") from(sourceSets.getByName("main").kotlin.srcDirs) From d46e4dbf4beff2753e70caacdfbbbbf9ae7927dc Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 29 May 2023 11:31:26 +0300 Subject: [PATCH 25/35] fix coverage build --- .github/workflows/coverage.yml | 8 +------- docs/badges/branches.svg | 1 + docs/badges/coverage-summary.json | 1 + docs/badges/jacoco.svg | 1 + 4 files changed, 4 insertions(+), 7 deletions(-) create mode 100644 docs/badges/branches.svg create mode 100644 docs/badges/coverage-summary.json create mode 100644 docs/badges/jacoco.svg diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index eb7640d16..0cbe0a5b2 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -16,12 +16,6 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Checkout badges branch to a badges directory nested inside first checkout - uses: actions/checkout@v3 - with: - ref: badges - path: badges - - name: Set up JDK 11 uses: actions/setup-java@v3 with: @@ -38,7 +32,7 @@ jobs: id: jacoco uses: cicirello/jacoco-badge-generator@v2 with: - badges-directory: badges + badges-directory: docs/badges generate-branches-badge: true generate-summary: true diff --git a/docs/badges/branches.svg b/docs/badges/branches.svg new file mode 100644 index 000000000..c225d1b8e --- /dev/null +++ b/docs/badges/branches.svg @@ -0,0 +1 @@ +branches100% \ No newline at end of file diff --git a/docs/badges/coverage-summary.json b/docs/badges/coverage-summary.json new file mode 100644 index 000000000..435a7bced --- /dev/null +++ b/docs/badges/coverage-summary.json @@ -0,0 +1 @@ +{"branches": 100, "coverage": 100} \ No newline at end of file diff --git a/docs/badges/jacoco.svg b/docs/badges/jacoco.svg new file mode 100644 index 000000000..c718fa80b --- /dev/null +++ b/docs/badges/jacoco.svg @@ -0,0 +1 @@ +coverage100% \ No newline at end of file From 303e4b14375533caa405ea6e2806d9e9f1cbed51 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 29 May 2023 12:23:32 +0300 Subject: [PATCH 26/35] makes builds faster by adding indexes for cascading deletes of bytecodelocatons --- .../src/main/resources/sqlite/add-indexes.sql | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/jacodb-core/src/main/resources/sqlite/add-indexes.sql b/jacodb-core/src/main/resources/sqlite/add-indexes.sql index ef2a4d7ef..f4465a71b 100644 --- a/jacodb-core/src/main/resources/sqlite/add-indexes.sql +++ b/jacodb-core/src/main/resources/sqlite/add-indexes.sql @@ -1,4 +1,24 @@ CREATE INDEX IF NOT EXISTS "Classes_name" ON "Classes" ("name"); + +CREATE INDEX IF NOT EXISTS "Classes_outerMethodId" ON "Classes" ("outer_method"); +CREATE INDEX IF NOT EXISTS "ClassInnerClasses_classId" ON "ClassInnerClasses" ("class_id"); +CREATE INDEX IF NOT EXISTS "ClassInnerClasses_classId" ON "ClassInnerClasses" ("class_id"); +CREATE INDEX IF NOT EXISTS "ClassInnerClasses_innerClassId" ON "ClassInnerClasses" ("inner_class_id"); + +CREATE INDEX IF NOT EXISTS "ClassHierarchies_classId" ON "ClassHierarchies" ("class_id"); +CREATE INDEX IF NOT EXISTS "ClassHierarchies_superId" ON "ClassHierarchies" ("super_id"); + +CREATE INDEX IF NOT EXISTS "Annotations_classId" ON "Annotations" ("class_id"); +CREATE INDEX IF NOT EXISTS "Annotations_fieldId" ON "Annotations" ("field_id"); +CREATE INDEX IF NOT EXISTS "Annotations_methodId" ON "Annotations" ("method_id"); +CREATE INDEX IF NOT EXISTS "Annotations_paramsId" ON "Annotations" ("param_id"); + +CREATE INDEX IF NOT EXISTS "Classes_location" ON "Classes" ("location_id"); +CREATE INDEX IF NOT EXISTS "Fields_classId" ON "Fields" ("class_id"); +CREATE INDEX IF NOT EXISTS "Methods_classId" ON "Methods" ("class_id"); + +CREATE INDEX IF NOT EXISTS "MethodParameters_methodId" ON "MethodParameters" ("method_id"); + CREATE UNIQUE INDEX IF NOT EXISTS "Symbols_name" ON "Symbols" ("name"); CREATE UNIQUE INDEX IF NOT EXISTS "Bytecodelocations_hash" ON "BytecodeLocations" ("uniqueId"); CREATE UNIQUE INDEX IF NOT EXISTS "Methods_class_id_name_desc" ON "Methods" ("class_id", "name", "desc"); From 08d0cf44fcc418da8bb042e842eb96d0905aef9d Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 29 May 2023 13:07:48 +0300 Subject: [PATCH 27/35] fix tests --- .../kotlin/org/jacodb/impl/features/HierarchyExtension.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt index 9d1c2f6cf..f2ecd52a9 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt @@ -49,6 +49,7 @@ class HierarchyExtensionImpl(private val cp: JcClasspath) : HierarchyExtension { JOIN Classes ON Classes.id = hierarchy.class_id JOIN Symbols ON Symbols.id = Classes.name WHERE location_id in ($locationIds) + ORDER BY Classes.id """.trimIndent() private fun directSubClassesQuery(locationIds: String, sinceId: Long?) = """ @@ -56,7 +57,8 @@ class HierarchyExtensionImpl(private val cp: JcClasspath) : HierarchyExtension { JOIN Symbols ON Symbols.id = ClassHierarchies.super_id JOIN Symbols as SymbolsName ON SymbolsName.id = Classes.name JOIN Classes ON Classes.id = ClassHierarchies.class_id - WHERE Symbols.name = ? and ($sinceId is null or ClassHierarchies.class_id > $sinceId) AND Classes.location_id in ($locationIds) + WHERE Symbols.name = ? and ($sinceId is null or ClassHierarchies.class_id > $sinceId) AND Classes.location_id in ($locationIds) + ORDER BY Classes.id """.trimIndent() } From fbd93ddc9335f4c2814445d62a4059c5462bb159 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 29 May 2023 15:34:29 +0300 Subject: [PATCH 28/35] get rid of custom serializer --- .../kotlin/org/jacodb/impl/types/Objects.kt | 52 +------------------ 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/types/Objects.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/types/Objects.kt index 964462bd2..a40d4669d 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/types/Objects.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/types/Objects.kt @@ -108,59 +108,9 @@ sealed class AnnotationValue @Serializable open class AnnotationValueList(val annotations: List) : AnnotationValue() -@Serializable(with = PrimitiveValueSerializer::class) +@Serializable class PrimitiveValue(val dataType: AnnotationValueKind, val value: Any) : AnnotationValue() -object PrimitiveValueSerializer : KSerializer { - override val descriptor: SerialDescriptor = buildClassSerialDescriptor("PrimitiveValue") { - element("dataType", serialDescriptor()) - element("value", buildClassSerialDescriptor("Any")) - } - - @Suppress("UNCHECKED_CAST") - private val dataTypeSerializers: Map> = - mapOf( - AnnotationValueKind.STRING to serializer(), - AnnotationValueKind.BYTE to serializer(), - AnnotationValueKind.SHORT to serializer(), - AnnotationValueKind.CHAR to serializer(), - AnnotationValueKind.LONG to serializer(), - AnnotationValueKind.INT to serializer(), - AnnotationValueKind.FLOAT to serializer(), - AnnotationValueKind.DOUBLE to serializer(), - AnnotationValueKind.BYTE to serializer(), - AnnotationValueKind.BOOLEAN to serializer() - //list them all - ).mapValues { (_, v) -> v as KSerializer } - - private fun getPayloadSerializer(dataType: AnnotationValueKind): KSerializer = dataTypeSerializers[dataType] - ?: throw SerializationException("Serializer for class $dataType is not registered in PacketSerializer") - - override fun serialize(encoder: Encoder, value: PrimitiveValue) { - encoder.encodeStructure(descriptor) { - encodeStringElement(descriptor, 0, value.dataType.name) - encodeSerializableElement(descriptor, 1, getPayloadSerializer(value.dataType), value.value) - } - } - - @ExperimentalSerializationApi - override fun deserialize(decoder: Decoder): PrimitiveValue = decoder.decodeStructure(descriptor) { - val dataType = AnnotationValueKind.valueOf(decodeStringElement(descriptor, 0)) - if (decodeSequentially()) { - val payload = decodeSerializableElement(descriptor, 1, getPayloadSerializer(dataType)) - PrimitiveValue(dataType, payload) - } else { - require(decodeElementIndex(descriptor) == 0) { "dataType field should precede payload field" } - val payload = when (val index = decodeElementIndex(descriptor)) { - 1 -> decodeSerializableElement(descriptor, 1, getPayloadSerializer(dataType)) - CompositeDecoder.DECODE_DONE -> throw SerializationException("payload field is missing") - else -> error("Unexpected index: $index") - } - PrimitiveValue(dataType, payload) - } - } -} - @Serializable class ClassRef(val className: String) : AnnotationValue() From 59f1c40effa59b13502872bce3326f5f620305b7 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 29 May 2023 15:42:20 +0300 Subject: [PATCH 29/35] fix coverage report action --- .github/workflows/coverage.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 0cbe0a5b2..8c9fc8881 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -1,4 +1,4 @@ -name: Update coverage +name: Generate coverage report on: push: @@ -35,6 +35,9 @@ jobs: badges-directory: docs/badges generate-branches-badge: true generate-summary: true + jacoco-csv-file: > + jacodb-analysis/build/reports/jacoco/test/jacocoTestReport.csv + jacodb-core/build/reports/jacoco/test/jacocoTestReport.csv - name: Log coverage percentages to workflow output run: | From 4dfb5822c10b51900f352fce892afd2fa56ac605 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 29 May 2023 15:59:59 +0300 Subject: [PATCH 30/35] fix coverage report action --- .github/workflows/coverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 8c9fc8881..20f3e4445 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -47,12 +47,12 @@ jobs: uses: actions/upload-artifact@v3 with: name: jacoco-report - path: build/reports/jacoco/ + path: jacodb-*/build/reports/jacoco/ - name: Commit and push the coverage badges and summary file if: ${{ github.event_name != 'pull_request' }} run: | - cd badges + cd docs/badges if [[ `git status --porcelain *.svg *.json` ]]; then git config --global user.name 'github-actions' git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com' From 0dcd436c9aa87f17999bbfb695f7d9aa7fa24596 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 13:20:19 +0000 Subject: [PATCH 31/35] Autogenerated JaCoCo coverage badges --- docs/badges/branches.svg | 2 +- docs/badges/coverage-summary.json | 2 +- docs/badges/jacoco.svg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/badges/branches.svg b/docs/badges/branches.svg index c225d1b8e..3935ab969 100644 --- a/docs/badges/branches.svg +++ b/docs/badges/branches.svg @@ -1 +1 @@ -branches100% \ No newline at end of file +branches68.3% \ No newline at end of file diff --git a/docs/badges/coverage-summary.json b/docs/badges/coverage-summary.json index 435a7bced..483ff35cc 100644 --- a/docs/badges/coverage-summary.json +++ b/docs/badges/coverage-summary.json @@ -1 +1 @@ -{"branches": 100, "coverage": 100} \ No newline at end of file +{"branches": 68.34195614683419, "coverage": 70.38198957259596} \ No newline at end of file diff --git a/docs/badges/jacoco.svg b/docs/badges/jacoco.svg index c718fa80b..31127ea7b 100644 --- a/docs/badges/jacoco.svg +++ b/docs/badges/jacoco.svg @@ -1 +1 @@ -coverage100% \ No newline at end of file +coverage70.3% \ No newline at end of file From 741944cb0500160f13749a508ec46cc4a9bd6383 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 29 May 2023 16:26:12 +0300 Subject: [PATCH 32/35] add coverage badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c8bb51e5e..1e1173f9a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![ci status](https://github.com/UnitTestBot/jacodb/actions/workflows/build-and-test.yml/badge.svg)](https://github.com/UnitTestBot/jacodb/actions/workflows/build-and-test.yml) +[Code coverage](./docs/badges/jacoco.svg) ## Overview From ef42c85969493fadc8e723954375374fc228e247 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 13:37:49 +0000 Subject: [PATCH 33/35] Autogenerated JaCoCo coverage badges --- docs/badges/coverage-summary.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/badges/coverage-summary.json b/docs/badges/coverage-summary.json index 483ff35cc..877f64433 100644 --- a/docs/badges/coverage-summary.json +++ b/docs/badges/coverage-summary.json @@ -1 +1 @@ -{"branches": 68.34195614683419, "coverage": 70.38198957259596} \ No newline at end of file +{"branches": 68.31731953683173, "coverage": 70.38049567516694} \ No newline at end of file From 8e19579ecaa3054f1726005a6f8772df7e80a5be Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 29 May 2023 16:41:09 +0300 Subject: [PATCH 34/35] fix coverage badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e1173f9a..c93974f61 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![ci status](https://github.com/UnitTestBot/jacodb/actions/workflows/build-and-test.yml/badge.svg)](https://github.com/UnitTestBot/jacodb/actions/workflows/build-and-test.yml) -[Code coverage](./docs/badges/jacoco.svg) +[![Coverage](./docs/badges/jacoco.svg)](https://github.com/UnitTestBot/jacodb/actions/workflows/coverage.yml) ## Overview From d7f9df25ff95c358705a9653f4ccd617d03397ef Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 13:54:52 +0000 Subject: [PATCH 35/35] Autogenerated JaCoCo coverage badges --- docs/badges/coverage-summary.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/badges/coverage-summary.json b/docs/badges/coverage-summary.json index 877f64433..483ff35cc 100644 --- a/docs/badges/coverage-summary.json +++ b/docs/badges/coverage-summary.json @@ -1 +1 @@ -{"branches": 68.31731953683173, "coverage": 70.38049567516694} \ No newline at end of file +{"branches": 68.34195614683419, "coverage": 70.38198957259596} \ No newline at end of file