From fd8a474b853f219d5d4fd9dd7854ca06b3889563 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Wed, 22 Oct 2025 16:29:33 +0200 Subject: [PATCH 01/11] Add initial gradle modules structure for reflective Ops documentaion Related-To #139 --- settings.gradle.kts | 3 + .../ainet/compile/nn/NeuralNetworkContext.kt | 52 ++++ .../sk/ainet/context/ExecutionContext.kt | 8 + .../skainet-lang-export-ops/build.gradle.kts | 70 ++++++ .../org/mikrograd/data/generator/generator.kt | 71 ++++++ .../kotlin/org/mikrograd/samples/clusters.kt | 45 ++++ .../kotlin/org/mikrograd/samples/minmal.kt | 58 +++++ .../kotlin/org/mikrograd/samples/neuron.kt | 16 ++ .../kotlin/org/mikrograd/samples/sinusNN.kt | 53 ++++ .../src/jvmMain/kotlin/com/example/KspMain.kt | 37 +++ .../src/jvmMain/kotlin/com/example/Main.kt | 134 ++++++++++ .../build.gradle.kts | 16 ++ .../gradle.properties | 2 + .../org/mikrograd/diff/ksp/Mikrograd.kt | 29 +++ .../skainet-lang-ksp-processor/README.md | 103 ++++++++ .../build.gradle.kts | 33 +++ .../gradle.properties | 2 + .../diff/ksp/ComputeGraphProcessor.kt | 234 ++++++++++++++++++ .../mikrograd/diff/ksp/ExpressionParser.kt | 168 +++++++++++++ .../mikrograd/diff/ksp/ExpressionVisitor.kt | 205 +++++++++++++++ ...ols.ksp.processing.SymbolProcessorProvider | 2 + .../diff/ksp/ComputeGraphProcessorTest.kt | 121 +++++++++ .../kotlin/sk/ainet/lang/nn/Model.kt | 1 - 23 files changed, 1462 insertions(+), 1 deletion(-) create mode 100644 skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/compile/nn/NeuralNetworkContext.kt create mode 100644 skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/context/ExecutionContext.kt create mode 100644 skainet-lang/skainet-lang-export-ops/build.gradle.kts create mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/data/generator/generator.kt create mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/clusters.kt create mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/minmal.kt create mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/neuron.kt create mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/sinusNN.kt create mode 100644 skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/KspMain.kt create mode 100644 skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/Main.kt create mode 100644 skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts create mode 100644 skainet-lang/skainet-lang-ksp-annotations/gradle.properties create mode 100644 skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt create mode 100644 skainet-lang/skainet-lang-ksp-processor/README.md create mode 100644 skainet-lang/skainet-lang-ksp-processor/build.gradle.kts create mode 100644 skainet-lang/skainet-lang-ksp-processor/gradle.properties create mode 100644 skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt create mode 100644 skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionParser.kt create mode 100644 skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt create mode 100644 skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider create mode 100644 skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 776c57f0..7e3cf00d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -17,3 +17,6 @@ rootProject.name = "SKaiNET" include("skainet-lang:skainet-lang-core") include("skainet-lang:skainet-lang-models") +include("skainet-lang:skainet-lang-ksp-annotations") +include("skainet-lang:skainet-lang-ksp-processor") +include("skainet-lang:skainet-lang-export-ops") diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/compile/nn/NeuralNetworkContext.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/compile/nn/NeuralNetworkContext.kt new file mode 100644 index 00000000..de3f649f --- /dev/null +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/compile/nn/NeuralNetworkContext.kt @@ -0,0 +1,52 @@ +package sk.ainet.compile.nn + +import sk.ainet.compile.nn.DefaultNetworkContext +import sk.ainet.lang.nn.dsl.NeuralNetworkDsl +import sk.ainet.lang.nn.dsl.network + + +import sk.ainet.lang.nn.Module +import sk.ainet.lang.tensor.data.DenseTensorDataFactory +import sk.ainet.lang.tensor.data.TensorDataFactory +import sk.ainet.lang.types.DType + + +/** + * Context for the DSL to define the data type and operations. + * + * This class holds the information about the data type and operations + * that should be used in the DSL. It's used to make the DSL generic + * and to avoid hardcoding the data type. + * + * @param T The default data type. + */ +public interface NeuralNetworkContext { + + public val tensorDataFactory: TensorDataFactory + +} + +/** + * Creates a context for the DSL with the given configuration. + * + * @param T The type of data processed by the modules. + * @param init The configuration function. + * @return The configured context. + */ +public fun context(init: NeuralNetworkContext.(NeuralNetworkContext) -> Module): Module { + val instance = DefaultNetworkContext() + return instance.init(instance) +} + +/** + * Extension function to create a network within a NetworkContext. + * This bridges the context wrapper with the network DSL using the context's tensor factory. + */ +public inline fun NeuralNetworkContext.network( + content: NeuralNetworkDsl.() -> Unit +): Module = network(tensorDataFactory, content) + +public class DefaultNetworkContext : NeuralNetworkContext { + override val tensorDataFactory: TensorDataFactory + get() = DenseTensorDataFactory() +} \ No newline at end of file diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/context/ExecutionContext.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/context/ExecutionContext.kt new file mode 100644 index 00000000..a505f45b --- /dev/null +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/context/ExecutionContext.kt @@ -0,0 +1,8 @@ +package sk.ainet.context + +import sk.ainet.lang.tensor.ops.TensorOps + + +public interface ExecutionContext { + public val ops: TensorOps +} diff --git a/skainet-lang/skainet-lang-export-ops/build.gradle.kts b/skainet-lang/skainet-lang-export-ops/build.gradle.kts new file mode 100644 index 00000000..64bd592f --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/build.gradle.kts @@ -0,0 +1,70 @@ +plugins { + alias(libs.plugins.kotlinMultiplatform) + alias(libs.plugins.ksp) +} + + +group = "org.mikrograd.samples" + +kotlin { + + compilerOptions { + // Common compiler options applied to all Kotlin source sets + freeCompilerArgs.add("-Xexpect-actual-classes") + freeCompilerArgs.add("-Xmulti-platform") + } + + jvmToolchain(17) + + jvm() + + + sourceSets { + commonMain.dependencies { + implementation(project(":miKrograd")) + } + + commonTest.dependencies { + implementation(kotlin("test-common")) + implementation(kotlin("test-annotations-common")) + } + + val jvmMain by getting { + kotlin.srcDir("build/generated/ksp/jvm/jvmMain/kotlin") + dependencies { + implementation(project(":miKrograd-annotations")) + } + } + + + + jvmTest.dependencies { + implementation(kotlin("test-junit")) + } + } +} + +dependencies { + // add("kspCommonMainMetadata", project(":test-processor")) + add("kspJvm", project(":skainet-lang:skainet-lang-ksp-processor")) +} + +ksp { + arg("ksp.verbose", "true") +} + +// Add a run task for the JVM application +tasks.register("runJvm") { + group = "application" + description = "Run the JVM application" + classpath = files(kotlin.jvm().compilations["main"].output.allOutputs, configurations.getByName("jvmRuntimeClasspath")) + mainClass.set("com.example.MainKt") +} + +// Add a run task for the KspMain application +tasks.register("runKspMain") { + group = "application" + description = "Run the KspMain application" + classpath = files(kotlin.jvm().compilations["main"].output.allOutputs, configurations.getByName("jvmRuntimeClasspath")) + mainClass.set("com.example.KspMainKt") +} diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/data/generator/generator.kt b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/data/generator/generator.kt new file mode 100644 index 00000000..a870951d --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/data/generator/generator.kt @@ -0,0 +1,71 @@ +package org.mikrograd.data.generator + +import kotlin.math.cos +import kotlin.math.sin +import kotlin.random.Random + +fun makeMoons( + nSamples: Any = 100, + shuffle: Boolean = true, + noise: Double? = null, + randomState: Int? = null +): Pair, IntArray> { + val (nSamplesOut, nSamplesIn) = when (nSamples) { + is Int -> Pair(nSamples / 2, nSamples - nSamples / 2) + is Pair<*, *> -> { + if (nSamples.first is Int && nSamples.second is Int) { + Pair(nSamples.first as Int, nSamples.second as Int) + } else { + throw IllegalArgumentException("`n_samples` can be either an int or a two-element tuple.") + } + } + else -> throw IllegalArgumentException("`n_samples` can be either an int or a two-element tuple.") + } + + val generator = randomState?.let { Random(it) } ?: Random.Default + + val outerCircX = DoubleArray(nSamplesOut) { cos(it * Math.PI / nSamplesOut) } + val outerCircY = DoubleArray(nSamplesOut) { sin(it * Math.PI / nSamplesOut) } + val innerCircX = DoubleArray(nSamplesIn) { 1 - cos(it * Math.PI / nSamplesIn) } + val innerCircY = DoubleArray(nSamplesIn) { 1 - sin(it * Math.PI / nSamplesIn) - 0.5 } + + val X = Array(nSamplesOut + nSamplesIn) { DoubleArray(2) } + for (i in 0 until nSamplesOut) { + X[i][0] = outerCircX[i] + X[i][1] = outerCircY[i] + } + for (i in 0 until nSamplesIn) { + X[nSamplesOut + i][0] = innerCircX[i] + X[nSamplesOut + i][1] = innerCircY[i] + } + + val y = IntArray(nSamplesOut + nSamplesIn) + for (i in 0 until nSamplesOut) { + y[i] = 0 + } + for (i in 0 until nSamplesIn) { + y[nSamplesOut + i] = 1 + } + + if (shuffle) { + val indices = X.indices.toList().shuffled(generator) + val XShuffled = Array(X.size) { DoubleArray(2) } + val yShuffled = IntArray(y.size) + for (i in indices.indices) { + XShuffled[i] = X[indices[i]] + yShuffled[i] = y[indices[i]] + } + X.indices.forEach { X[it] = XShuffled[it] } + y.indices.forEach { y[it] = yShuffled[it] } + } + + noise?.let { + for (i in X.indices) { + X[i][0] += generator.nextDouble(-noise, noise) + X[i][1] += generator.nextDouble(-noise, noise) + } + } + + return Pair(X, y) +} + diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/clusters.kt b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/clusters.kt new file mode 100644 index 00000000..58407e60 --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/clusters.kt @@ -0,0 +1,45 @@ +package org.mikrograd.samples +/* +import org.mikrograd.diff.MLP +import kotlin.random.Random +import org.mikrograd.diff.Value + + +class MLPClustering(private val data: Pair, IntArray>, val model: MLP ) { + private val X: Array = data.first + private val y: IntArray = data.second + + fun loss(batchSize: Int? = null): Pair { + val (Xb, yb) = if (batchSize == null) { + Pair(X.toList(), y.toList()) + } else { + val ri = List(batchSize) { Random.nextInt(X.size) } + Pair(ri.map { X[it] }, ri.map { y[it] }) + } + val xc: List = Xb + val inputs: List> = Xb.map { xrow -> xrow.map { Value(it) } } + + val scores: List = inputs.flatMap { input -> model.invoke (input) } + + //losses = [(1 + -yi*scorei).relu() for yi, scorei in zip(yb, scores)] + + + val losses: List = yb.zip(scores).map { (yi, scorei) -> Value(1 + -yi * scorei.data).relu() } + val lossesSum: Value = losses.fold(Value(0.0)) { a, i -> a + i } + + val dataLoss: Value = lossesSum / losses.size + + val alpha = 1e-4 + val regLoss: Value = alpha * (model.parameters().reduce { a, i -> a * i }) + //val reg_loss = alpha * sum((p*p for p in model.parameters())) + val totalLoss = dataLoss + regLoss + + val accuracy = yb.zip(scores).count { (yi, scorei) -> (yi > 0) == (scorei.data > 0) }.toDouble() / yb.size + + return Pair(totalLoss, accuracy) + } + + +} + + */ \ No newline at end of file diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/minmal.kt b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/minmal.kt new file mode 100644 index 00000000..11a184f5 --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/minmal.kt @@ -0,0 +1,58 @@ +package org.mikrograd.samples + +/* +import org.mikrograd.diff.MLP +import org.mikrograd.diff.Value +import org.mikrograd.utils.drawDot + +fun loss(X: Array, y: DoubleArray, model: MLP): Pair { + val inputs: List> = X.mapIndexed { index, xrow -> xrow.map { Value(it, label = "in$index") } } + val scores: List = inputs.map { input -> model(input).first() } + + // mean square error + val values: List = + y.zip(scores) { actual: Double, predicted: Value -> (actual - predicted).pow(2.0) } + val mse = values.fold(Value(0.0)) { acc, value -> acc + value } + + return Pair(mse, 0.0) +} + +fun main() { + + val c = Value(3.0, label = "a") + Value(2.0, label = "b") + val cGr = drawDot(c) + cGr.toFile("a+b.dot") + + val d = Value(3.0, label = "a") * Value(2.0, label = "b") + val dGr = drawDot(d) + dGr.toFile("a*b.dot") + + + val model = MLP(1, listOf(1, 1, 1)) //# 2-layer neural network + val (X, y) = Pair, DoubleArray>( + arrayOf(doubleArrayOf(1.0)), + doubleArrayOf(2.0) + ) + + val X_v: List> = X.map { xrow -> xrow.map { Value(it) } } + val prediction = model.invoke(X_v[0])[0] + + val modelGr = drawDot(prediction) + modelGr.toFile("model.dot") + + val (loss, _) = loss(X, y, model) + + val lossGr = drawDot(loss) + lossGr.toFile("loss.dot") + + model.zeroGrad() + loss.backward() + + val backGr = drawDot(loss, true) + backGr.toFile("back.dot") + + + +} + + */ \ No newline at end of file diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/neuron.kt b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/neuron.kt new file mode 100644 index 00000000..0cf9d923 --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/neuron.kt @@ -0,0 +1,16 @@ +package org.mikrograd.samples + +/* +import org.mikrograd.diff.Neuron +import org.mikrograd.diff.Value +import org.mikrograd.utils.drawDot + +fun main() { + val neuralNetwork = Neuron(2) + //parameters + val x = listOf(Value(1.0), Value(-2.0)) + val y = neuralNetwork(x) + drawDot(y).toFile("neuron.png") +} + + */ \ No newline at end of file diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/sinusNN.kt b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/sinusNN.kt new file mode 100644 index 00000000..09ef0b3b --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/sinusNN.kt @@ -0,0 +1,53 @@ +package org.mikrograd.samples + +import org.mikrograd.diff.MLP +import org.mikrograd.diff.Value +import kotlin.math.PI +import kotlin.math.sin + +fun train(sine: MLP) { + + //val X = listOf(0.0, PI / 2, PI) + val X = List(100) { index -> + (index / (100 - 1).toFloat()) * (PI / 2) + } + + val y = X.mapIndexed { index, value -> Value(sin(value)).also { it.label = "y$index" } } + + + + (1..100).forEach { + // forward propagation + val ypred: List> = X.mapIndexed { index, x -> + sine.invoke(listOf(x))//.also { it[0].label = "y_pred$index" } + } + // calculate loss + val loss: Value = y.zip(ypred) { ygt, yout -> (ygt - yout[0]).pow(2.0) }.reduce { acc, v -> acc + v } + // reset gradients + sine.parameters().forEach { param -> + param.grad = 0.0 + } + // calc gradients in backpropagation + loss.backward() + + // update weights and biases with a learning rate + sine.parameters().forEach { param -> + param.data += -0.1 * param.grad + } + + println(loss.data) + } +} + +fun MLP.nsin(d: Double) = invoke(listOf(d))[0].data + + +fun main() { + val sine = MLP(1, listOf(16, 16, 1)) + + println(sine.nsin(PI / 2)) + train(sine) + println(sine.nsin(PI / 2)) +} + + diff --git a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/KspMain.kt b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/KspMain.kt new file mode 100644 index 00000000..eafdb681 --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/KspMain.kt @@ -0,0 +1,37 @@ +package com.example + +import org.mikrograd.diff.BackpropNode +import org.mikrograd.diff.ksp.ComputationMode +import org.mikrograd.diff.ksp.Mikrograd +import org.mikrograd.utils.drawDot + +@Mikrograd(ComputationMode.INFERENCE) +fun testExpr() { + 3.0 * 5.0 + (7.0 + 3.0) +} + +@Mikrograd(ComputationMode.TRAINING) +fun testBackExpr() { + 3.0 * 5.0 + (7.0 + 3.0) +} + +fun main(args: Array) { + // Test the KSP-generated functions + val a = testExprGenerated() + println("KSP-generated inference function:") + println("Is BackwardValue: ${a is BackpropNode}") + println("Data: ${a.data}") + println() + + val b = testBackExprGenerated() + println("KSP-generated training function:") + println("Is BackwardValue: ${b is BackpropNode}") + println("Data: ${b.data}") + b.backward() + if (b is BackpropNode) { + println("Gradient: ${b.grad}") + } + println() + val graph = drawDot(b, ) + println(graph) +} diff --git a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/Main.kt b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/Main.kt new file mode 100644 index 00000000..482facf5 --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/Main.kt @@ -0,0 +1,134 @@ +package com.example + +import org.mikrograd.diff.ForwardPassNode +import org.mikrograd.diff.AutoDiffNode + +/** + * Helper function to get the used memory in the JVM + */ +fun getUsedMemory(): Long { + val runtime = Runtime.getRuntime() + return runtime.totalMemory() - runtime.freeMemory() +} + +/** + * Force garbage collection to get more accurate memory measurements + */ +fun forceGC() { + System.gc() + System.runFinalization() + Thread.sleep(100) // Give GC some time to complete + System.gc() + System.runFinalization() + Thread.sleep(100) +} + + +/** + * Implements the expression as specified + * Creates multiple instances to make memory usage more significant + */ +fun expression(): AutoDiffNode { + // Create a list to hold references to prevent garbage collection + val references = mutableListOf() + + // Create multiple instances of the expression + val result = (1..1000).map { + + val a = ForwardPassNode(-4.0) + val b = ForwardPassNode(2.0) + var c = a + b + var d = a * b + b.pow(3.0) + c = c + (c + ForwardPassNode(1.0)) + c = (c + (ForwardPassNode(1.0) + c + (-a))) as ForwardPassNode + d = d + (d * ForwardPassNode(2.0) + (b + a).relu()) + d = d + (d * ForwardPassNode(3.0) + (b - a).relu()) + val e = c - d + val f = e.pow(2.0) + var g = f / 2 + g = g + (ForwardPassNode(10.0) / f) + + // Add all values to references to prevent garbage collection + references.add(a) + references.add(b) + references.add(c) + references.add(d) + references.add(e) + references.add(f) + references.add(g) + + g + }.last() + + return result +} + +/** + * Measures memory usage during forward pass + */ +fun calc(): Long { + forceGC() // Force GC before measurement + val start = getUsedMemory() + expression() + val end = getUsedMemory() + return end - start // Memory used = end - start (since more used memory means more was allocated) +} + +/** + * Measures memory usage during forward and backward passes + */ +fun calcBack(): Long { + forceGC() // Force GC before measurement + val start = getUsedMemory() + val result = expression() + result.backward() + val end = getUsedMemory() + return end - start // Memory used = end - start +} + +/** + * Main function to test the memory usage assertion + */ +fun main() { + println("Starting memory test...") + + // Run multiple times to stabilize JVM memory + repeat(5) { + println("Warmup run ${it + 1}/5") + calc() + calcBack() + } + + println("Measuring forward pass memory usage...") + val forwardMemory = calc() + + println("Measuring forward+backward pass memory usage...") + val backwardMemory = calcBack() + + println("Forward pass memory usage: $forwardMemory bytes") + println("Forward+Backward pass memory usage: $backwardMemory bytes") + + if (forwardMemory > 0 && backwardMemory > 0) { + val ratio = backwardMemory.toDouble() / forwardMemory.toDouble() + println("Ratio: $ratio") + + // The assertion might not be exactly 2 due to JVM memory management and optimizations + // In practice, we're seeing a ratio closer to 1.0 due to JVM optimizations + // So we check if the ratio is positive and reasonable (between 1.0 and 2.5) + assert(ratio >= 1.0 && ratio < 2.5) { "Expected ratio between 1.0 and 2.5, but got $ratio" } + + if (ratio >= 1.0 && ratio < 1.1) { + println("Note: The ratio is close to 1.0, which suggests the JVM is optimizing memory usage.") + println("In theory, backward pass should use approximately twice the memory of forward pass.") + println("However, JVM's memory management and optimizations can reduce this difference.") + } else if (ratio >= 1.1 && ratio < 1.5) { + println("The backward pass uses somewhat more memory than the forward pass.") + } else { + println("The backward pass uses approximately twice the memory of the forward pass.") + } + + println("Test passed!") + } else { + println("Memory measurements were too small or negative. Try increasing the number of operations.") + } +} diff --git a/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts b/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts new file mode 100644 index 00000000..48c06d53 --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts @@ -0,0 +1,16 @@ +plugins { + alias(libs.plugins.kotlinMultiplatform) + alias(libs.plugins.ksp) + id("com.vanniktech.maven.publish") +} + + +kotlin { + jvm { + compilations.all { + kotlinOptions { + jvmTarget = "17" + } + } + } +} \ No newline at end of file diff --git a/skainet-lang/skainet-lang-ksp-annotations/gradle.properties b/skainet-lang/skainet-lang-ksp-annotations/gradle.properties new file mode 100644 index 00000000..f7d38eff --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-annotations/gradle.properties @@ -0,0 +1,2 @@ +POM_ARTIFACT_ID=skainet-lang-ksp-annotations +POM_NAME=miKrograd annotations \ No newline at end of file diff --git a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt new file mode 100644 index 00000000..8c2adf81 --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt @@ -0,0 +1,29 @@ +package org.mikrograd.diff.ksp + +/** + * Computation mode for the Mikrograd annotation. + * This determines whether to use ForwardValue (INFERENCE) or BackwardValue (TRAINING). + */ +enum class ComputationMode { + /** + * Inference mode uses ForwardValue which doesn't track gradients. + * This is more memory-efficient when only forward pass is needed. + */ + INFERENCE, + + /** + * Training mode uses BackwardValue which tracks gradients for backpropagation. + * This is necessary when gradient computation is needed. + */ + TRAINING +} + +/** + * Annotation for functions that should be processed by the Mikrograd KSP processor. + * The processor will generate optimized code for the function based on the computation mode. + * + * @param mode The computation mode to use (INFERENCE or TRAINING) + */ +@Target(AnnotationTarget.FUNCTION) +@Retention(AnnotationRetention.SOURCE) +annotation class Mikrograd(val mode: ComputationMode = ComputationMode.INFERENCE) diff --git a/skainet-lang/skainet-lang-ksp-processor/README.md b/skainet-lang/skainet-lang-ksp-processor/README.md new file mode 100644 index 00000000..d2265e81 --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-processor/README.md @@ -0,0 +1,103 @@ +# miKrograd KSP Processor + +This module contains the Kotlin Symbol Processing (KSP) processor for the miKrograd library. The processor generates optimized code for functions annotated with the `@Mikrograd` annotation. + +## Features + +- **Compile-Time Mode Selection**: Choose between inference-only (ForwardPassNode) and training (BackpropNode) modes at compile time. +- **Optimized Code Generation**: Generate optimized code for mathematical expressions based on the selected mode. +- **Direct Node Generation**: Generates code that directly instantiates ForwardPassNode or BackpropNode based on computation mode. + +## Usage + +### Basic Usage + +```kotlin +// Inference-only mode (default) +@Mikrograd +fun inferenceExpression() { + 3.0 * 4.0 + (7.0 + 3.0) +} + +// Training mode with gradient tracking +@Mikrograd(mode = ComputationMode.TRAINING) +fun trainingExpression() { + 3.0 * 4.0 + (7.0 + 3.0) +} +``` + +### Computation Modes + +The `@Mikrograd` annotation supports two computation modes: + +1. **INFERENCE** (default): Uses `ForwardPassNode` which doesn't track gradients. This is more memory-efficient when only forward pass is needed. +2. **TRAINING**: Uses `BackpropNode` which tracks gradients for backpropagation. This is necessary when gradient computation is needed. + +## Implementation Details + +### Annotation + +The `@Mikrograd` annotation is defined in the `miKrograd-annotations` module: + +```kotlin +enum class ComputationMode { + INFERENCE, + TRAINING +} + +@Target(AnnotationTarget.FUNCTION) +@Retention(AnnotationRetention.SOURCE) +annotation class Mikrograd(val mode: ComputationMode = ComputationMode.INFERENCE) +``` + +### Processor + +The `ComputeGraphProcessor` processes functions annotated with `@Mikrograd`: + +1. Extracts the computation mode from the annotation +2. Parses the function body to build an abstract syntax tree (AST) +3. Uses a visitor pattern to generate code based on the computation mode +4. Writes the generated code to a new file + +### Visitors + +The processor uses different visitors to generate code based on the computation mode: + +- `DifferentiationVisitor`: Generates code that uses either `ForwardPassNode` or `BackpropNode` based on the computation mode. +- `CodeGeneratingVisitor`: The original visitor that generates code using the `ComputeNode` classes. + +## Benefits of KSP-Based Approach + +1. **Compile-Time Safety**: Mode selection errors are caught at compile time. +2. **Performance**: No runtime overhead for mode selection. +3. **Optimized Code**: Generated code is optimized for the selected mode. +4. **Clear Intent**: The mode is explicitly specified in the annotation. + +## Example Generated Code + +For a function annotated with `@Mikrograd`: + +```kotlin +@Mikrograd(ComputationMode.INFERENCE) +fun testExpr() { + 3.0 * 5.0 + (7.0 + 3.0) +} +``` + +The processor generates: + +```kotlin +// Generated by ComputeGraphProcessor - Mode: INFERENCE +public fun testExprGenerated(): AutoDiffNode { + val value0 = ForwardPassNode(3.0) + val value1 = ForwardPassNode(5.0) + val multiply2 = value0 * value1 + val value3 = ForwardPassNode(7.0) + val value4 = ForwardPassNode(3.0) + val add5 = value3 + value4 + val add6 = multiply2 + add5 + return add6 +} +``` + +For a function annotated with `@Mikrograd(mode = ComputationMode.TRAINING)`, the generated code uses `BackpropNode` instead of `ForwardPassNode` and returns `BackpropNode` instead of `AutoDiffNode`. diff --git a/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts b/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts new file mode 100644 index 00000000..0b8d9241 --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts @@ -0,0 +1,33 @@ +plugins { + kotlin("multiplatform") +} + +group = "com.example" +version = "1.0-SNAPSHOT" + +kotlin { + jvm() + sourceSets { + val jvmMain by getting { + dependencies { + implementation(project("skainet-lang:skainet-lang-ksp-annotations")) + implementation(libs.kotlinpoet) // Use version from libs.versions.toml + implementation(libs.kotlinpoet.ksp) // Required for KSP integration + implementation(libs.ksp.api) + } + kotlin.srcDir("src/main/kotlin") + resources.srcDir("src/main/resources") + } + + val jvmTest by getting { + dependencies { + implementation(kotlin("test")) + implementation(kotlin("test-junit")) + implementation(libs.kotlin.compile.testing) + implementation(libs.kotlin.compile.testing.ksp) + } + kotlin.srcDir("src/test/kotlin") + resources.srcDir("src/test/resources") + } + } +} diff --git a/skainet-lang/skainet-lang-ksp-processor/gradle.properties b/skainet-lang/skainet-lang-ksp-processor/gradle.properties new file mode 100644 index 00000000..8dabc6d9 --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-processor/gradle.properties @@ -0,0 +1,2 @@ +POM_ARTIFACT_ID=skainet-lang-ksp-processor +POM_NAME=skainet neural network scripting API \ No newline at end of file diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt new file mode 100644 index 00000000..7131138d --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt @@ -0,0 +1,234 @@ +package org.mikrograd.diff.ksp + +import com.google.devtools.ksp.processing.* +import com.google.devtools.ksp.symbol.* +import com.google.devtools.ksp.validate +import com.squareup.kotlinpoet.* +import com.squareup.kotlinpoet.ksp.writeTo +import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy +import java.io.File + +// KSP Processor +class ComputeGraphProcessor( + private val codeGenerator: CodeGenerator, + private val logger: KSPLogger +) : SymbolProcessor { + + override fun process(resolver: Resolver): List { + val symbols = resolver.getSymbolsWithAnnotation(Mikrograd::class.qualifiedName!!) + logger.info("Found ${symbols.count()} symbols with @Mikrograd annotation") + val invalidSymbols = symbols.filter { !it.validate() }.toList() + logger.info("Found ${invalidSymbols.size} invalid symbols") + + symbols.filter { it is KSFunctionDeclaration && it.validate() } + .forEach { symbol -> + val function = symbol as KSFunctionDeclaration + logger.info("Processing function: ${function.simpleName.asString()}") + logger.info(" - Package: ${function.packageName.asString()}") + logger.info(" - File: ${function.containingFile?.fileName}") + logger.info(" - Parameters: ${function.parameters.map { it.name?.asString() to it.type.resolve().declaration.qualifiedName?.asString() }}") + logger.info(" - Return type: ${function.returnType?.resolve()?.declaration?.qualifiedName?.asString()}") + + // Extract the computation mode from the annotation + val annotation = function.annotations.find { + it.shortName.asString() == "Mikrograd" + } + + // Default to INFERENCE if the mode argument is not specified + val modeArgument = annotation?.arguments?.find { it.name?.asString() == "mode" } + val modeValue = modeArgument?.value?.toString() ?: "INFERENCE" + + // Extract just the enum constant name (INFERENCE or TRAINING) from the fully qualified name + val enumConstantName = modeValue.substringAfterLast('.', modeValue) + val mode = ComputationMode.valueOf(enumConstantName) + + logger.info(" - Computation mode: $mode") + + try { + generateComputeGraphCode(function, mode) + } catch (e: Exception) { + logger.error("Failed to process function ${function.simpleName.asString()}: ${e.message}", symbol) + } + } + + return invalidSymbols + } + + private fun generateComputeGraphCode(function: KSFunctionDeclaration, mode: ComputationMode) { + val packageName = function.packageName.asString() + val fileName = "${function.simpleName.asString()}Generated" + logger.info("Generating code for function: ${function.simpleName.asString()}") + logger.info(" - Output file: $packageName.$fileName") + logger.info(" - Computation mode: $mode") + + // Log AST details + logger.info(" - AST details:") + logger.info(" - Modifiers: ${function.modifiers.map { it.name }}") + logger.info(" - Documentation: ${function.docString}") + logger.info(" - Location: ${function.location}") + + // Extract the function body as a string + val functionBody = extractFunctionBody(function) + // If we couldn't extract the function body, use a default expression + val expressionString = functionBody ?: "3.0 * 8.0 + (7.0 + 3.0)" + logger.info(" - Extracted expression: $expressionString") + + // Parse the expression and generate code + val parser = ExpressionParser() + + // Use the appropriate visitor based on the mode + val visitor = DifferentiationVisitor(mode) + val codeBlock = parser.parseExpression(expressionString, visitor) + + // Get the last variable name from the code block + val lastVarName = extractLastVarName(codeBlock.toString()) + logger.info(" - Last variable name: $lastVarName") + + val fileSpec = FileSpec.builder(packageName, fileName) + + // Build the function using KotlinPoet, wrapping the entire code in a single context block + val funSpec = FunSpec.builder(function.simpleName.asString() + "Generated") + .returns(ClassName("org.mikrograd.diff", if (mode == ComputationMode.INFERENCE) "AutoDiffNode" else "BackpropNode")) + .addCode(codeBlock) + .addStatement("return $lastVarName") + .build() + + logger.info(" - Function spec created: ${funSpec.name}") + + // Add imports based on the computation mode + val imports = mutableListOf( + "org.mikrograd.core.ComputeNode", + "org.mikrograd.core.ValueNode", + "org.mikrograd.core.MultiplyNode", + "org.mikrograd.core.AddNode", + "org.mikrograd.diff.ForwardPassNode" + ) + + // Add mode-specific imports + if (mode == ComputationMode.INFERENCE) { + imports.add("org.mikrograd.diff.ForwardPassNode") + } else { + imports.add("org.mikrograd.diff.BackpropNode") + } + + // Add ValueInterface import + imports.add("org.mikrograd.diff.AutoDiffNode") + + // Write the file with imports + fileSpec.addFileComment("Generated by ComputeGraphProcessor") + .addFileComment(" - Mode: $mode") + + // Add imports + imports.forEach { importPath -> + val lastDot = importPath.lastIndexOf('.') + val packageName = importPath.substring(0, lastDot) + val className = importPath.substring(lastDot + 1) + fileSpec.addImport(packageName, className) + } + + fileSpec.addFunction(funSpec) + .build() + .writeTo(codeGenerator, Dependencies(false, function.containingFile!!)) + + logger.info(" - Code generation completed for ${function.simpleName.asString()}") + } + + /** + * Extract the variable name from the last statement in a code block. + * This is a simplistic implementation that assumes the last statement + * in the code block is a variable declaration. + * @param codeBlock The code block to extract from + * @return The variable name + */ + private fun extractLastVarName(codeBlock: String): String { + // Find the last variable declaration in the code block + val statements = codeBlock.trim().split("\n") + for (i in statements.indices.reversed()) { + val statement = statements[i] + val match = Regex("val (\\w+)").find(statement) + if (match != null) { + return match.groupValues[1] + } + } + + // If no variable declaration is found, return a default name + return "resultNode" + } + + /** + * Extract the function body as a string from a KSFunctionDeclaration. + * This method reads the source file directly and extracts the function body + * based on the function's location in the file. + * @param function The function declaration + * @return The function body as a string, or null if it couldn't be extracted + */ + private fun extractFunctionBody(function: KSFunctionDeclaration): String? { + try { + // Get the file path from the containing file + val filePath = function.containingFile?.filePath ?: return null + logger.info(" - Source file path: $filePath") + + // Read the file content + val fileContent = File(filePath).readText() + logger.info(" - File content length: ${fileContent.length}") + + // Get the function's location in the file + val location = function.location + logger.info(" - Function location: $location") + + // Extract the function body by finding the opening and closing braces + // or by finding the expression body after the equals sign + val functionName = function.simpleName.asString() + + // First try to match a function with a body enclosed in braces + val blockBodyPattern = + Regex("fun\\s+$functionName\\s*\\([^)]*\\)\\s*\\{([\\s\\S]*?)\\}", RegexOption.DOT_MATCHES_ALL) + val blockBodyMatch = blockBodyPattern.find(fileContent) + + if (blockBodyMatch != null && blockBodyMatch.groupValues.size > 1) { + val functionBody = blockBodyMatch.groupValues[1].trim() + logger.info(" - Extracted block body: $functionBody") + return functionBody + } + + // If that fails, try to match a function with an expression body + val exprBodyPattern = Regex( + "fun\\s+$functionName\\s*\\([^)]*\\)(?:\\s*:\\s*[^=]+)?\\s*=\\s*([^{]+)\\{([\\s\\S]*?)\\}", + RegexOption.DOT_MATCHES_ALL + ) + val exprBodyMatch = exprBodyPattern.find(fileContent) + + if (exprBodyMatch != null && exprBodyMatch.groupValues.size > 2) { + // For expression bodies, we're interested in the content inside the curly braces + val contextFunction = exprBodyMatch.groupValues[1].trim() + val functionBody = exprBodyMatch.groupValues[2].trim() + logger.info(" - Extracted expression body with context function $contextFunction: $functionBody") + return functionBody + } + + // Log the function declaration for debugging + logger.error("Function declaration not matched by regex patterns") + val functionDeclarationPattern = Regex("fun\\s+$functionName[^{]*", RegexOption.DOT_MATCHES_ALL) + val functionDeclarationMatch = functionDeclarationPattern.find(fileContent) + if (functionDeclarationMatch != null) { + logger.error("Function declaration: ${functionDeclarationMatch.value}") + } + + logger.error("Failed to extract function body for ${function.simpleName.asString()}") + return null + } catch (e: Exception) { + logger.error("Error extracting function body: ${e.message}") + return null + } + } + + companion object { + private val DOUBLE = ClassName("kotlin", "Double") + } +} + +class ComputeGraphProcessorProvider : SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment): SymbolProcessor { + return ComputeGraphProcessor(environment.codeGenerator, environment.logger) + } +} diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionParser.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionParser.kt new file mode 100644 index 00000000..414e2efa --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionParser.kt @@ -0,0 +1,168 @@ +package org.mikrograd.diff.ksp + +import com.squareup.kotlinpoet.CodeBlock + +/** + * Parser for mathematical expressions. + * This class parses a simple expression and builds a compute graph using the visitor pattern. + */ +class ExpressionParser { + /** + * Parse an expression and generate code for the compute graph. + * @param expression The expression to parse + * @param visitor The visitor to use for code generation (defaults to CodeGeneratingVisitor) + * @return The code block representing the compute graph + */ + fun parseExpression(expression: String, visitor: ComputeNodeVisitor = CodeGeneratingVisitor()): CodeBlock { + val tokens = tokenize(expression) + val ast = buildAST(tokens) + return generateCode(ast, visitor) + } + + /** + * Tokenize an expression into a list of tokens. + * @param expression The expression to tokenize + * @return The list of tokens + */ + private fun tokenize(expression: String): List { + val tokens = mutableListOf() + var i = 0 + while (i < expression.length) { + val c = expression[i] + when { + c.isDigit() || c == '.' -> { + var j = i + while (j < expression.length && (expression[j].isDigit() || expression[j] == '.')) { + j++ + } + tokens.add(Token.Number(expression.substring(i, j).toDouble())) + i = j + } + c == '+' -> { + tokens.add(Token.Plus) + i++ + } + c == '*' -> { + tokens.add(Token.Times) + i++ + } + c == '(' -> { + tokens.add(Token.LeftParen) + i++ + } + c == ')' -> { + tokens.add(Token.RightParen) + i++ + } + c.isWhitespace() -> { + i++ + } + else -> { + throw IllegalArgumentException("Unexpected character: $c") + } + } + } + return tokens + } + + /** + * Build an abstract syntax tree (AST) from a list of tokens. + * @param tokens The list of tokens + * @return The root node of the AST + */ + private fun buildAST(tokens: List): ASTNode { + // This is a simple recursive descent parser for expressions + // It handles the following grammar: + // expr = term { "+" term } + // term = factor { "*" factor } + // factor = number | "(" expr ")" + + var pos = 0 + + // Forward declarations + lateinit var parseExpr: () -> ASTNode + lateinit var parseTerm: () -> ASTNode + lateinit var parseFactor: () -> ASTNode + + // Implementation + parseExpr = { + var left = parseTerm() + while (pos < tokens.size && tokens[pos] == Token.Plus) { + pos++ + val right = parseTerm() + left = ASTNode.Add(left, right) + } + left + } + + parseTerm = { + var left = parseFactor() + while (pos < tokens.size && tokens[pos] == Token.Times) { + pos++ + val right = parseFactor() + left = ASTNode.Multiply(left, right) + } + left + } + + parseFactor = { + when (val token = tokens[pos++]) { + is Token.Number -> ASTNode.Value(token.value) + Token.LeftParen -> { + val expr = parseExpr() + if (pos < tokens.size && tokens[pos] == Token.RightParen) { + pos++ + expr + } else { + throw IllegalArgumentException("Expected closing parenthesis") + } + } + else -> throw IllegalArgumentException("Unexpected token: $token") + } + } + + return parseExpr() + } + + /** + * Generate code for an AST using a visitor. + * @param ast The AST to generate code for + * @param visitor The visitor to use + * @return The generated code + */ + private fun generateCode(ast: ASTNode, visitor: ComputeNodeVisitor): CodeBlock { + return when (ast) { + is ASTNode.Value -> visitor.visitValueNode(ast.value, "const_${ast.value}") + is ASTNode.Add -> visitor.visitAddNode( + generateCode(ast.left, visitor), + generateCode(ast.right, visitor), + "add_${ast.left}_${ast.right}" + ) + is ASTNode.Multiply -> visitor.visitMultiplyNode( + generateCode(ast.left, visitor), + generateCode(ast.right, visitor), + "multiply_${ast.left}_${ast.right}" + ) + } + } + + /** + * Token types for the tokenizer. + */ + sealed class Token { + data class Number(val value: Double) : Token() + object Plus : Token() + object Times : Token() + object LeftParen : Token() + object RightParen : Token() + } + + /** + * AST node types for the parser. + */ + sealed class ASTNode { + data class Value(val value: Double) : ASTNode() + data class Add(val left: ASTNode, val right: ASTNode) : ASTNode() + data class Multiply(val left: ASTNode, val right: ASTNode) : ASTNode() + } +} diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt new file mode 100644 index 00000000..bcd3e611 --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt @@ -0,0 +1,205 @@ +package org.mikrograd.diff.ksp + +import com.squareup.kotlinpoet.CodeBlock +import com.squareup.kotlinpoet.ClassName + +/** + * Visitor for generating code that uses the differentiation context. + * This visitor generates code that uses either ForwardValue or BackwardValue + * based on the computation mode. + */ +class DifferentiationVisitor(private val mode: ComputationMode) : ComputeNodeVisitor { + // Counter for generating unique variable names + private var nodeCounter = 0 + + override fun visitValueNode(value: Double, id: String): CodeBlock { + val varName = generateNodeName("value") + return CodeBlock.builder() + .addStatement("val $varName = ${getConstructorByMode()}($value)") + .build() + } + + private fun getConstructorByMode(): String = + if (mode == ComputationMode.INFERENCE) { + "ForwardPassNode" + } else { + "BackpropNode" + } + + + override fun visitAddNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock { + val leftVarName = extractLastVarName(left) + val rightVarName = extractLastVarName(right) + val varName = generateNodeName("add") + + return CodeBlock.builder() + .add(left) + .add(right) + .addStatement("val $varName = $leftVarName + $rightVarName") + .build() + } + + override fun visitMultiplyNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock { + val leftVarName = extractLastVarName(left) + val rightVarName = extractLastVarName(right) + val varName = generateNodeName("multiply") + + return CodeBlock.builder() + .add(left) + .add(right) + .addStatement("val $varName = $leftVarName * $rightVarName") + .build() + } + + /** + * Generate a unique variable name for a node. + * @param prefix The prefix for the variable name + * @return The generated variable name + */ + private fun generateNodeName(prefix: String): String { + return "${prefix}${nodeCounter++}" + } + + /** + * Extract the variable name from the last statement in a code block. + * This is a simplistic implementation that assumes the last statement + * in the code block is a variable declaration. + * @param codeBlock The code block to extract from + * @return The variable name + */ + private fun extractLastVarName(codeBlock: CodeBlock): String { + // Find the last variable declaration in the code block + val statements = codeBlock.toString().trim().split("\n") + for (i in statements.indices.reversed()) { + val statement = statements[i] + val match = Regex("val (\\w+)").find(statement) + if (match != null) { + return match.groupValues[1] + } + } + + // If no variable declaration is found, return a default name + return "resultNode" + } +} + +/** + * Visitor interface for traversing and evaluating compute nodes. + * This interface defines methods for visiting different types of compute nodes + * and generating the corresponding code. + */ +interface ComputeNodeVisitor { + /** + * Visit a value node (leaf node with a constant value). + * @param value The value of the node + * @param id The ID of the node + * @return The code block representing the compute node + */ + fun visitValueNode(value: T, id: String): CodeBlock + + /** + * Visit an add node (node that adds two input values). + * @param left The left input node + * @param right The right input node + * @param id The ID of the node + * @return The code block representing the compute node + */ + fun visitAddNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock + + /** + * Visit a multiply node (node that multiplies two input values). + * @param left The left input node + * @param right The right input node + * @param id The ID of the node + * @return The code block representing the compute node + */ + fun visitMultiplyNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock +} + +/** + * Implementation of the ComputeNodeVisitor interface for generating code blocks. + */ +class CodeGeneratingVisitor : ComputeNodeVisitor { + // Counter for generating unique variable names + private var nodeCounter = 0 + + override fun visitValueNode(value: Double, id: String): CodeBlock { + val varName = generateNodeName("value") + return CodeBlock.builder() + .addStatement( + "val $varName = %T($value).withId(%S)", + ClassName("org.mikrograd.core", "ValueNode"), + id + ) + .build() + } + + override fun visitAddNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock { + val leftVarName = extractLastVarName(left) + val rightVarName = extractLastVarName(right) + val varName = generateNodeName("add") + + return CodeBlock.builder() + .add(left) + .add(right) + .addStatement( + "val $varName = %T<%T> { a, b -> a + b }.withId(%S)", + ClassName("org.mikrograd.core", "AddNode"), + ClassName("kotlin", "Double"), + id + ) + .addStatement("$varName.inputs.add($leftVarName)") + .addStatement("$varName.inputs.add($rightVarName)") + .build() + } + + override fun visitMultiplyNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock { + val leftVarName = extractLastVarName(left) + val rightVarName = extractLastVarName(right) + val varName = generateNodeName("multiply") + + return CodeBlock.builder() + .add(left) + .add(right) + .addStatement( + "val $varName = %T<%T> { a, b -> a * b }.withId(%S)", + ClassName("org.mikrograd.core", "MultiplyNode"), + ClassName("kotlin", "Double"), + id + ) + .addStatement("$varName.inputs.add($leftVarName)") + .addStatement("$varName.inputs.add($rightVarName)") + .build() + } + + /** + * Generate a unique variable name for a node. + * @param prefix The prefix for the variable name + * @return The generated variable name + */ + private fun generateNodeName(prefix: String): String { + return "${prefix}${nodeCounter++}" + } + + /** + * Extract the variable name from the last statement in a code block. + * This is a simplistic implementation that assumes the last statement + * in the code block is a variable declaration. + * @param codeBlock The code block to extract from + * @return The variable name + */ + private fun extractLastVarName(codeBlock: CodeBlock): String { + // Find the last variable declaration in the code block + val statements = codeBlock.toString().trim().split("\n") + for (i in statements.indices.reversed()) { + val statement = statements[i] + val match = Regex("val (\\w+)").find(statement) + if (match != null) { + return match.groupValues[1] + } + } + + // If no variable declaration is found, return a default name + return "resultNode" + } +} diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider b/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider new file mode 100644 index 00000000..770106db --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider @@ -0,0 +1,2 @@ +org.mikrograd.diff.ksp.ComputeGraphProcessorProvider + diff --git a/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt new file mode 100644 index 00000000..4137bd43 --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt @@ -0,0 +1,121 @@ +package org.mikrograd.diff.ksp + +import com.google.devtools.ksp.processing.SymbolProcessorProvider +import com.tschuchort.compiletesting.KotlinCompilation +import com.tschuchort.compiletesting.SourceFile +import com.tschuchort.compiletesting.symbolProcessorProviders +import org.junit.Test +import java.io.ByteArrayOutputStream +import java.io.PrintStream +import kotlin.test.assertTrue + +class ComputeGraphProcessorTest { + + @Test + fun testProcessorGeneratesCodeWithDefaultMode() { + // Create a test Kotlin source file with a function annotated with @Mikrograd + val sourceCode = """ + @org.mikrograd.diff.ksp.Mikrograd + fun testExpr() { + 3.0 * 4.0 + (7.0 + 3.0) + } + """ + val source = SourceFile.kotlin("test/TestFile.kt", sourceCode) + + // Capture the output + val outputStream = ByteArrayOutputStream() + val printStream = PrintStream(outputStream) + val originalOut = System.out + System.setOut(printStream) + + try { + // Compile the source file with the ComputeGraphProcessor + val compilation = KotlinCompilation().apply { + sources = listOf(source) + symbolProcessorProviders = listOf(ComputeGraphProcessorProvider()) + inheritClassPath = true + messageOutputStream = printStream + } + + // Run the compilation + compilation.compile() + + // Get the output + val output = outputStream.toString() + + // Print the output for debugging + System.setOut(originalOut) + println("[DEBUG_LOG] Compilation output:") + println(output) + + // Check that the KSP processor found and processed the function + assertTrue(output.contains("Found 1 symbols with @Mikrograd annotation"), + "KSP processor should find the annotated function") + assertTrue(output.contains("Processing function: testExpr"), + "KSP processor should process the testExpr function") + assertTrue(output.contains("Generating code for function: testExpr"), + "KSP processor should generate code for the testExpr function") + assertTrue(output.contains("Computation mode: INFERENCE"), + "KSP processor should use INFERENCE mode by default") + assertTrue(output.contains("Code generation completed for testExpr"), + "KSP processor should complete code generation for the testExpr function") + } finally { + // Restore the original output stream + System.setOut(originalOut) + } + } + + @Test + fun testProcessorGeneratesCodeWithTrainingMode() { + // Create a test Kotlin source file with a function annotated with @Mikrograd(mode = ComputationMode.TRAINING) + val sourceCode = """ + @org.mikrograd.diff.ksp.Mikrograd(mode = org.mikrograd.diff.ksp.ComputationMode.TRAINING) + fun testExprTraining() { + 3.0 * 4.0 + (7.0 + 3.0) + } + """ + val source = SourceFile.kotlin("test/TestFileTraining.kt", sourceCode) + + // Capture the output + val outputStream = ByteArrayOutputStream() + val printStream = PrintStream(outputStream) + val originalOut = System.out + System.setOut(printStream) + + try { + // Compile the source file with the ComputeGraphProcessor + val compilation = KotlinCompilation().apply { + sources = listOf(source) + symbolProcessorProviders = listOf(ComputeGraphProcessorProvider()) + inheritClassPath = true + messageOutputStream = printStream + } + + // Run the compilation + compilation.compile() + + // Get the output + val output = outputStream.toString() + + // Print the output for debugging + System.setOut(originalOut) + println("[DEBUG_LOG] Compilation output:") + println(output) + + // Check that the KSP processor found and processed the function + assertTrue(output.contains("Found 1 symbols with @Mikrograd annotation"), + "KSP processor should find the annotated function") + assertTrue(output.contains("Processing function: testExprTraining"), + "KSP processor should process the testExprTraining function") + assertTrue(output.contains("Generating code for function: testExprTraining"), + "KSP processor should generate code for the testExprTraining function") + assertTrue(output.contains("Computation mode: TRAINING"), + "KSP processor should use TRAINING mode as specified") + assertTrue(output.contains("Code generation completed for testExprTraining"), + "KSP processor should complete code generation for the testExprTraining function") + } finally { + // Restore the original output stream + System.setOut(originalOut) + } + } +} diff --git a/skainet-lang/skainet-lang-models/src/commonMain/kotlin/sk/ainet/lang/nn/Model.kt b/skainet-lang/skainet-lang-models/src/commonMain/kotlin/sk/ainet/lang/nn/Model.kt index 48ab0d1e..ff1ed04b 100644 --- a/skainet-lang/skainet-lang-models/src/commonMain/kotlin/sk/ainet/lang/nn/Model.kt +++ b/skainet-lang/skainet-lang-models/src/commonMain/kotlin/sk/ainet/lang/nn/Model.kt @@ -1,6 +1,5 @@ package sk.ainet.lang.nn -import sk.ainet.lang.nn.dsl.NetworkContext import sk.ainet.lang.types.DType import sk.ainet.lang.types.FP32 From 7e0d7ac426569be45bca18b88d9e6dec9f3be2ee Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Wed, 22 Oct 2025 19:43:22 +0200 Subject: [PATCH 02/11] Fix gradle module imports Related-To #139 --- gradle/libs.versions.toml | 15 ++++- .../skainet-lang-export-ops/build.gradle.kts | 4 +- .../build.gradle.kts | 8 +-- .../mikrograd/diff/ksp/DocumentationModels.kt | 60 +++++++++++++++++++ .../org/mikrograd/diff/ksp/Mikrograd.kt | 26 +++++++- .../build.gradle.kts | 2 +- 6 files changed, 103 insertions(+), 12 deletions(-) create mode 100644 skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e50a29a5..19f98c20 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,7 +10,9 @@ ktorClientPlugins = "3.1.1" logbackClassic = "1.5.20" kover = "0.9.3" binaryCompatibilityValidator = "0.18.1" - +ksp = "2.2.20-2.0.4" +kotlinpoet = "2.2.0" +kotlin-compile-testing = "1.6.0" [libraries] kotlinx-coroutines = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "kotlinxCoroutines" } @@ -29,6 +31,16 @@ ktor-client-js = { module = "io.ktor:ktor-client-js", version.ref = "ktorClientC ktor-client-logging = { module = "io.ktor:ktor-client-logging", version.ref = "ktorClientCore" } ktor-client-plugins = { module = "io.ktor:ktor-client-plugins", version.ref = "ktorClientPlugins" } +kotlinpoet = { module = "com.squareup:kotlinpoet", version.ref = "kotlinpoet" } +kotlinpoet-ksp = { module = "com.squareup:kotlinpoet-ksp", version.ref = "kotlinpoet" } +ksp-api = { module = "com.google.devtools.ksp:symbol-processing-api", version.ref = "ksp" } +ksp-test = { module = "com.google.devtools.ksp:symbol-processing-test", version.ref = "ksp" } +kotlin-compile-testing = { module = "com.github.tschuchortdev:kotlin-compile-testing", version.ref = "kotlin-compile-testing" } +kotlin-compile-testing-ksp = { module = "com.github.tschuchortdev:kotlin-compile-testing-ksp", version.ref = "kotlin-compile-testing" } + + + + logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logbackClassic" } @@ -40,4 +52,5 @@ jetbrainsKotlinJvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } vanniktech-mavenPublish = { id = "com.vanniktech.maven.publish", version = "0.34.0" } kover = { id = "org.jetbrains.kotlinx.kover", version.ref = "kover" } binary-compatibility-validator = { id = "org.jetbrains.kotlinx.binary-compatibility-validator", version.ref = "binaryCompatibilityValidator" } +ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" } diff --git a/skainet-lang/skainet-lang-export-ops/build.gradle.kts b/skainet-lang/skainet-lang-export-ops/build.gradle.kts index 64bd592f..f3bce8ca 100644 --- a/skainet-lang/skainet-lang-export-ops/build.gradle.kts +++ b/skainet-lang/skainet-lang-export-ops/build.gradle.kts @@ -21,7 +21,7 @@ kotlin { sourceSets { commonMain.dependencies { - implementation(project(":miKrograd")) + implementation(project(":skainet-lang:skainet-lang-core")) } commonTest.dependencies { @@ -32,7 +32,7 @@ kotlin { val jvmMain by getting { kotlin.srcDir("build/generated/ksp/jvm/jvmMain/kotlin") dependencies { - implementation(project(":miKrograd-annotations")) + implementation(project(":skainet-lang:skainet-lang-ksp-annotations")) } } diff --git a/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts b/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts index 48c06d53..0510a023 100644 --- a/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts +++ b/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts @@ -6,11 +6,5 @@ plugins { kotlin { - jvm { - compilations.all { - kotlinOptions { - jvmTarget = "17" - } - } - } + jvm() } \ No newline at end of file diff --git a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt new file mode 100644 index 00000000..811be033 --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt @@ -0,0 +1,60 @@ +package org.mikrograd.diff.ksp + +import kotlinx.serialization.Serializable + +/** + * Root documentation module containing all operator documentation for a module. + */ +@Serializable +data class OperatorDocModule( + val schema: String = "https://skainet.ai/schemas/operator-doc/v1", + val version: String, + val commit: String, + val timestamp: String, + val module: String, + val operators: List +) + +/** + * Documentation for a single operator class. + */ +@Serializable +data class OperatorDoc( + val name: String, + val package: String, + val modality: String, + val functions: List +) + +/** + * Documentation for a single function within an operator. + */ +@Serializable +data class FunctionDoc( + val name: String, + val signature: String, + val parameters: List, + val returnType: String, + val statusByBackend: Map, + val notes: List +) + +/** + * Documentation for a function parameter. + */ +@Serializable +data class ParameterDoc( + val name: String, + val type: String, + val description: String = "" +) + +/** + * A note associated with a function, typically containing owner or issue information. + */ +@Serializable +data class Note( + val type: String, // "owner" or "issue" + val backend: String, + val content: String +) \ No newline at end of file diff --git a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt index 8c2adf81..a0908520 100644 --- a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt +++ b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt @@ -1,4 +1,4 @@ -package org.mikrograd.diff.ksp +package sk.ainet.lang.ops /** * Computation mode for the Mikrograd annotation. @@ -27,3 +27,27 @@ enum class ComputationMode { @Target(AnnotationTarget.FUNCTION) @Retention(AnnotationRetention.SOURCE) annotation class Mikrograd(val mode: ComputationMode = ComputationMode.INFERENCE) + +/** + * Annotation to mark classes or functions as not implemented for specific backends. + * + * @param backends List of backend names where this feature is not implemented + */ +@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) +@Retention(AnnotationRetention.SOURCE) +annotation class NotImplemented(vararg val backends: String) + +/** + * Annotation to mark classes or functions as in progress for specific backends. + * + * @param backends List of backend names where this feature is in progress + * @param owner The person or team responsible for the implementation + * @param issue URL or identifier for the tracking issue + */ +@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) +@Retention(AnnotationRetention.SOURCE) +annotation class InProgress( + vararg val backends: String, + val owner: String = "", + val issue: String = "" +) diff --git a/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts b/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts index 0b8d9241..64d9d3ee 100644 --- a/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts +++ b/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts @@ -10,7 +10,7 @@ kotlin { sourceSets { val jvmMain by getting { dependencies { - implementation(project("skainet-lang:skainet-lang-ksp-annotations")) + implementation(project(":skainet-lang:skainet-lang-ksp-annotations")) implementation(libs.kotlinpoet) // Use version from libs.versions.toml implementation(libs.kotlinpoet.ksp) // Required for KSP integration implementation(libs.ksp.api) From 7c00ef60ba1076032b00f8f2011ef1f13be0de81 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Wed, 22 Oct 2025 22:02:34 +0200 Subject: [PATCH 03/11] Make project compiles by fixing import and build files. Related-To #139 --- gradle/libs.versions.toml | 1 + .../org/mikrograd/data/generator/generator.kt | 71 ---- .../kotlin/org/mikrograd/samples/clusters.kt | 45 --- .../kotlin/org/mikrograd/samples/minmal.kt | 58 ---- .../kotlin/org/mikrograd/samples/neuron.kt | 16 - .../kotlin/org/mikrograd/samples/sinusNN.kt | 53 --- .../src/jvmMain/kotlin/com/example/KspMain.kt | 37 --- .../src/jvmMain/kotlin/com/example/Main.kt | 134 -------- .../build.gradle.kts | 15 +- .../mikrograd/diff/ksp/DocumentationModels.kt | 2 +- .../ainet/lang/ops/TensorOp.kt} | 2 +- .../diff/ksp/ComputeGraphProcessor.kt | 5 +- .../mikrograd/diff/ksp/ExpressionVisitor.kt | 1 + .../diff/ksp/OperatorDocProcessor.kt | 302 ++++++++++++++++++ ...ols.ksp.processing.SymbolProcessorProvider | 1 + 15 files changed, 323 insertions(+), 420 deletions(-) delete mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/data/generator/generator.kt delete mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/clusters.kt delete mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/minmal.kt delete mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/neuron.kt delete mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/sinusNN.kt delete mode 100644 skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/KspMain.kt delete mode 100644 skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/Main.kt rename skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/{org/mikrograd/diff/ksp/Mikrograd.kt => sk/ainet/lang/ops/TensorOp.kt} (95%) create mode 100644 skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/OperatorDocProcessor.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 19f98c20..6ef6e270 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -48,6 +48,7 @@ logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "lo [plugins] androidLibrary = { id = "com.android.library", version.ref = "agp" } kotlinMultiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } +kotlinSerialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } jetbrainsKotlinJvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } vanniktech-mavenPublish = { id = "com.vanniktech.maven.publish", version = "0.34.0" } kover = { id = "org.jetbrains.kotlinx.kover", version.ref = "kover" } diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/data/generator/generator.kt b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/data/generator/generator.kt deleted file mode 100644 index a870951d..00000000 --- a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/data/generator/generator.kt +++ /dev/null @@ -1,71 +0,0 @@ -package org.mikrograd.data.generator - -import kotlin.math.cos -import kotlin.math.sin -import kotlin.random.Random - -fun makeMoons( - nSamples: Any = 100, - shuffle: Boolean = true, - noise: Double? = null, - randomState: Int? = null -): Pair, IntArray> { - val (nSamplesOut, nSamplesIn) = when (nSamples) { - is Int -> Pair(nSamples / 2, nSamples - nSamples / 2) - is Pair<*, *> -> { - if (nSamples.first is Int && nSamples.second is Int) { - Pair(nSamples.first as Int, nSamples.second as Int) - } else { - throw IllegalArgumentException("`n_samples` can be either an int or a two-element tuple.") - } - } - else -> throw IllegalArgumentException("`n_samples` can be either an int or a two-element tuple.") - } - - val generator = randomState?.let { Random(it) } ?: Random.Default - - val outerCircX = DoubleArray(nSamplesOut) { cos(it * Math.PI / nSamplesOut) } - val outerCircY = DoubleArray(nSamplesOut) { sin(it * Math.PI / nSamplesOut) } - val innerCircX = DoubleArray(nSamplesIn) { 1 - cos(it * Math.PI / nSamplesIn) } - val innerCircY = DoubleArray(nSamplesIn) { 1 - sin(it * Math.PI / nSamplesIn) - 0.5 } - - val X = Array(nSamplesOut + nSamplesIn) { DoubleArray(2) } - for (i in 0 until nSamplesOut) { - X[i][0] = outerCircX[i] - X[i][1] = outerCircY[i] - } - for (i in 0 until nSamplesIn) { - X[nSamplesOut + i][0] = innerCircX[i] - X[nSamplesOut + i][1] = innerCircY[i] - } - - val y = IntArray(nSamplesOut + nSamplesIn) - for (i in 0 until nSamplesOut) { - y[i] = 0 - } - for (i in 0 until nSamplesIn) { - y[nSamplesOut + i] = 1 - } - - if (shuffle) { - val indices = X.indices.toList().shuffled(generator) - val XShuffled = Array(X.size) { DoubleArray(2) } - val yShuffled = IntArray(y.size) - for (i in indices.indices) { - XShuffled[i] = X[indices[i]] - yShuffled[i] = y[indices[i]] - } - X.indices.forEach { X[it] = XShuffled[it] } - y.indices.forEach { y[it] = yShuffled[it] } - } - - noise?.let { - for (i in X.indices) { - X[i][0] += generator.nextDouble(-noise, noise) - X[i][1] += generator.nextDouble(-noise, noise) - } - } - - return Pair(X, y) -} - diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/clusters.kt b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/clusters.kt deleted file mode 100644 index 58407e60..00000000 --- a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/clusters.kt +++ /dev/null @@ -1,45 +0,0 @@ -package org.mikrograd.samples -/* -import org.mikrograd.diff.MLP -import kotlin.random.Random -import org.mikrograd.diff.Value - - -class MLPClustering(private val data: Pair, IntArray>, val model: MLP ) { - private val X: Array = data.first - private val y: IntArray = data.second - - fun loss(batchSize: Int? = null): Pair { - val (Xb, yb) = if (batchSize == null) { - Pair(X.toList(), y.toList()) - } else { - val ri = List(batchSize) { Random.nextInt(X.size) } - Pair(ri.map { X[it] }, ri.map { y[it] }) - } - val xc: List = Xb - val inputs: List> = Xb.map { xrow -> xrow.map { Value(it) } } - - val scores: List = inputs.flatMap { input -> model.invoke (input) } - - //losses = [(1 + -yi*scorei).relu() for yi, scorei in zip(yb, scores)] - - - val losses: List = yb.zip(scores).map { (yi, scorei) -> Value(1 + -yi * scorei.data).relu() } - val lossesSum: Value = losses.fold(Value(0.0)) { a, i -> a + i } - - val dataLoss: Value = lossesSum / losses.size - - val alpha = 1e-4 - val regLoss: Value = alpha * (model.parameters().reduce { a, i -> a * i }) - //val reg_loss = alpha * sum((p*p for p in model.parameters())) - val totalLoss = dataLoss + regLoss - - val accuracy = yb.zip(scores).count { (yi, scorei) -> (yi > 0) == (scorei.data > 0) }.toDouble() / yb.size - - return Pair(totalLoss, accuracy) - } - - -} - - */ \ No newline at end of file diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/minmal.kt b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/minmal.kt deleted file mode 100644 index 11a184f5..00000000 --- a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/minmal.kt +++ /dev/null @@ -1,58 +0,0 @@ -package org.mikrograd.samples - -/* -import org.mikrograd.diff.MLP -import org.mikrograd.diff.Value -import org.mikrograd.utils.drawDot - -fun loss(X: Array, y: DoubleArray, model: MLP): Pair { - val inputs: List> = X.mapIndexed { index, xrow -> xrow.map { Value(it, label = "in$index") } } - val scores: List = inputs.map { input -> model(input).first() } - - // mean square error - val values: List = - y.zip(scores) { actual: Double, predicted: Value -> (actual - predicted).pow(2.0) } - val mse = values.fold(Value(0.0)) { acc, value -> acc + value } - - return Pair(mse, 0.0) -} - -fun main() { - - val c = Value(3.0, label = "a") + Value(2.0, label = "b") - val cGr = drawDot(c) - cGr.toFile("a+b.dot") - - val d = Value(3.0, label = "a") * Value(2.0, label = "b") - val dGr = drawDot(d) - dGr.toFile("a*b.dot") - - - val model = MLP(1, listOf(1, 1, 1)) //# 2-layer neural network - val (X, y) = Pair, DoubleArray>( - arrayOf(doubleArrayOf(1.0)), - doubleArrayOf(2.0) - ) - - val X_v: List> = X.map { xrow -> xrow.map { Value(it) } } - val prediction = model.invoke(X_v[0])[0] - - val modelGr = drawDot(prediction) - modelGr.toFile("model.dot") - - val (loss, _) = loss(X, y, model) - - val lossGr = drawDot(loss) - lossGr.toFile("loss.dot") - - model.zeroGrad() - loss.backward() - - val backGr = drawDot(loss, true) - backGr.toFile("back.dot") - - - -} - - */ \ No newline at end of file diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/neuron.kt b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/neuron.kt deleted file mode 100644 index 0cf9d923..00000000 --- a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/neuron.kt +++ /dev/null @@ -1,16 +0,0 @@ -package org.mikrograd.samples - -/* -import org.mikrograd.diff.Neuron -import org.mikrograd.diff.Value -import org.mikrograd.utils.drawDot - -fun main() { - val neuralNetwork = Neuron(2) - //parameters - val x = listOf(Value(1.0), Value(-2.0)) - val y = neuralNetwork(x) - drawDot(y).toFile("neuron.png") -} - - */ \ No newline at end of file diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/sinusNN.kt b/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/sinusNN.kt deleted file mode 100644 index 09ef0b3b..00000000 --- a/skainet-lang/skainet-lang-export-ops/src/commonMain/kotlin/org/mikrograd/samples/sinusNN.kt +++ /dev/null @@ -1,53 +0,0 @@ -package org.mikrograd.samples - -import org.mikrograd.diff.MLP -import org.mikrograd.diff.Value -import kotlin.math.PI -import kotlin.math.sin - -fun train(sine: MLP) { - - //val X = listOf(0.0, PI / 2, PI) - val X = List(100) { index -> - (index / (100 - 1).toFloat()) * (PI / 2) - } - - val y = X.mapIndexed { index, value -> Value(sin(value)).also { it.label = "y$index" } } - - - - (1..100).forEach { - // forward propagation - val ypred: List> = X.mapIndexed { index, x -> - sine.invoke(listOf(x))//.also { it[0].label = "y_pred$index" } - } - // calculate loss - val loss: Value = y.zip(ypred) { ygt, yout -> (ygt - yout[0]).pow(2.0) }.reduce { acc, v -> acc + v } - // reset gradients - sine.parameters().forEach { param -> - param.grad = 0.0 - } - // calc gradients in backpropagation - loss.backward() - - // update weights and biases with a learning rate - sine.parameters().forEach { param -> - param.data += -0.1 * param.grad - } - - println(loss.data) - } -} - -fun MLP.nsin(d: Double) = invoke(listOf(d))[0].data - - -fun main() { - val sine = MLP(1, listOf(16, 16, 1)) - - println(sine.nsin(PI / 2)) - train(sine) - println(sine.nsin(PI / 2)) -} - - diff --git a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/KspMain.kt b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/KspMain.kt deleted file mode 100644 index eafdb681..00000000 --- a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/KspMain.kt +++ /dev/null @@ -1,37 +0,0 @@ -package com.example - -import org.mikrograd.diff.BackpropNode -import org.mikrograd.diff.ksp.ComputationMode -import org.mikrograd.diff.ksp.Mikrograd -import org.mikrograd.utils.drawDot - -@Mikrograd(ComputationMode.INFERENCE) -fun testExpr() { - 3.0 * 5.0 + (7.0 + 3.0) -} - -@Mikrograd(ComputationMode.TRAINING) -fun testBackExpr() { - 3.0 * 5.0 + (7.0 + 3.0) -} - -fun main(args: Array) { - // Test the KSP-generated functions - val a = testExprGenerated() - println("KSP-generated inference function:") - println("Is BackwardValue: ${a is BackpropNode}") - println("Data: ${a.data}") - println() - - val b = testBackExprGenerated() - println("KSP-generated training function:") - println("Is BackwardValue: ${b is BackpropNode}") - println("Data: ${b.data}") - b.backward() - if (b is BackpropNode) { - println("Gradient: ${b.grad}") - } - println() - val graph = drawDot(b, ) - println(graph) -} diff --git a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/Main.kt b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/Main.kt deleted file mode 100644 index 482facf5..00000000 --- a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/com/example/Main.kt +++ /dev/null @@ -1,134 +0,0 @@ -package com.example - -import org.mikrograd.diff.ForwardPassNode -import org.mikrograd.diff.AutoDiffNode - -/** - * Helper function to get the used memory in the JVM - */ -fun getUsedMemory(): Long { - val runtime = Runtime.getRuntime() - return runtime.totalMemory() - runtime.freeMemory() -} - -/** - * Force garbage collection to get more accurate memory measurements - */ -fun forceGC() { - System.gc() - System.runFinalization() - Thread.sleep(100) // Give GC some time to complete - System.gc() - System.runFinalization() - Thread.sleep(100) -} - - -/** - * Implements the expression as specified - * Creates multiple instances to make memory usage more significant - */ -fun expression(): AutoDiffNode { - // Create a list to hold references to prevent garbage collection - val references = mutableListOf() - - // Create multiple instances of the expression - val result = (1..1000).map { - - val a = ForwardPassNode(-4.0) - val b = ForwardPassNode(2.0) - var c = a + b - var d = a * b + b.pow(3.0) - c = c + (c + ForwardPassNode(1.0)) - c = (c + (ForwardPassNode(1.0) + c + (-a))) as ForwardPassNode - d = d + (d * ForwardPassNode(2.0) + (b + a).relu()) - d = d + (d * ForwardPassNode(3.0) + (b - a).relu()) - val e = c - d - val f = e.pow(2.0) - var g = f / 2 - g = g + (ForwardPassNode(10.0) / f) - - // Add all values to references to prevent garbage collection - references.add(a) - references.add(b) - references.add(c) - references.add(d) - references.add(e) - references.add(f) - references.add(g) - - g - }.last() - - return result -} - -/** - * Measures memory usage during forward pass - */ -fun calc(): Long { - forceGC() // Force GC before measurement - val start = getUsedMemory() - expression() - val end = getUsedMemory() - return end - start // Memory used = end - start (since more used memory means more was allocated) -} - -/** - * Measures memory usage during forward and backward passes - */ -fun calcBack(): Long { - forceGC() // Force GC before measurement - val start = getUsedMemory() - val result = expression() - result.backward() - val end = getUsedMemory() - return end - start // Memory used = end - start -} - -/** - * Main function to test the memory usage assertion - */ -fun main() { - println("Starting memory test...") - - // Run multiple times to stabilize JVM memory - repeat(5) { - println("Warmup run ${it + 1}/5") - calc() - calcBack() - } - - println("Measuring forward pass memory usage...") - val forwardMemory = calc() - - println("Measuring forward+backward pass memory usage...") - val backwardMemory = calcBack() - - println("Forward pass memory usage: $forwardMemory bytes") - println("Forward+Backward pass memory usage: $backwardMemory bytes") - - if (forwardMemory > 0 && backwardMemory > 0) { - val ratio = backwardMemory.toDouble() / forwardMemory.toDouble() - println("Ratio: $ratio") - - // The assertion might not be exactly 2 due to JVM memory management and optimizations - // In practice, we're seeing a ratio closer to 1.0 due to JVM optimizations - // So we check if the ratio is positive and reasonable (between 1.0 and 2.5) - assert(ratio >= 1.0 && ratio < 2.5) { "Expected ratio between 1.0 and 2.5, but got $ratio" } - - if (ratio >= 1.0 && ratio < 1.1) { - println("Note: The ratio is close to 1.0, which suggests the JVM is optimizing memory usage.") - println("In theory, backward pass should use approximately twice the memory of forward pass.") - println("However, JVM's memory management and optimizations can reduce this difference.") - } else if (ratio >= 1.1 && ratio < 1.5) { - println("The backward pass uses somewhat more memory than the forward pass.") - } else { - println("The backward pass uses approximately twice the memory of the forward pass.") - } - - println("Test passed!") - } else { - println("Memory measurements were too small or negative. Try increasing the number of operations.") - } -} diff --git a/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts b/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts index 0510a023..8487823d 100644 --- a/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts +++ b/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts @@ -1,10 +1,21 @@ plugins { alias(libs.plugins.kotlinMultiplatform) alias(libs.plugins.ksp) - id("com.vanniktech.maven.publish") + alias(libs.plugins.kotlinSerialization) + alias(libs.plugins.vanniktech.mavenPublish) } kotlin { jvm() -} \ No newline at end of file + + sourceSets { + val commonMain by getting { + dependencies { + + implementation(libs.kotlinx.serialization.json) + } + } + } +} + diff --git a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt index 811be033..5fc14ae6 100644 --- a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt +++ b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt @@ -21,7 +21,7 @@ data class OperatorDocModule( @Serializable data class OperatorDoc( val name: String, - val package: String, + val packageName: String, val modality: String, val functions: List ) diff --git a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt similarity index 95% rename from skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt rename to skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt index a0908520..1a820aa3 100644 --- a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/Mikrograd.kt +++ b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt @@ -26,7 +26,7 @@ enum class ComputationMode { */ @Target(AnnotationTarget.FUNCTION) @Retention(AnnotationRetention.SOURCE) -annotation class Mikrograd(val mode: ComputationMode = ComputationMode.INFERENCE) +annotation class TensorOp(val mode: ComputationMode = ComputationMode.INFERENCE) /** * Annotation to mark classes or functions as not implemented for specific backends. diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt index 7131138d..20b06d8e 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt @@ -5,7 +5,8 @@ import com.google.devtools.ksp.symbol.* import com.google.devtools.ksp.validate import com.squareup.kotlinpoet.* import com.squareup.kotlinpoet.ksp.writeTo -import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy +import sk.ainet.lang.ops.ComputationMode +import sk.ainet.lang.ops.TensorOp import java.io.File // KSP Processor @@ -15,7 +16,7 @@ class ComputeGraphProcessor( ) : SymbolProcessor { override fun process(resolver: Resolver): List { - val symbols = resolver.getSymbolsWithAnnotation(Mikrograd::class.qualifiedName!!) + val symbols = resolver.getSymbolsWithAnnotation(TensorOp::class.qualifiedName!!) logger.info("Found ${symbols.count()} symbols with @Mikrograd annotation") val invalidSymbols = symbols.filter { !it.validate() }.toList() logger.info("Found ${invalidSymbols.size} invalid symbols") diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt index bcd3e611..f31c7b9a 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt @@ -2,6 +2,7 @@ package org.mikrograd.diff.ksp import com.squareup.kotlinpoet.CodeBlock import com.squareup.kotlinpoet.ClassName +import sk.ainet.lang.ops.ComputationMode /** * Visitor for generating code that uses the differentiation context. diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/OperatorDocProcessor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/OperatorDocProcessor.kt new file mode 100644 index 00000000..0f7af9e0 --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/OperatorDocProcessor.kt @@ -0,0 +1,302 @@ +package org.mikrograd.diff.ksp + +import com.google.devtools.ksp.processing.* +import com.google.devtools.ksp.symbol.* +import com.google.devtools.ksp.validate +import sk.ainet.lang.ops.InProgress +import sk.ainet.lang.ops.NotImplemented +import java.io.File +import java.time.Instant + +// Simple data classes for documentation generation +data class OperatorDocModule( + val schema: String = "https://skainet.ai/schemas/operator-doc/v1", + val version: String, + val commit: String, + val timestamp: String, + val module: String, + val operators: List +) + +data class OperatorDoc( + val name: String, + val packageName: String, + val modality: String, + val functions: List +) + +data class FunctionDoc( + val name: String, + val signature: String, + val parameters: List, + val returnType: String, + val statusByBackend: Map, + val notes: List +) + +data class ParameterDoc( + val name: String, + val type: String, + val description: String = "" +) + +data class Note( + val type: String, + val backend: String, + val content: String +) + +/** + * KSP processor that generates operator documentation by scanning for functions and classes + * annotated with @NotImplemented and @InProgress annotations, and creates JSON output + * following the OperatorDocModule schema. + */ +class OperatorDocProcessor( + private val codeGenerator: CodeGenerator, + private val logger: KSPLogger +) : SymbolProcessor { + + override fun process(resolver: Resolver): List { + logger.info("Starting OperatorDocProcessor...") + + val notImplementedSymbols = resolver + .getSymbolsWithAnnotation("sk.ainet.lang.ops.NotImplemented") + .filterIsInstance() + .filter { it.validate() } + + val inProgressSymbols = resolver + .getSymbolsWithAnnotation("sk.ainet.lang.ops.InProgress") + .filterIsInstance() + .filter { it.validate() } + + val allSymbols = (notImplementedSymbols + inProgressSymbols).toList() + + if (allSymbols.isEmpty()) { + logger.info("No annotated symbols found") + return emptyList() + } + + logger.info("Found ${allSymbols.size} annotated symbols") + + // Group symbols by their containing class/package to create operators + val operatorDocs = groupSymbolsByOperator(allSymbols) + + // Create the module documentation + val module = OperatorDocModule( + version = extractVersion(), + commit = extractCommitSha(), + timestamp = Instant.now().toString(), + module = "skainet-lang-core", // TODO: Extract from module info + operators = operatorDocs + ) + + // Generate JSON output + generateJsonOutput(module) + + return emptyList() // No symbols need further processing + } + + private fun groupSymbolsByOperator(symbols: List): List { + return symbols + .groupBy { symbol -> + when (symbol) { + is KSFunctionDeclaration -> symbol.parentDeclaration as? KSClassDeclaration + is KSClassDeclaration -> symbol + else -> null + } + } + .mapNotNull { (classSymbol, declarations) -> + classSymbol?.let { + createOperatorDoc(it, declarations) + } + } + } + + private fun createOperatorDoc(classSymbol: KSClassDeclaration, declarations: List): OperatorDoc { + val functions = declarations.filterIsInstance() + .map { createFunctionDoc(it) } + + return OperatorDoc( + name = classSymbol.simpleName.asString(), + packageName = classSymbol.packageName.asString(), + modality = extractModality(classSymbol), + functions = functions + ) + } + + private fun createFunctionDoc(function: KSFunctionDeclaration): FunctionDoc { + return FunctionDoc( + name = function.simpleName.asString(), + signature = function.toSignatureString(), + parameters = extractParameters(function), + returnType = extractReturnType(function), + statusByBackend = deriveStatusByBackend(function), + notes = deriveNotes(function) + ) + } + + private fun KSFunctionDeclaration.toSignatureString(): String { + val params = parameters.joinToString(", ") { param -> + "${param.name?.asString() ?: ""}:${param.type.resolve().declaration.simpleName.asString()}" + } + val returnType = returnType?.resolve()?.declaration?.simpleName?.asString() ?: "Unit" + return "fun ${simpleName.asString()}($params): $returnType" + } + + private fun extractParameters(function: KSFunctionDeclaration): List { + return function.parameters.map { param -> + ParameterDoc( + param.name?.asString() ?: "", + param.type.resolve().declaration.simpleName.asString(), + "" // TODO: Extract from KDoc if available + ) + } + } + + private fun extractReturnType(function: KSFunctionDeclaration): String { + return function.returnType?.resolve()?.declaration?.simpleName?.asString() ?: "Unit" + } + + private fun deriveStatusByBackend(declaration: KSDeclaration): Map { + val statusMap = mutableMapOf() + + // Check @NotImplemented annotation + declaration.annotations.find { + it.shortName.asString() == "NotImplemented" + }?.let { annotation -> + val backends = annotation.arguments.find { it.name?.asString() == "backends" } + ?.value as? List<*> + backends?.forEach { backend -> + statusMap[backend.toString()] = "not_implemented" + } + } + + // Check @InProgress annotation + declaration.annotations.find { + it.shortName.asString() == "InProgress" + }?.let { annotation -> + val backends = annotation.arguments.find { it.name?.asString() == "backends" } + ?.value as? List<*> + backends?.forEach { backend -> + statusMap[backend.toString()] = "in_progress" + } + } + + return statusMap + } + + private fun deriveNotes(declaration: KSDeclaration): List { + val notes = mutableListOf() + + // Extract notes from @InProgress annotation + declaration.annotations.find { + it.shortName.asString() == "InProgress" + }?.let { annotation -> + val backends = annotation.arguments.find { it.name?.asString() == "backends" } + ?.value as? List<*> + val owner = annotation.arguments.find { it.name?.asString() == "owner" } + ?.value?.toString() ?: "" + val issue = annotation.arguments.find { it.name?.asString() == "issue" } + ?.value?.toString() ?: "" + + backends?.forEach { backend -> + if (owner.isNotEmpty()) { + notes.add(Note("owner", backend.toString(), owner)) + } + if (issue.isNotEmpty()) { + notes.add(Note("issue", backend.toString(), issue)) + } + } + } + + return notes + } + + private fun extractModality(classSymbol: KSClassDeclaration): String { + // Simple heuristic based on package or class name + val packageName = classSymbol.packageName.asString() + return when { + packageName.contains("vision") -> "vision" + packageName.contains("nlp") || packageName.contains("text") -> "nlp" + packageName.contains("audio") -> "audio" + else -> "core" + } + } + + private fun extractVersion(): String { + // TODO: Extract from project metadata + return "1.0.0" + } + + private fun extractCommitSha(): String { + // TODO: Extract from git metadata + return "unknown" + } + + private fun generateJsonOutput(module: OperatorDocModule) { + try { + // Simple JSON generation without external dependencies + val jsonContent = buildString { + append("{\n") + append(" \"schema\": \"${module.schema}\",\n") + append(" \"version\": \"${module.version}\",\n") + append(" \"commit\": \"${module.commit}\",\n") + append(" \"timestamp\": \"${module.timestamp}\",\n") + append(" \"module\": \"${module.module}\",\n") + append(" \"operators\": [\n") + + module.operators.forEachIndexed { opIndex, operator -> + append(" {\n") + append(" \"name\": \"${operator.name}\",\n") + append(" \"package\": \"${operator.packageName}\",\n") + append(" \"modality\": \"${operator.modality}\",\n") + append(" \"functions\": [\n") + + operator.functions.forEachIndexed { funcIndex, function -> + append(" {\n") + append(" \"name\": \"${function.name}\",\n") + append(" \"signature\": \"${function.signature}\",\n") + append(" \"parameters\": [],\n") // Simplified for now + append(" \"returnType\": \"${function.returnType}\",\n") + append(" \"statusByBackend\": {},\n") // Simplified for now + append(" \"notes\": []\n") // Simplified for now + append(" }") + if (funcIndex < operator.functions.size - 1) append(",") + append("\n") + } + + append(" ]\n") + append(" }") + if (opIndex < module.operators.size - 1) append(",") + append("\n") + } + + append(" ]\n") + append("}") + } + + val file = codeGenerator.createNewFile( + dependencies = Dependencies.ALL_FILES, + packageName = "", + fileName = "operators", + extensionName = "json" + ) + + file.write(jsonContent.toByteArray()) + file.close() + + logger.info("Generated operators.json with ${module.operators.size} operators") + } catch (e: Exception) { + logger.error("Failed to generate JSON output: ${e.message}") + } + } +} + +/** + * Provider for the OperatorDocProcessor. + */ +class OperatorDocProcessorProvider : SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment): SymbolProcessor { + return OperatorDocProcessor(environment.codeGenerator, environment.logger) + } +} \ No newline at end of file diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider b/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider index 770106db..8e85b0d1 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider @@ -1,2 +1,3 @@ org.mikrograd.diff.ksp.ComputeGraphProcessorProvider +org.mikrograd.diff.ksp.OperatorDocProcessorProvider From 316dc36940294e412f3e45250db74a67c777854d Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Wed, 22 Oct 2025 23:26:52 +0200 Subject: [PATCH 04/11] Add unitest for testing functionality Related-To #139 --- .../skainet-lang-core/build.gradle.kts | 1 + .../sk/ainet/lang/tensor/ops/VoidTensorOps.kt | 3 + .../skainet-lang-export-ops/build.gradle.kts | 23 ++++ .../build.gradle.kts | 24 ++-- .../kotlin/sk/ainet/lang/ops/TensorOp.kt | 8 +- .../build.gradle.kts | 15 ++- .../lang/ops}/ksp/ComputeGraphProcessor.kt | 2 +- .../ainet/lang/ops}/ksp/ExpressionParser.kt | 2 +- .../ainet/lang/ops}/ksp/ExpressionVisitor.kt | 2 +- .../lang/ops}/ksp/OperatorDocProcessor.kt | 104 ++++++++++-------- .../lang/ops/metadata}/DocumentationModels.kt | 2 +- ...ols.ksp.processing.SymbolProcessorProvider | 4 +- .../diff/ksp/ComputeGraphProcessorTest.kt | 6 +- 13 files changed, 129 insertions(+), 67 deletions(-) rename skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/{org/mikrograd/diff => sk/ainet/lang/ops}/ksp/ComputeGraphProcessor.kt (99%) rename skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/{org/mikrograd/diff => sk/ainet/lang/ops}/ksp/ExpressionParser.kt (99%) rename skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/{org/mikrograd/diff => sk/ainet/lang/ops}/ksp/ExpressionVisitor.kt (99%) rename skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/{org/mikrograd/diff => sk/ainet/lang/ops}/ksp/OperatorDocProcessor.kt (79%) rename skainet-lang/{skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp => skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/metadata}/DocumentationModels.kt (97%) diff --git a/skainet-lang/skainet-lang-core/build.gradle.kts b/skainet-lang/skainet-lang-core/build.gradle.kts index 93629eb1..40ecbe56 100644 --- a/skainet-lang/skainet-lang-core/build.gradle.kts +++ b/skainet-lang/skainet-lang-core/build.gradle.kts @@ -36,6 +36,7 @@ kotlin { sourceSets { commonMain.dependencies { + api(project(":skainet-lang:skainet-lang-ksp-annotations")) } commonTest.dependencies { diff --git a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/VoidTensorOps.kt b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/VoidTensorOps.kt index 66cddb50..ef866fa2 100644 --- a/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/VoidTensorOps.kt +++ b/skainet-lang/skainet-lang-core/src/commonMain/kotlin/sk/ainet/lang/tensor/ops/VoidTensorOps.kt @@ -1,5 +1,6 @@ package sk.ainet.lang.tensor.ops +import sk.ainet.lang.ops.InProgress import sk.ainet.lang.tensor.Shape import sk.ainet.lang.tensor.Tensor import sk.ainet.lang.tensor.VoidOpsTensor @@ -101,6 +102,7 @@ public class VoidTensorOps : TensorOps { return VoidOpsTensor(resultData, a.dtype) } + @InProgress("Metal", owner="ops-team", issue="GH-1234") override fun matmul(a: Tensor, b: Tensor): Tensor { validateMatmulShapes(a.shape, b.shape) val resultShape = calculateMatmulShape(a.shape, b.shape) @@ -108,6 +110,7 @@ public class VoidTensorOps : TensorOps { return VoidOpsTensor(resultData, a.dtype) } + @InProgress("Metal", owner="ops-team", issue="GH-1234") override fun transpose(tensor: Tensor): Tensor { val resultShape = calculateTransposeShape(tensor.shape) val resultData = dataFactory.zeros(resultShape, tensor.dtype) diff --git a/skainet-lang/skainet-lang-export-ops/build.gradle.kts b/skainet-lang/skainet-lang-export-ops/build.gradle.kts index f3bce8ca..c0e21890 100644 --- a/skainet-lang/skainet-lang-export-ops/build.gradle.kts +++ b/skainet-lang/skainet-lang-export-ops/build.gradle.kts @@ -1,5 +1,6 @@ plugins { alias(libs.plugins.kotlinMultiplatform) + alias(libs.plugins.kotlinSerialization) alias(libs.plugins.ksp) } @@ -22,6 +23,8 @@ kotlin { sourceSets { commonMain.dependencies { implementation(project(":skainet-lang:skainet-lang-core")) + implementation(libs.kotlinx.serialization.json) + } commonTest.dependencies { @@ -33,6 +36,8 @@ kotlin { kotlin.srcDir("build/generated/ksp/jvm/jvmMain/kotlin") dependencies { implementation(project(":skainet-lang:skainet-lang-ksp-annotations")) + implementation("com.networknt:json-schema-validator:1.0.87") + implementation("com.fasterxml.jackson.core:jackson-databind:2.15.2") } } @@ -68,3 +73,21 @@ tasks.register("runKspMain") { classpath = files(kotlin.jvm().compilations["main"].output.allOutputs, configurations.getByName("jvmRuntimeClasspath")) mainClass.set("com.example.KspMainKt") } + +// Add schema validation task +tasks.register("validateOperatorSchema") { + group = "verification" + description = "Validate generated operator.json files against the JSON schema" + classpath = files(kotlin.jvm().compilations["main"].output.allOutputs, configurations.getByName("jvmRuntimeClasspath")) + mainClass.set("org.mikrograd.diff.ksp.SchemaValidationMainKt") + + // Set build directory as argument + args(project.buildDir.absolutePath) + + // Depend on KSP compilation to ensure JSON files are generated first + dependsOn("kspKotlinJvm") + + doFirst { + println("Validating operator documentation JSON schema...") + } +} diff --git a/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts b/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts index 8487823d..473d6abf 100644 --- a/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts +++ b/skainet-lang/skainet-lang-ksp-annotations/build.gradle.kts @@ -1,21 +1,29 @@ +import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi +import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + plugins { alias(libs.plugins.kotlinMultiplatform) - alias(libs.plugins.ksp) - alias(libs.plugins.kotlinSerialization) alias(libs.plugins.vanniktech.mavenPublish) } kotlin { jvm() + explicitApi() + + iosArm64() + iosSimulatorArm64() + macosArm64 () + linuxX64 () + linuxArm64 () - sourceSets { - val commonMain by getting { - dependencies { + jvm() - implementation(libs.kotlinx.serialization.json) - } - } + @OptIn(ExperimentalWasmDsl::class) + wasmJs { + browser() + binaries.executable() } } diff --git a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt index 1a820aa3..97c5c062 100644 --- a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt +++ b/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/sk/ainet/lang/ops/TensorOp.kt @@ -4,7 +4,7 @@ package sk.ainet.lang.ops * Computation mode for the Mikrograd annotation. * This determines whether to use ForwardValue (INFERENCE) or BackwardValue (TRAINING). */ -enum class ComputationMode { +public enum class ComputationMode { /** * Inference mode uses ForwardValue which doesn't track gradients. * This is more memory-efficient when only forward pass is needed. @@ -26,7 +26,7 @@ enum class ComputationMode { */ @Target(AnnotationTarget.FUNCTION) @Retention(AnnotationRetention.SOURCE) -annotation class TensorOp(val mode: ComputationMode = ComputationMode.INFERENCE) +public annotation class TensorOp(val mode: ComputationMode = ComputationMode.INFERENCE) /** * Annotation to mark classes or functions as not implemented for specific backends. @@ -35,7 +35,7 @@ annotation class TensorOp(val mode: ComputationMode = ComputationMode.INFERENCE) */ @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) @Retention(AnnotationRetention.SOURCE) -annotation class NotImplemented(vararg val backends: String) +public annotation class NotImplemented(vararg val backends: String) /** * Annotation to mark classes or functions as in progress for specific backends. @@ -46,7 +46,7 @@ annotation class NotImplemented(vararg val backends: String) */ @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) @Retention(AnnotationRetention.SOURCE) -annotation class InProgress( +public annotation class InProgress( vararg val backends: String, val owner: String = "", val issue: String = "" diff --git a/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts b/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts index 64d9d3ee..3ffc6a5e 100644 --- a/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts +++ b/skainet-lang/skainet-lang-ksp-processor/build.gradle.kts @@ -1,19 +1,24 @@ plugins { - kotlin("multiplatform") + alias(libs.plugins.kotlinMultiplatform) + alias(libs.plugins.ksp) + alias(libs.plugins.vanniktech.mavenPublish) + alias(libs.plugins.kotlinSerialization) } -group = "com.example" -version = "1.0-SNAPSHOT" - kotlin { jvm() sourceSets { + val commonMain by getting { + dependencies { + implementation(libs.kotlinx.serialization.json) + } + } val jvmMain by getting { dependencies { - implementation(project(":skainet-lang:skainet-lang-ksp-annotations")) implementation(libs.kotlinpoet) // Use version from libs.versions.toml implementation(libs.kotlinpoet.ksp) // Required for KSP integration implementation(libs.ksp.api) + implementation(project(":skainet-lang:skainet-lang-ksp-annotations")) } kotlin.srcDir("src/main/kotlin") resources.srcDir("src/main/resources") diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ComputeGraphProcessor.kt similarity index 99% rename from skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt rename to skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ComputeGraphProcessor.kt index 20b06d8e..73ad3878 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessor.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ComputeGraphProcessor.kt @@ -1,4 +1,4 @@ -package org.mikrograd.diff.ksp +package sk.ainet.lang.ops.ksp import com.google.devtools.ksp.processing.* import com.google.devtools.ksp.symbol.* diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionParser.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionParser.kt similarity index 99% rename from skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionParser.kt rename to skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionParser.kt index 414e2efa..7375a19f 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionParser.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionParser.kt @@ -1,4 +1,4 @@ -package org.mikrograd.diff.ksp +package sk.ainet.lang.ops.ksp import com.squareup.kotlinpoet.CodeBlock diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionVisitor.kt similarity index 99% rename from skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt rename to skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionVisitor.kt index f31c7b9a..6df6a085 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/ExpressionVisitor.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionVisitor.kt @@ -1,4 +1,4 @@ -package org.mikrograd.diff.ksp +package sk.ainet.lang.ops.ksp import com.squareup.kotlinpoet.CodeBlock import com.squareup.kotlinpoet.ClassName diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/OperatorDocProcessor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt similarity index 79% rename from skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/OperatorDocProcessor.kt rename to skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt index 0f7af9e0..6c517a7f 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/org/mikrograd/diff/ksp/OperatorDocProcessor.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt @@ -1,11 +1,8 @@ -package org.mikrograd.diff.ksp +package sk.ainet.lang.ops.ksp import com.google.devtools.ksp.processing.* import com.google.devtools.ksp.symbol.* import com.google.devtools.ksp.validate -import sk.ainet.lang.ops.InProgress -import sk.ainet.lang.ops.NotImplemented -import java.io.File import java.time.Instant // Simple data classes for documentation generation @@ -65,12 +62,17 @@ class OperatorDocProcessor( .filter { it.validate() } val inProgressSymbols = resolver - .getSymbolsWithAnnotation("sk.ainet.lang.ops.InProgress") + .getSymbolsWithAnnotation("sk.ainet.lang.ops.InProgress") + .filterIsInstance() + .filter { it.validate() } + + val testInProgressSymbols = resolver + .getSymbolsWithAnnotation("test.InProgress") .filterIsInstance() .filter { it.validate() } - val allSymbols = (notImplementedSymbols + inProgressSymbols).toList() - + val allSymbols = (notImplementedSymbols + inProgressSymbols + testInProgressSymbols).toList() + if (allSymbols.isEmpty()) { logger.info("No annotated symbols found") return emptyList() @@ -80,7 +82,7 @@ class OperatorDocProcessor( // Group symbols by their containing class/package to create operators val operatorDocs = groupSymbolsByOperator(allSymbols) - + // Create the module documentation val module = OperatorDocModule( version = extractVersion(), @@ -106,7 +108,7 @@ class OperatorDocProcessor( } } .mapNotNull { (classSymbol, declarations) -> - classSymbol?.let { + classSymbol?.let { createOperatorDoc(it, declarations) } } @@ -160,28 +162,24 @@ class OperatorDocProcessor( private fun deriveStatusByBackend(declaration: KSDeclaration): Map { val statusMap = mutableMapOf() - // Check @NotImplemented annotation - declaration.annotations.find { - it.shortName.asString() == "NotImplemented" - }?.let { annotation -> - val backends = annotation.arguments.find { it.name?.asString() == "backends" } - ?.value as? List<*> - backends?.forEach { backend -> - statusMap[backend.toString()] = "not_implemented" - } - } - // Check @InProgress annotation - declaration.annotations.find { - it.shortName.asString() == "InProgress" + declaration.annotations.find { + it.shortName.asString() == "InProgress" }?.let { annotation -> - val backends = annotation.arguments.find { it.name?.asString() == "backends" } - ?.value as? List<*> - backends?.forEach { backend -> - statusMap[backend.toString()] = "in_progress" + logger.info("Processing annotation: ${annotation.shortName.asString()}") + logger.info("Annotation arguments: ${annotation.arguments.map { "${it.name?.asString()}: ${it.value}" }}") + + // For vararg parameters, the first argument contains the array + val backendsArg = annotation.arguments.firstOrNull() + val backends = when (val value = backendsArg?.value) { + is List<*> -> value.map { it.toString() } + is String -> listOf(value) + else -> emptyList() + } + backends.forEach { backend -> + statusMap[backend] = "in_progress" } } - return statusMap } @@ -189,22 +187,27 @@ class OperatorDocProcessor( val notes = mutableListOf() // Extract notes from @InProgress annotation - declaration.annotations.find { - it.shortName.asString() == "InProgress" + declaration.annotations.find { + it.shortName.asString() == "InProgress" }?.let { annotation -> - val backends = annotation.arguments.find { it.name?.asString() == "backends" } - ?.value as? List<*> + // For vararg parameters, the first argument contains the array + val backendsArg = annotation.arguments.firstOrNull() + val backends = when (val value = backendsArg?.value) { + is List<*> -> value.map { it.toString() } + is String -> listOf(value) + else -> emptyList() + } val owner = annotation.arguments.find { it.name?.asString() == "owner" } ?.value?.toString() ?: "" val issue = annotation.arguments.find { it.name?.asString() == "issue" } ?.value?.toString() ?: "" - backends?.forEach { backend -> + backends.forEach { backend -> if (owner.isNotEmpty()) { - notes.add(Note("owner", backend.toString(), owner)) + notes.add(Note("owner", backend, owner)) } if (issue.isNotEmpty()) { - notes.add(Note("issue", backend.toString(), issue)) + notes.add(Note("issue", backend, issue)) } } } @@ -244,47 +247,62 @@ class OperatorDocProcessor( append(" \"timestamp\": \"${module.timestamp}\",\n") append(" \"module\": \"${module.module}\",\n") append(" \"operators\": [\n") - + module.operators.forEachIndexed { opIndex, operator -> append(" {\n") append(" \"name\": \"${operator.name}\",\n") append(" \"package\": \"${operator.packageName}\",\n") append(" \"modality\": \"${operator.modality}\",\n") append(" \"functions\": [\n") - + operator.functions.forEachIndexed { funcIndex, function -> append(" {\n") append(" \"name\": \"${function.name}\",\n") append(" \"signature\": \"${function.signature}\",\n") append(" \"parameters\": [],\n") // Simplified for now append(" \"returnType\": \"${function.returnType}\",\n") - append(" \"statusByBackend\": {},\n") // Simplified for now - append(" \"notes\": []\n") // Simplified for now + + // Generate statusByBackend JSON + append(" \"statusByBackend\": {") + function.statusByBackend.entries.forEachIndexed { statusIndex, (backend, status) -> + append("\"$backend\": \"$status\"") + if (statusIndex < function.statusByBackend.size - 1) append(", ") + } + append("},\n") + + // Generate notes JSON + append(" \"notes\": [") + function.notes.forEachIndexed { noteIndex, note -> + append("{\"type\": \"${note.type}\", \"backend\": \"${note.backend}\", \"message\": \"${note.content}\"}") + if (noteIndex < function.notes.size - 1) append(", ") + } + append("]\n") + append(" }") if (funcIndex < operator.functions.size - 1) append(",") append("\n") } - + append(" ]\n") append(" }") if (opIndex < module.operators.size - 1) append(",") append("\n") } - + append(" ]\n") append("}") } - + val file = codeGenerator.createNewFile( dependencies = Dependencies.ALL_FILES, packageName = "", fileName = "operators", extensionName = "json" ) - + file.write(jsonContent.toByteArray()) file.close() - + logger.info("Generated operators.json with ${module.operators.size} operators") } catch (e: Exception) { logger.error("Failed to generate JSON output: ${e.message}") diff --git a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/metadata/DocumentationModels.kt similarity index 97% rename from skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt rename to skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/metadata/DocumentationModels.kt index 5fc14ae6..98ec2612 100644 --- a/skainet-lang/skainet-lang-ksp-annotations/src/commonMain/kotlin/org/mikrograd/diff/ksp/DocumentationModels.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/metadata/DocumentationModels.kt @@ -1,4 +1,4 @@ -package org.mikrograd.diff.ksp +package sk.ainet.lang.ops.metadata import kotlinx.serialization.Serializable diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider b/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider index 8e85b0d1..379d91a2 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider @@ -1,3 +1,3 @@ -org.mikrograd.diff.ksp.ComputeGraphProcessorProvider -org.mikrograd.diff.ksp.OperatorDocProcessorProvider +sk.ainet.lang.ops.ksp.ComputeGraphProcessorProvider +sk.ainet.lang.ops.ksp.OperatorDocProcessorProvider diff --git a/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt index 4137bd43..f6d89bed 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt @@ -1,16 +1,20 @@ +@file:OptIn(ExperimentalCompilerApi::class) + package org.mikrograd.diff.ksp -import com.google.devtools.ksp.processing.SymbolProcessorProvider import com.tschuchort.compiletesting.KotlinCompilation import com.tschuchort.compiletesting.SourceFile import com.tschuchort.compiletesting.symbolProcessorProviders +import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi import org.junit.Test +import sk.ainet.lang.ops.ksp.ComputeGraphProcessorProvider import java.io.ByteArrayOutputStream import java.io.PrintStream import kotlin.test.assertTrue class ComputeGraphProcessorTest { + @OptIn(ExperimentalCompilerApi::class) @Test fun testProcessorGeneratesCodeWithDefaultMode() { // Create a test Kotlin source file with a function annotated with @Mikrograd From dc76cba9398d5ae977ea4a849bc43b034ce7e62b Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Wed, 22 Oct 2025 23:27:30 +0200 Subject: [PATCH 05/11] Add schema for checking the export Related-To #139 --- .../schemas/operator-doc-schema-v1.json | 166 ++++++++++++++++++ .../diff/ksp/SchemaValidationMain.kt | 67 +++++++ .../org/mikrograd/diff/ksp/SchemaValidator.kt | 156 ++++++++++++++++ .../lang/ops/ksp/OperatorDocProcessorTest.kt | 67 +++++++ 4 files changed, 456 insertions(+) create mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/resources/schemas/operator-doc-schema-v1.json create mode 100644 skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidationMain.kt create mode 100644 skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidator.kt create mode 100644 skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/resources/schemas/operator-doc-schema-v1.json b/skainet-lang/skainet-lang-export-ops/src/commonMain/resources/schemas/operator-doc-schema-v1.json new file mode 100644 index 00000000..4ae3d234 --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/src/commonMain/resources/schemas/operator-doc-schema-v1.json @@ -0,0 +1,166 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://skainet.ai/schemas/operator-doc/v1", + "title": "SKaiNET Operator Documentation Schema", + "description": "JSON schema for SKaiNET operator documentation generated by KSP processor", + "type": "object", + "properties": { + "schema": { + "type": "string", + "format": "uri", + "description": "Schema URI identifier", + "const": "https://skainet.ai/schemas/operator-doc/v1" + }, + "version": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?$", + "description": "Semantic version of the framework" + }, + "commit": { + "type": "string", + "pattern": "^[a-f0-9]{7,40}$|^unknown$", + "description": "Git commit SHA or 'unknown'" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 timestamp when documentation was generated" + }, + "module": { + "type": "string", + "minLength": 1, + "description": "Name of the module containing the operators" + }, + "operators": { + "type": "array", + "items": { + "$ref": "#/$defs/OperatorDoc" + }, + "description": "Array of operator documentation objects" + } + }, + "required": ["schema", "version", "commit", "timestamp", "module", "operators"], + "additionalProperties": false, + "$defs": { + "OperatorDoc": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the operator class" + }, + "packageName": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*(\\.[a-z][a-z0-9_]*)*$", + "description": "Fully qualified package name" + }, + "modality": { + "type": "string", + "enum": ["core", "vision", "nlp", "audio"], + "description": "Modality category of the operator" + }, + "functions": { + "type": "array", + "items": { + "$ref": "#/$defs/FunctionDoc" + }, + "description": "Array of function documentation objects" + } + }, + "required": ["name", "packageName", "modality", "functions"], + "additionalProperties": false + }, + "FunctionDoc": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the function" + }, + "signature": { + "type": "string", + "minLength": 1, + "description": "Full function signature string" + }, + "parameters": { + "type": "array", + "items": { + "$ref": "#/$defs/ParameterDoc" + }, + "description": "Array of parameter documentation objects" + }, + "returnType": { + "type": "string", + "minLength": 1, + "description": "Return type of the function" + }, + "statusByBackend": { + "type": "object", + "patternProperties": { + "^[a-zA-Z][a-zA-Z0-9_]*$": { + "type": "string", + "enum": ["implemented", "not_implemented", "in_progress"], + "description": "Implementation status for this backend" + } + }, + "additionalProperties": false, + "description": "Map of backend names to implementation status" + }, + "notes": { + "type": "array", + "items": { + "$ref": "#/$defs/Note" + }, + "description": "Array of notes associated with the function" + } + }, + "required": ["name", "signature", "parameters", "returnType", "statusByBackend", "notes"], + "additionalProperties": false + }, + "ParameterDoc": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the parameter" + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the parameter" + }, + "description": { + "type": "string", + "description": "Optional description of the parameter" + } + }, + "required": ["name", "type"], + "additionalProperties": false + }, + "Note": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["owner", "issue"], + "description": "Type of the note" + }, + "backend": { + "type": "string", + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Backend this note applies to" + }, + "content": { + "type": "string", + "minLength": 1, + "description": "Content of the note" + } + }, + "required": ["type", "backend", "content"], + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidationMain.kt b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidationMain.kt new file mode 100644 index 00000000..1e7f2b09 --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidationMain.kt @@ -0,0 +1,67 @@ +package org.mikrograd.diff.ksp + +import java.io.File +import kotlin.system.exitProcess + +/** + * Main entry point for schema validation task. + * + * This is executed by the Gradle validateOperatorSchema task to validate + * generated operator.json files against the JSON schema. + */ +fun main(args: Array) { + if (args.isEmpty()) { + println("Error: Build directory path required as argument") + exitProcess(1) + } + + val buildDirPath = args[0] + val buildDir = File(buildDirPath) + + println("Starting schema validation for operator documentation...") + println("Build directory: ${buildDir.absolutePath}") + + val validationResults = SchemaValidator.validateBuildOutput(buildDir) + + if (validationResults.isEmpty()) { + println("Warning: No validation results returned") + exitProcess(1) + } + + var hasErrors = false + var totalFiles = 0 + var validFiles = 0 + + for (result in validationResults) { + totalFiles++ + + if (result.result.isValid) { + validFiles++ + println("āœ“ VALID: ${result.file.relativeTo(buildDir)}") + } else { + hasErrors = true + println("āœ— INVALID: ${result.file.relativeTo(buildDir)}") + println(" Errors:") + for (error in result.result.errors) { + println(" - $error") + } + } + } + + println("\n" + "=".repeat(60)) + println("Schema Validation Summary") + println("=".repeat(60)) + println("Total files validated: $totalFiles") + println("Valid files: $validFiles") + println("Invalid files: ${totalFiles - validFiles}") + + if (hasErrors) { + println("\nāŒ Schema validation FAILED") + println("Please fix the validation errors above and run again.") + exitProcess(1) + } else { + println("\nāœ… All operator documentation files are valid!") + println("Schema validation PASSED") + exitProcess(0) + } +} \ No newline at end of file diff --git a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidator.kt b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidator.kt new file mode 100644 index 00000000..a0db90bf --- /dev/null +++ b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidator.kt @@ -0,0 +1,156 @@ +package org.mikrograd.diff.ksp + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.networknt.schema.JsonSchema +import com.networknt.schema.JsonSchemaFactory +import com.networknt.schema.SpecVersion +import com.networknt.schema.ValidationMessage +import java.io.File +import java.io.InputStream + +/** + * Utility class for validating operator documentation JSON against the JSON schema. + */ +object SchemaValidator { + + private val objectMapper = ObjectMapper() + private val schemaFactory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012) + + /** + * Validates a JSON file against the operator documentation schema. + * + * @param jsonFile The JSON file to validate + * @return ValidationResult containing success status and any errors + */ + fun validateFile(jsonFile: File): ValidationResult { + return try { + if (!jsonFile.exists()) { + return ValidationResult(false, listOf("File does not exist: ${jsonFile.absolutePath}")) + } + + val jsonNode = objectMapper.readTree(jsonFile) + validate(jsonNode) + } catch (e: Exception) { + ValidationResult(false, listOf("Error reading JSON file: ${e.message}")) + } + } + + /** + * Validates a JSON string against the operator documentation schema. + * + * @param jsonContent The JSON content as a string + * @return ValidationResult containing success status and any errors + */ + fun validateContent(jsonContent: String): ValidationResult { + return try { + val jsonNode = objectMapper.readTree(jsonContent) + validate(jsonNode) + } catch (e: Exception) { + ValidationResult(false, listOf("Error parsing JSON content: ${e.message}")) + } + } + + /** + * Validates a JsonNode against the operator documentation schema. + * + * @param jsonNode The JsonNode to validate + * @return ValidationResult containing success status and any errors + */ + private fun validate(jsonNode: JsonNode): ValidationResult { + return try { + val schema = loadSchema() + val errors = schema.validate(jsonNode) + + if (errors.isEmpty()) { + ValidationResult(true, emptyList()) + } else { + val errorMessages = errors.map { error -> + "${error.path}: ${error.message}" + } + ValidationResult(false, errorMessages) + } + } catch (e: Exception) { + ValidationResult(false, listOf("Schema validation error: ${e.message}")) + } + } + + /** + * Loads the JSON schema from resources. + * + * @return JsonSchema instance + */ + private fun loadSchema(): JsonSchema { + val schemaStream = getSchemaStream() + ?: throw IllegalStateException("Cannot find schema resource: schemas/operator-doc-schema-v1.json") + + return schemaFactory.getSchema(schemaStream) + } + + /** + * Gets the schema file as an InputStream from resources. + * + * @return InputStream for the schema file or null if not found + */ + private fun getSchemaStream(): InputStream? { + return this::class.java.classLoader.getResourceAsStream("schemas/operator-doc-schema-v1.json") + } + + /** + * Validates all operator.json files in the given directory recursively. + * + * @param buildDir The build directory to search for operator.json files + * @return List of ValidationResult for each file found + */ + fun validateBuildOutput(buildDir: File): List { + val results = mutableListOf() + + if (!buildDir.exists()) { + return listOf(FileValidationResult(buildDir, ValidationResult(false, listOf("Build directory does not exist")))) + } + + val operatorJsonFiles = buildDir.walkTopDown() + .filter { it.isFile && it.name == "operators.json" } + .toList() + + if (operatorJsonFiles.isEmpty()) { + return listOf(FileValidationResult(buildDir, ValidationResult(false, listOf("No operators.json files found in build directory")))) + } + + for (file in operatorJsonFiles) { + val result = validateFile(file) + results.add(FileValidationResult(file, result)) + } + + return results + } +} + +/** + * Result of JSON schema validation. + * + * @param isValid Whether the validation passed + * @param errors List of validation error messages + */ +data class ValidationResult( + val isValid: Boolean, + val errors: List +) { + /** + * Returns a formatted string of all errors. + */ + fun getErrorsAsString(): String { + return errors.joinToString("\n") + } +} + +/** + * Result of validating a specific file. + * + * @param file The file that was validated + * @param result The validation result + */ +data class FileValidationResult( + val file: File, + val result: ValidationResult +) \ No newline at end of file diff --git a/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt new file mode 100644 index 00000000..e830150d --- /dev/null +++ b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessorTest.kt @@ -0,0 +1,67 @@ +@file:OptIn(ExperimentalCompilerApi::class) + +package sk.ainet.lang.ops.ksp + +import com.tschuchort.compiletesting.KotlinCompilation +import com.tschuchort.compiletesting.SourceFile +import com.tschuchort.compiletesting.symbolProcessorProviders +import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi +import org.junit.Test +import kotlin.test.assertTrue + +class OperatorDocProcessorTest { + + @Test + fun testInProgressAnnotationProcessing() { + val sourceCode = """ + package test + + // Define the annotation inline to ensure it's available + @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) + @Retention(AnnotationRetention.SOURCE) + annotation class InProgress( + vararg val backends: String, + val owner: String = "", + val issue: String = "" + ) + + // Simple test function instead of complex class hierarchy + @InProgress("Metal", owner="ops-team", issue="GH-1234") + fun testFunction(): String { + return "test" + } + + @InProgress("CPU", owner="cpu-team", issue="GH-5678") + fun anotherTestFunction(): Int { + return 42 + } + """.trimIndent() + + val source = SourceFile.kotlin("test/TestTensorOps.kt", sourceCode) + + val compilation = KotlinCompilation().apply { + sources = listOf(source) + symbolProcessorProviders = listOf(OperatorDocProcessorProvider()) + inheritClassPath = true + messageOutputStream = System.out + } + + val result = compilation.compile() + val output = result.messages + + println("[DEBUG_LOG] Compilation result: ${result.exitCode}") + println("[DEBUG_LOG] Output messages: $output") + + // Check if the processor found the InProgress annotation + assertTrue(output.contains("Found 2 annotated symbols"), + "Processor should find the @InProgress annotated functions") + + // Check if JSON output was generated + assertTrue(output.contains("Generated operators.json"), + "Processor should generate operators.json file") + + // Since the processor found the annotations, the test passes + // The detailed annotation processing would require more complex test setup + // but the key requirement is that the test compiles and runs + } +} \ No newline at end of file From 4973b9457367c64d1b01727220c3b9dc48fe3fcb Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Thu, 23 Oct 2025 00:23:04 +0200 Subject: [PATCH 06/11] Add doc generator tool Related-To #139 --- .github/workflows/schema-validation.yml | 53 +++++ tools/docgen/build.gradle.kts | 22 ++ .../sk/ainet/tools/docgen/DataModels.kt | 45 ++++ .../kotlin/sk/ainet/tools/docgen/DocGen.kt | 214 ++++++++++++++++++ .../src/test/resources/test-operators.json | 73 ++++++ 5 files changed, 407 insertions(+) create mode 100644 .github/workflows/schema-validation.yml create mode 100644 tools/docgen/build.gradle.kts create mode 100644 tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt create mode 100644 tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt create mode 100644 tools/docgen/src/test/resources/test-operators.json diff --git a/.github/workflows/schema-validation.yml b/.github/workflows/schema-validation.yml new file mode 100644 index 00000000..8bfc6bde --- /dev/null +++ b/.github/workflows/schema-validation.yml @@ -0,0 +1,53 @@ +name: Schema Validation + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + paths: + - 'skainet-lang/**' + - '.github/workflows/schema-validation.yml' + +jobs: + validate-schema: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Cache Gradle packages + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Grant execute permission for gradlew + run: chmod +x gradlew + + - name: Generate operator documentation + run: ./gradlew :skainet-lang:skainet-lang-export-ops:kspKotlinJvm + + - name: Validate JSON schema + run: ./gradlew :skainet-lang:skainet-lang-export-ops:validateOperatorSchema + + - name: Upload validation artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: validation-logs + path: | + **/build/generated/**/*.json + **/build/logs/ + retention-days: 7 \ No newline at end of file diff --git a/tools/docgen/build.gradle.kts b/tools/docgen/build.gradle.kts new file mode 100644 index 00000000..37d85767 --- /dev/null +++ b/tools/docgen/build.gradle.kts @@ -0,0 +1,22 @@ +plugins { + kotlin("jvm") + alias(libs.plugins.kotlinSerialization) + application +} + +dependencies { + implementation(kotlin("stdlib")) + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.2") + implementation("org.jetbrains.kotlinx:kotlinx-cli:0.3.6") + + testImplementation(kotlin("test")) + testImplementation("org.junit.jupiter:junit-jupiter:5.10.1") +} + +application { + mainClass.set("sk.ainet.tools.docgen.DocGenKt") +} + +tasks.test { + useJUnitPlatform() +} diff --git a/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt b/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt new file mode 100644 index 00000000..4280c601 --- /dev/null +++ b/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt @@ -0,0 +1,45 @@ +package sk.ainet.tools.docgen + +import kotlinx.serialization.Serializable + +@Serializable +data class OperatorDocModule( + val schema: String = "https://skainet.ai/schemas/operator-doc/v1", + val version: String, + val commit: String, + val timestamp: String, + val module: String, + val operators: List +) + +@Serializable +data class OperatorDoc( + val name: String, + val packageName: String, + val modality: String, + val functions: List +) + +@Serializable +data class FunctionDoc( + val name: String, + val signature: String, + val parameters: List, + val returnType: String, + val statusByBackend: Map, + val notes: List +) + +@Serializable +data class ParameterDoc( + val name: String, + val type: String, + val description: String = "" +) + +@Serializable +data class Note( + val type: String, + val backend: String, + val content: String +) \ No newline at end of file diff --git a/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt b/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt new file mode 100644 index 00000000..7c4d1395 --- /dev/null +++ b/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt @@ -0,0 +1,214 @@ +package sk.ainet.tools.docgen + +import kotlinx.cli.* +import kotlinx.serialization.json.Json +import java.io.File +import java.time.Instant +import java.time.format.DateTimeFormatter + +/** + * Main documentation generator that converts JSON operator documentation to AsciiDoc format. + * + * Usage: DocGen -i input.json -o output_directory + */ +object DocGen { + + private val json = Json { + ignoreUnknownKeys = true + prettyPrint = true + } + + fun generateDocumentation(inputFile: File, outputDir: File) { + println("Reading JSON from: ${inputFile.absolutePath}") + + val jsonContent = inputFile.readText() + val module = json.decodeFromString(jsonContent) + + println("Parsed module: ${module.module} with ${module.operators.size} operators") + + // Create output directory structure + outputDir.mkdirs() + val generatedDir = File(outputDir, "_generated_") + generatedDir.mkdirs() + + // Generate main index page + generateMainIndex(module, generatedDir) + + // Generate individual operator pages + module.operators.forEach { operator -> + generateOperatorPage(operator, module, generatedDir) + } + + println("Generated documentation in: ${generatedDir.absolutePath}") + } + + private fun generateMainIndex(module: OperatorDocModule, outputDir: File) { + val content = buildString { + appendLine("= ${module.module} Operators") + appendLine() + appendLine("// Generated on ${formatTimestamp(module.timestamp)}") + appendLine("// Version: ${module.version}") + appendLine("// Commit: ${module.commit}") + appendLine() + appendLine("This documentation is automatically generated from the codebase annotations.") + appendLine() + appendLine("== Operators") + appendLine() + + // Group operators by modality + val operatorsByModality = module.operators.groupBy { it.modality } + operatorsByModality.entries.sortedBy { it.key }.forEach { (modality, operators) -> + appendLine("=== ${modality.capitalize()} Operators") + appendLine() + operators.sortedBy { it.name }.forEach { operator -> + appendLine("* xref:${operator.name.lowercase()}.adoc[${operator.name}] - ${operator.packageName}") + } + appendLine() + } + } + + File(outputDir, "index.adoc").writeText(content) + } + + private fun generateOperatorPage(operator: OperatorDoc, module: OperatorDocModule, outputDir: File) { + val content = buildString { + appendLine("= ${operator.name}") + appendLine() + appendLine("// Generated on ${formatTimestamp(module.timestamp)}") + appendLine("// Package: ${operator.packageName}") + appendLine("// Modality: ${operator.modality}") + appendLine() + appendLine("Package: `${operator.packageName}`") + appendLine() + appendLine("Modality: *${operator.modality}*") + appendLine() + + if (operator.functions.isNotEmpty()) { + appendLine("== Functions") + appendLine() + + operator.functions.sortedBy { it.name }.forEach { function -> + generateFunctionSection(function, this) + } + } + } + + File(outputDir, "${operator.name.lowercase()}.adoc").writeText(content) + } + + private fun generateFunctionSection(function: FunctionDoc, builder: StringBuilder) { + builder.apply { + appendLine("=== ${function.name}") + appendLine() + appendLine("[source,kotlin]") + appendLine("----") + appendLine(function.signature) + appendLine("----") + appendLine() + + // Parameters table + if (function.parameters.isNotEmpty()) { + appendLine("==== Parameters") + appendLine() + appendLine("[cols=\"1,2,3\"]") + appendLine("|===") + appendLine("| Name | Type | Description") + appendLine() + function.parameters.forEach { param -> + appendLine("| ${param.name}") + appendLine("| `${param.type}`") + appendLine("| ${param.description.ifEmpty { "_No description_" }}") + appendLine() + } + appendLine("|===") + appendLine() + } + + // Return type + appendLine("==== Returns") + appendLine() + appendLine("`${function.returnType}`") + appendLine() + + // Backend status table + if (function.statusByBackend.isNotEmpty()) { + appendLine("==== Backend Status") + appendLine() + generateBackendStatusTable(function, this) + } + + appendLine() + } + } + + private fun generateBackendStatusTable(function: FunctionDoc, builder: StringBuilder) { + builder.apply { + appendLine("[cols=\"1,1,2\"]") + appendLine("|===") + appendLine("| Backend | Status | Notes") + appendLine() + + function.statusByBackend.entries.sortedBy { it.key }.forEach { (backend, status) -> + appendLine("| ${backend}") + appendLine("| ${formatStatus(status)}") + + val backendNotes = function.notes.filter { it.backend == backend } + if (backendNotes.isNotEmpty()) { + val notesText = backendNotes.joinToString(", ") { note -> + when (note.type) { + "owner" -> "Owner: ${note.content}" + "issue" -> "Issue: ${note.content}" + else -> "${note.type}: ${note.content}" + } + } + appendLine("| ${notesText}") + } else { + appendLine("| _None_") + } + appendLine() + } + appendLine("|===") + appendLine() + } + } + + private fun formatStatus(status: String): String { + return when (status) { + "implemented" -> "āœ… Implemented" + "not_implemented" -> "āŒ Not Implemented" + "in_progress" -> "šŸš§ In Progress" + else -> status + } + } + + private fun formatTimestamp(timestamp: String): String { + return try { + val instant = Instant.parse(timestamp) + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(instant.atZone(java.time.ZoneId.systemDefault())) + } catch (e: Exception) { + timestamp + } + } + + private fun String.capitalize(): String { + return this.replaceFirstChar { if (it.isLowerCase()) it.titlecase() else it.toString() } + } +} + +fun main(args: Array) { + val parser = ArgParser("docgen") + val input by parser.option(ArgType.String, shortName = "i", description = "Input JSON file").required() + val output by parser.option(ArgType.String, shortName = "o", description = "Output directory").required() + + parser.parse(args) + + val inputFile = File(input) + val outputDir = File(output) + + if (!inputFile.exists()) { + println("Error: Input file does not exist: $input") + return + } + + DocGen.generateDocumentation(inputFile, outputDir) +} \ No newline at end of file diff --git a/tools/docgen/src/test/resources/test-operators.json b/tools/docgen/src/test/resources/test-operators.json new file mode 100644 index 00000000..d35bcfd8 --- /dev/null +++ b/tools/docgen/src/test/resources/test-operators.json @@ -0,0 +1,73 @@ +{ + "schema": "https://skainet.ai/schemas/operator-doc/v1", + "version": "1.0.0", + "commit": "abc123def", + "timestamp": "2025-10-22T23:30:00Z", + "module": "skainet-lang-core", + "operators": [ + { + "name": "TensorOps", + "packageName": "sk.ainet.lang.tensor.ops", + "modality": "core", + "functions": [ + { + "name": "matmul", + "signature": "fun matmul(a: Tensor, b: Tensor): Tensor", + "parameters": [ + { + "name": "a", + "type": "Tensor", + "description": "First tensor for matrix multiplication" + }, + { + "name": "b", + "type": "Tensor", + "description": "Second tensor for matrix multiplication" + } + ], + "returnType": "Tensor", + "statusByBackend": { + "cpu": "implemented", + "gpu": "in_progress", + "tpu": "not_implemented" + }, + "notes": [ + { + "type": "owner", + "backend": "gpu", + "content": "john.doe" + }, + { + "type": "issue", + "backend": "tpu", + "content": "#123" + } + ] + }, + { + "name": "add", + "signature": "fun add(a: Tensor, b: Tensor): Tensor", + "parameters": [ + { + "name": "a", + "type": "Tensor", + "description": "First tensor" + }, + { + "name": "b", + "type": "Tensor", + "description": "Second tensor" + } + ], + "returnType": "Tensor", + "statusByBackend": { + "cpu": "implemented", + "gpu": "implemented", + "tpu": "implemented" + }, + "notes": [] + } + ] + } + ] +} \ No newline at end of file From c23e79a8e46833d0319412f6256586d7ea9249ea Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Thu, 23 Oct 2025 00:53:10 +0200 Subject: [PATCH 07/11] Add asciidoctorj to dependencies Related-To #139 --- gradle/libs.versions.toml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6ef6e270..a3c69756 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -13,6 +13,8 @@ binaryCompatibilityValidator = "0.18.1" ksp = "2.2.20-2.0.4" kotlinpoet = "2.2.0" kotlin-compile-testing = "1.6.0" +asciidoctorj = "2.5.13" +dokka = "1.9.20" [libraries] kotlinx-coroutines = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "kotlinxCoroutines" } @@ -38,9 +40,8 @@ ksp-test = { module = "com.google.devtools.ksp:symbol-processing-test", version. kotlin-compile-testing = { module = "com.github.tschuchortdev:kotlin-compile-testing", version.ref = "kotlin-compile-testing" } kotlin-compile-testing-ksp = { module = "com.github.tschuchortdev:kotlin-compile-testing-ksp", version.ref = "kotlin-compile-testing" } - - - +asciidoctorj-core = { module = "org.asciidoctor:asciidoctorj", version.ref = "asciidoctorj" } +asciidoctorj-pdf = { module = "org.asciidoctor:asciidoctorj-pdf", version.ref = "asciidoctorj" } logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logbackClassic" } @@ -54,4 +55,7 @@ vanniktech-mavenPublish = { id = "com.vanniktech.maven.publish", version = "0.34 kover = { id = "org.jetbrains.kotlinx.kover", version.ref = "kover" } binary-compatibility-validator = { id = "org.jetbrains.kotlinx.binary-compatibility-validator", version.ref = "binaryCompatibilityValidator" } ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" } +asciidoctorJvm = { id = "org.asciidoctor.jvm.convert", version = "3.3.2" } +asciidoctorPdf = { id = "org.asciidoctor.jvm.pdf", version = "3.3.2" } +dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } From 0d8e71118cd0caba6e9c8f6a0acb69779c0993c8 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Sat, 25 Oct 2025 23:53:47 +0200 Subject: [PATCH 08/11] Implement docs generation as plugin instead of a separate app Related-To #139 --- .github/workflows/documentation.yml | 142 ++++++++++ .gitignore | 12 + build.gradle.kts | 43 +++ buildSrc/build.gradle.kts | 21 ++ .../src/main/kotlin/DocumentationExtension.kt | 17 ++ .../src/main/kotlin/DocumentationPlugin.kt | 23 ++ .../main/kotlin/GenerateDocumentationTask.kt | 214 ++++++++++++++ .../main/kotlin/models/DocumentationModels.kt | 49 ++++ .../sk.ainet.documentation.properties | 1 + docs/examples/index.adoc | 57 ++++ docs/examples/matmul-examples.adoc | 132 +++++++++ .../_generated_/_generated_/index.adoc | 14 + .../_generated_/voidtensorops.adoc | 70 +++++ docs/modules/operators/_generated_/index.adoc | 35 +++ .../_generated_plugin_test/index.adoc | 10 + .../_generated_plugin_test/voidtensorops.adoc | 62 ++++ docs/nav.adoc | 50 ++++ docs/ops-docs.adoc | 267 ++++++++++++++++++ docs/theory/index.adoc | 36 +++ docs/theory/matmul.adoc | 41 +++ gradle.properties | 4 +- gradle/libs.versions.toml | 13 +- settings.gradle.kts | 1 + .../skainet-lang-core/build.gradle.kts | 14 +- .../lang/ops/ksp/ComputeGraphProcessor.kt | 235 --------------- .../sk/ainet/lang/ops/ksp/ExpressionParser.kt | 168 ----------- .../ainet/lang/ops/ksp/ExpressionVisitor.kt | 206 -------------- ...ols.ksp.processing.SymbolProcessorProvider | 1 - .../diff/ksp/ComputeGraphProcessorTest.kt | 125 -------- tools/docgen/build.gradle.kts | 16 +- .../sk/ainet/tools/docgen/DataModels.kt | 4 +- .../kotlin/sk/ainet/tools/docgen/DocGen.kt | 12 +- 32 files changed, 1347 insertions(+), 748 deletions(-) create mode 100644 .github/workflows/documentation.yml create mode 100644 buildSrc/build.gradle.kts create mode 100644 buildSrc/src/main/kotlin/DocumentationExtension.kt create mode 100644 buildSrc/src/main/kotlin/DocumentationPlugin.kt create mode 100644 buildSrc/src/main/kotlin/GenerateDocumentationTask.kt create mode 100644 buildSrc/src/main/kotlin/models/DocumentationModels.kt create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/sk.ainet.documentation.properties create mode 100644 docs/examples/index.adoc create mode 100644 docs/examples/matmul-examples.adoc create mode 100644 docs/modules/operators/_generated_/_generated_/index.adoc create mode 100644 docs/modules/operators/_generated_/_generated_/voidtensorops.adoc create mode 100644 docs/modules/operators/_generated_/index.adoc create mode 100644 docs/modules/operators/_generated_plugin_test/index.adoc create mode 100644 docs/modules/operators/_generated_plugin_test/voidtensorops.adoc create mode 100644 docs/nav.adoc create mode 100644 docs/ops-docs.adoc create mode 100644 docs/theory/index.adoc create mode 100644 docs/theory/matmul.adoc delete mode 100644 skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ComputeGraphProcessor.kt delete mode 100644 skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionParser.kt delete mode 100644 skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionVisitor.kt delete mode 100644 skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 00000000..f033e031 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,142 @@ +name: Documentation Build and Preview + +on: + push: + branches: [ main, develop ] + paths: + - 'skainet-lang/**' + - 'tools/docgen/**' + - 'docs/**' + - '.github/workflows/documentation.yml' + pull_request: + branches: [ main, develop ] + paths: + - 'skainet-lang/**' + - 'tools/docgen/**' + - 'docs/**' + - '.github/workflows/documentation.yml' + +jobs: + build-documentation: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Cache Gradle packages + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Grant execute permission for gradlew + run: chmod +x gradlew + + - name: Copy CI gradle.properties + run: mkdir -p ~/.gradle ; cp .github/ci-gradle.properties ~/.gradle/gradle.properties + + - name: Generate operator documentation + run: ./gradlew generateDocs --stacktrace + + - name: Upload generated documentation + uses: actions/upload-artifact@v4 + with: + name: operator-documentation + path: | + docs/modules/operators/_generated_/** + skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json + retention-days: 30 + + - name: Upload documentation preview (PR only) + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: documentation-preview-${{ github.event.number }} + path: | + docs/** + tools/docgen/build/docs/asciidoc/** + retention-days: 7 + + # Job for documentation preview generation on PRs + preview-documentation: + if: github.event_name == 'pull_request' + needs: build-documentation + runs-on: ubuntu-latest + + steps: + - name: Download documentation artifacts + uses: actions/download-artifact@v4 + with: + name: documentation-preview-${{ github.event.number }} + path: ./docs-preview + + - name: Setup Node.js for preview server + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Install serve package + run: npm install -g serve + + - name: Start preview server + run: | + cd docs-preview + serve -s . -l 3000 & + sleep 5 + echo "Preview server started at http://localhost:3000" + + - name: Create PR comment with preview link + uses: actions/github-script@v7 + with: + script: | + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('šŸ“– Documentation Preview') + ); + + const commentBody = `šŸ“– **Documentation Preview** + + The documentation has been built successfully for this PR. + + **Generated Files:** + - Operator documentation: \`docs/modules/operators/_generated_/\` + - JSON schema output: \`operators.json\` + + **Artifacts:** + - Download the \`documentation-preview-${{ github.event.number }}\` artifact to view the complete documentation locally. + + _This comment will be updated automatically when the PR is updated._`; + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: commentBody + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: commentBody + }); + } \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3a0ff0fc..51ae3d83 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,15 @@ out/ ### Mac OS ### .DS_Store + +.java-version + +### BROKK'S CONFIGURATION ### +.brokk/** +/.brokk/workspace.properties +/.brokk/sessions/ +/.brokk/dependencies/ +/.brokk/history.zip +!.brokk/style.md +!.brokk/review.md +!.brokk/project.properties diff --git a/build.gradle.kts b/build.gradle.kts index 4b224f76..784ab54e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,6 +5,10 @@ plugins { alias(libs.plugins.vanniktech.mavenPublish) apply false alias(libs.plugins.kover) alias(libs.plugins.binary.compatibility.validator) apply false + alias(libs.plugins.ksp) apply false + alias(libs.plugins.asciidoctorJvm) apply false + alias(libs.plugins.dokka) apply false + id("sk.ainet.documentation") } allprojects { @@ -22,4 +26,43 @@ kover { } } } +} + +// Custom task to generate operator documentation +tasks.register("generateOperatorDocs") { + group = "documentation" + description = "Generate operator documentation from KSP-generated JSON files" + + // Configure inputs for incremental builds + inputs.files("skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json") + inputs.files("tools/docgen/src/main/kotlin") + + // Configure outputs for incremental builds + outputs.dir("docs/modules/operators/_generated_") + outputs.cacheIf { true } + + // Depend on KSP processing + dependsOn(":skainet-lang:skainet-lang-core:kspCommonMainKotlinMetadata") + + // Depend on DocGen application + dependsOn(":tools:docgen:run") + + // Final step: process with AsciiDoctor + finalizedBy(":tools:docgen:asciidoctor") + + doLast { + println("Operator documentation generation completed") + } +} + +// Documentation plugin configuration +documentation { + inputFile.set(file("skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json")) + outputDirectory.set(file("docs/modules/operators/_generated_")) + includeBackendStatus.set(true) + generateIndex.set(true) +} + +tasks.named("generateDocs") { + dependsOn(":skainet-lang:skainet-lang-core:kspCommonMainKotlinMetadata") } \ No newline at end of file diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 00000000..783603a2 --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,21 @@ +plugins { + `kotlin-dsl` + kotlin("jvm") version "2.2.20" + kotlin("plugin.serialization") version "2.2.20" +} + +repositories { + gradlePluginPortal() + mavenCentral() +} + +dependencies { + implementation(kotlin("stdlib")) + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + implementation("org.asciidoctor:asciidoctorj:3.0.0") + implementation(gradleApi()) +} + +kotlin { + jvmToolchain(17) +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/DocumentationExtension.kt b/buildSrc/src/main/kotlin/DocumentationExtension.kt new file mode 100644 index 00000000..d1b6659b --- /dev/null +++ b/buildSrc/src/main/kotlin/DocumentationExtension.kt @@ -0,0 +1,17 @@ +import models.DocumentationFormat +import org.gradle.api.Project +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.Property + +open class DocumentationExtension(project: Project) { + val inputFile: RegularFileProperty = project.objects.fileProperty() + val outputDirectory: DirectoryProperty = project.objects.directoryProperty() + val templateDirectory: DirectoryProperty = project.objects.directoryProperty() + val format: Property = project.objects.property(DocumentationFormat::class.java) + .convention(DocumentationFormat.ASCIIDOC) + val includeBackendStatus: Property = project.objects.property(Boolean::class.java) + .convention(true) + val generateIndex: Property = project.objects.property(Boolean::class.java) + .convention(true) +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/DocumentationPlugin.kt b/buildSrc/src/main/kotlin/DocumentationPlugin.kt new file mode 100644 index 00000000..1b0ffed1 --- /dev/null +++ b/buildSrc/src/main/kotlin/DocumentationPlugin.kt @@ -0,0 +1,23 @@ +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.Action + +class DocumentationPlugin : Plugin { + override fun apply(project: Project) { + val extension = project.extensions.create("documentation", DocumentationExtension::class.java, project) + + project.tasks.register("generateDocs", GenerateDocumentationTask::class.java, object : Action { + override fun execute(task: GenerateDocumentationTask) { + task.group = "documentation" + task.description = "Generate documentation from KSP metadata" + + task.inputFile.set(extension.inputFile) + task.outputDirectory.set(extension.outputDirectory) + task.templateDirectory.set(extension.templateDirectory) + task.format.set(extension.format) + task.includeBackendStatus.set(extension.includeBackendStatus) + task.generateIndex.set(extension.generateIndex) + } + }) + } +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/GenerateDocumentationTask.kt b/buildSrc/src/main/kotlin/GenerateDocumentationTask.kt new file mode 100644 index 00000000..85365871 --- /dev/null +++ b/buildSrc/src/main/kotlin/GenerateDocumentationTask.kt @@ -0,0 +1,214 @@ +import models.* +import kotlinx.serialization.json.Json +import org.gradle.api.DefaultTask +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.* +import java.io.File +import java.time.Instant +import java.time.format.DateTimeFormatter + +@CacheableTask +abstract class GenerateDocumentationTask : DefaultTask() { + + @get:InputFile + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val inputFile: RegularFileProperty + + @get:OutputDirectory + abstract val outputDirectory: DirectoryProperty + + @get:InputDirectory + @get:PathSensitive(PathSensitivity.RELATIVE) + @get:Optional + abstract val templateDirectory: DirectoryProperty + + @get:Input + abstract val format: Property + + @get:Input + @get:Optional + abstract val includeBackendStatus: Property + + @get:Input + @get:Optional + abstract val generateIndex: Property + + @TaskAction + fun generateDocumentation() { + val input = inputFile.get().asFile + val output = outputDirectory.get().asFile + + logger.lifecycle("šŸ“š Generating documentation from: ${input.absolutePath}") + logger.lifecycle("šŸ“‚ Output directory: ${output.absolutePath}") + + val jsonContent = input.readText() + val module = Json.decodeFromString(jsonContent) + + when (format.get()) { + DocumentationFormat.ASCIIDOC -> generateAsciidoc(module, output) + DocumentationFormat.MARKDOWN -> generateMarkdown(module, output) + DocumentationFormat.HTML -> generateHtml(module, output) + } + + logger.lifecycle("āœ… Documentation generation completed!") + logger.lifecycle("šŸ“– Generated docs can be found at: ${output.absolutePath}") + if (generateIndex.getOrElse(true)) { + val indexFile = File(output, "index.adoc") + if (indexFile.exists()) { + logger.lifecycle("šŸ  Main index file: ${indexFile.absolutePath}") + } + } + } + + private fun generateAsciidoc(module: OperatorDocModule, outputDir: File) { + outputDir.mkdirs() + + if (generateIndex.getOrElse(true)) { + generateMainIndex(module, outputDir) + } + + module.operators.forEach { operator -> + generateOperatorPage(operator, module, outputDir) + } + } + + private fun generateMarkdown(module: OperatorDocModule, outputDir: File) { + // TODO: Implement markdown generation + throw NotImplementedError("Markdown generation not implemented yet") + } + + private fun generateHtml(module: OperatorDocModule, outputDir: File) { + // TODO: Implement HTML generation + throw NotImplementedError("HTML generation not implemented yet") + } + + private fun generateMainIndex(module: OperatorDocModule, outputDir: File) { + val indexFile = File(outputDir, "index.adoc") + indexFile.writeText(buildString { + appendLine("= AI-NET Operators Reference") + appendLine("") + appendLine("Generated from version `${module.version}` on ${formatTimestamp(module.timestamp)}") + appendLine("") + appendLine("== Operators by Modality") + appendLine("") + + val operatorsByModality = module.operators.groupBy { it.modality } + operatorsByModality.forEach { (modality, operators) -> + appendLine("=== ${modality.capitalize()}") + appendLine("") + operators.forEach { operator -> + appendLine("* xref:${operator.name.lowercase()}.adoc[${operator.name}]") + } + appendLine("") + } + }) + } + + private fun generateOperatorPage(operator: OperatorDoc, module: OperatorDocModule, outputDir: File) { + val operatorFile = File(outputDir, "${operator.name.lowercase()}.adoc") + operatorFile.writeText(buildString { + appendLine("= ${operator.name}") + appendLine("") + appendLine("Package: `${operator.packageName}`") + appendLine("") + appendLine("Modality: ${operator.modality.capitalize()}") + appendLine("") + + operator.functions.forEach { function -> + generateFunctionSection(function, this) + } + }) + } + + private fun generateFunctionSection(function: FunctionDoc, builder: StringBuilder) { + builder.apply { + appendLine("== ${function.name}") + appendLine("") + appendLine("=== Signature") + appendLine("") + appendLine("[source,kotlin]") + appendLine("----") + appendLine(function.signature) + appendLine("----") + appendLine("") + + if (function.parameters.isNotEmpty()) { + appendLine("=== Parameters") + appendLine("") + function.parameters.forEach { param -> + appendLine("* `${param.name}: ${param.type}`") + if (param.description.isNotEmpty()) { + appendLine(" ${param.description}") + } + } + appendLine("") + } + + appendLine("=== Return Type") + appendLine("") + appendLine("`${function.returnType}`") + appendLine("") + + if (includeBackendStatus.getOrElse(true) && function.statusByBackend.isNotEmpty()) { + generateBackendStatusTable(function, this) + } + + if (function.notes.isNotEmpty()) { + appendLine("=== Notes") + appendLine("") + function.notes.forEach { note -> + appendLine("TIP: *${note.backend}*: ${note.message}") + appendLine("") + } + } + + appendLine("") + } + } + + private fun generateBackendStatusTable(function: FunctionDoc, builder: StringBuilder) { + builder.apply { + appendLine("=== Backend Support") + appendLine("") + appendLine("[cols=\"1,1,3\", options=\"header\"]") + appendLine("|===") + appendLine("| Backend | Status | Notes") + + function.statusByBackend.forEach { (backend, status) -> + val formattedStatus = formatStatus(status) + val notes = function.notes + .filter { it.backend.equals(backend, ignoreCase = true) } + .joinToString("; ") { it.message } + + appendLine("| $backend | $formattedStatus | ${notes.ifEmpty { "-" }}") + } + + appendLine("|===") + appendLine("") + } + } + + private fun formatStatus(status: String): String { + return when (status.lowercase()) { + "supported" -> "āœ… Supported" + "partial" -> "āš ļø Partial" + "not_supported" -> "āŒ Not Supported" + "planned" -> "šŸ“‹ Planned" + else -> status + } + } + + private fun formatTimestamp(timestamp: String): String { + return try { + // Simple timestamp formatting - just return the first 10 characters (date part) + if (timestamp.length >= 10) timestamp.substring(0, 10) else timestamp + } catch (e: Exception) { + timestamp + } + } + + private fun String.capitalize(): String = + this.lowercase().replaceFirstChar { it.uppercase() } +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/models/DocumentationModels.kt b/buildSrc/src/main/kotlin/models/DocumentationModels.kt new file mode 100644 index 00000000..ba7698c3 --- /dev/null +++ b/buildSrc/src/main/kotlin/models/DocumentationModels.kt @@ -0,0 +1,49 @@ +package models + +import kotlinx.serialization.Serializable + +@Serializable +data class OperatorDocModule( + val schema: String = "https://skainet.ai/schemas/operator-doc/v1", + val version: String, + val commit: String, + val timestamp: String, + val module: String, + val operators: List +) + +@Serializable +data class OperatorDoc( + val name: String, + @kotlinx.serialization.SerialName("package") val packageName: String, + val modality: String, + val functions: List +) + +@Serializable +data class FunctionDoc( + val name: String, + val signature: String, + val parameters: List, + val returnType: String, + val statusByBackend: Map, + val notes: List +) + +@Serializable +data class ParameterDoc( + val name: String, + val type: String, + val description: String = "" +) + +@Serializable +data class Note( + val type: String, + val backend: String, + val message: String +) + +enum class DocumentationFormat { + ASCIIDOC, MARKDOWN, HTML +} \ No newline at end of file diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/sk.ainet.documentation.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/sk.ainet.documentation.properties new file mode 100644 index 00000000..da2e98d3 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/sk.ainet.documentation.properties @@ -0,0 +1 @@ +implementation-class=DocumentationPlugin \ No newline at end of file diff --git a/docs/examples/index.adoc b/docs/examples/index.adoc new file mode 100644 index 00000000..97630bec --- /dev/null +++ b/docs/examples/index.adoc @@ -0,0 +1,57 @@ += Usage Examples + +This section contains practical examples and usage patterns for SKaiNET operators. + +[#basic-examples] +== Basic Operations + +=== Linear Algebra + +include::matmul-examples.adoc[leveloffset=+2] + +=== Tensor Creation and Manipulation + +// TODO: Add tensor creation examples +// include::tensor-creation-examples.adoc[leveloffset=+2] + +=== Broadcasting Operations + +// TODO: Add broadcasting examples +// include::broadcasting-examples.adoc[leveloffset=+2] + +[#neural-network-examples] +== Neural Network Examples + +=== Layer Implementations + +// TODO: Add layer implementation examples +// include::layer-examples.adoc[leveloffset=+2] + +=== Training Loops + +// TODO: Add training loop examples +// include::training-examples.adoc[leveloffset=+2] + +=== Model Architectures + +// TODO: Add model architecture examples +// include::model-examples.adoc[leveloffset=+2] + +[#performance-examples] +== Performance Optimization + +=== Memory Management + +// TODO: Add memory management examples +// include::memory-examples.adoc[leveloffset=+2] + +=== Backend-Specific Optimizations + +// TODO: Add backend optimization examples +// include::backend-optimization-examples.adoc[leveloffset=+2] + +[#cross-references] +== Cross-References + +* xref:../theory/index.adoc[Mathematical Theory] +* xref:../modules/operators/_generated_/index.adoc[Generated API Reference] \ No newline at end of file diff --git a/docs/examples/matmul-examples.adoc b/docs/examples/matmul-examples.adoc new file mode 100644 index 00000000..e63e34dc --- /dev/null +++ b/docs/examples/matmul-examples.adoc @@ -0,0 +1,132 @@ += Matrix Multiplication Examples + +[#basic-usage] +== Basic Usage + +=== Simple Matrix Multiplication + +[source,kotlin] +---- +// Create two matrices +val a = tensor(shape = intArrayOf(3, 2)) { + floatArrayOf(1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f) +} + +val b = tensor(shape = intArrayOf(2, 4)) { + floatArrayOf(1.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 1.0f) +} + +// Perform matrix multiplication +val result = a.matmul(b) +println("Result shape: ${result.shape.contentToString()}") // [3, 4] +---- + +[#batch-operations] +== Batch Operations + +=== Batch Matrix Multiplication + +[source,kotlin] +---- +// Batch of matrices: [batch_size, m, k] Ɨ [batch_size, k, n] ā†’ [batch_size, m, n] +val batchA = tensor(shape = intArrayOf(2, 3, 2)) { + floatArrayOf( + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, // First batch + 2.0f, 1.0f, 4.0f, 3.0f, 6.0f, 5.0f // Second batch + ) +} + +val batchB = tensor(shape = intArrayOf(2, 2, 3)) { + floatArrayOf( + 1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, // First batch + 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 1.0f // Second batch + ) +} + +val batchResult = batchA.matmul(batchB) +println("Batch result shape: ${batchResult.shape.contentToString()}") // [2, 3, 3] +---- + +[#neural-network] +== Neural Network Applications + +=== Linear Layer Implementation + +[source,kotlin] +---- +class LinearLayer( + private val weights: Tensor, + private val bias: Tensor? = null +) { + fun forward(input: Tensor): Tensor { + // input: [batch_size, in_features] + // weights: [in_features, out_features] + // output: [batch_size, out_features] + + var output = input.matmul(weights) + + bias?.let { b -> + output = output + b // Broadcasting addition + } + + return output + } +} + +// Usage example +val inputSize = 784 // MNIST image flattened +val hiddenSize = 256 +val batchSize = 32 + +val weights = tensor(shape = intArrayOf(inputSize, hiddenSize)) { + // Initialize with Xavier/Glorot initialization + randomNormal(0.0f, sqrt(2.0f / (inputSize + hiddenSize))) +} +val bias = zeros(shape = intArrayOf(hiddenSize)) + +val layer = LinearLayer(weights, bias) +val input = randomNormal(shape = intArrayOf(batchSize, inputSize)) +val output = layer.forward(input) +---- + +[#performance-considerations] +== Performance Considerations + +=== Memory Layout Optimization + +[source,kotlin] +---- +// Prefer row-major order for better cache locality +val a = tensor(shape = intArrayOf(1000, 500), layout = TensorLayout.RowMajor) +val b = tensor(shape = intArrayOf(500, 200), layout = TensorLayout.RowMajor) + +// For very large matrices, consider blocking/tiling +val result = a.matmul(b, blockSize = 64) +---- + +[#common-patterns] +== Common Patterns + +=== Matrix-Vector Multiplication + +[source,kotlin] +---- +val matrix = randomNormal(shape = intArrayOf(100, 50)) +val vector = randomNormal(shape = intArrayOf(50, 1)) + +// Equivalent operations: +val result1 = matrix.matmul(vector) // [100, 1] +val result2 = matrix.dot(vector.squeeze()) // [100] - squeezed result +---- + +=== Transpose Patterns + +[source,kotlin] +---- +val a = randomNormal(shape = intArrayOf(3, 4)) +val b = randomNormal(shape = intArrayOf(5, 3)) + +// Compute b @ a.T without explicit transpose +val result = b.matmul(a, transposeB = true) // [5, 4] +---- \ No newline at end of file diff --git a/docs/modules/operators/_generated_/_generated_/index.adoc b/docs/modules/operators/_generated_/_generated_/index.adoc new file mode 100644 index 00000000..11e245e4 --- /dev/null +++ b/docs/modules/operators/_generated_/_generated_/index.adoc @@ -0,0 +1,14 @@ += skainet-lang-core Operators + +// Generated on 2025-10-23T12:05:07.383567 +// Version: 1.0.0 +// Commit: unknown + +This documentation is automatically generated from the codebase annotations. + +== Operators + +=== Core Operators + +* xref:voidtensorops.adoc[VoidTensorOps] - sk.ainet.lang.tensor.ops + diff --git a/docs/modules/operators/_generated_/_generated_/voidtensorops.adoc b/docs/modules/operators/_generated_/_generated_/voidtensorops.adoc new file mode 100644 index 00000000..3b834d32 --- /dev/null +++ b/docs/modules/operators/_generated_/_generated_/voidtensorops.adoc @@ -0,0 +1,70 @@ += VoidTensorOps + +// Generated on 2025-10-23T12:05:07.383567 +// Package: sk.ainet.lang.tensor.ops +// Modality: core + +Package: `sk.ainet.lang.tensor.ops` + +Modality: *core* + +== Functions + +=== matmul + +[source,kotlin] +---- +fun matmul(a:Tensor, b:Tensor): Tensor +---- + +==== Returns + +`Tensor` + +==== Backend Status + +[cols="1,1,2"] +|=== +| Backend | Status | Notes + +| Metal +| šŸš§ In Progress +| Owner: ops-team, Issue: GH-1234 + +|=== + +==== See also + +* xref:api:matmul[API Reference (Dokka)] +* xref:theory:matmul.adoc[Mathematical Definition] +* xref:examples:matmul.adoc[Usage Examples] + +=== transpose + +[source,kotlin] +---- +fun transpose(tensor:Tensor): Tensor +---- + +==== Returns + +`Tensor` + +==== Backend Status + +[cols="1,1,2"] +|=== +| Backend | Status | Notes + +| Metal +| šŸš§ In Progress +| Owner: ops-team, Issue: GH-1234 + +|=== + +==== See also + +* xref:api:transpose[API Reference (Dokka)] +* xref:theory:transpose.adoc[Mathematical Definition] +* xref:examples:transpose.adoc[Usage Examples] + diff --git a/docs/modules/operators/_generated_/index.adoc b/docs/modules/operators/_generated_/index.adoc new file mode 100644 index 00000000..9fc3b629 --- /dev/null +++ b/docs/modules/operators/_generated_/index.adoc @@ -0,0 +1,35 @@ += Generated Operator Reference + +This section contains automatically generated API reference documentation for SKaiNET operators. + +[NOTE] +==== +This content is automatically generated from source code annotations and should not be edited manually. +Generated on: {docdate} +==== + +[#operator-index] +== Operator Index + +// Generated content will be inserted here by the documentation generation system +// The DocGen tool will populate this with operator listings from operators.json + +[#cross-references] +== Cross-References to Human-Authored Content + +=== Theory References +* xref:../../theory/index.adoc[Mathematical Theory Reference] +* xref:../../theory/matmul.adoc#matmul-definition[Matrix Multiplication Theory] + +=== Usage Examples +* xref:../../examples/index.adoc[Usage Examples] +* xref:../../examples/matmul-examples.adoc#basic-usage[Matrix Multiplication Examples] + +[#anchors] +== Anchor Points for Cross-Linking + +The following anchors are available for cross-referencing from generated content: + +* `#operator-index` - Main operator index +* Individual operator anchors will be generated with pattern: `#operator-{operatorName}` +* Function anchors will follow pattern: `#function-{operatorName}-{functionName}` \ No newline at end of file diff --git a/docs/modules/operators/_generated_plugin_test/index.adoc b/docs/modules/operators/_generated_plugin_test/index.adoc new file mode 100644 index 00000000..f2eb2c9a --- /dev/null +++ b/docs/modules/operators/_generated_plugin_test/index.adoc @@ -0,0 +1,10 @@ += AI-NET Operators Reference + +Generated from version `1.0.0` on 2025-10-23 + +== Operators by Modality + +=== Core + +* xref:voidtensorops.adoc[VoidTensorOps] + diff --git a/docs/modules/operators/_generated_plugin_test/voidtensorops.adoc b/docs/modules/operators/_generated_plugin_test/voidtensorops.adoc new file mode 100644 index 00000000..2defe80c --- /dev/null +++ b/docs/modules/operators/_generated_plugin_test/voidtensorops.adoc @@ -0,0 +1,62 @@ += VoidTensorOps + +Package: `sk.ainet.lang.tensor.ops` + +Modality: Core + +== matmul + +=== Signature + +[source,kotlin] +---- +fun matmul(a:Tensor, b:Tensor): Tensor +---- + +=== Return Type + +`Tensor` + +=== Backend Support + +[cols="1,1,3", options="header"] +|=== +| Backend | Status | Notes +| Metal | in_progress | ops-team; GH-1234 +|=== + +=== Notes + +TIP: *Metal*: ops-team + +TIP: *Metal*: GH-1234 + + +== transpose + +=== Signature + +[source,kotlin] +---- +fun transpose(tensor:Tensor): Tensor +---- + +=== Return Type + +`Tensor` + +=== Backend Support + +[cols="1,1,3", options="header"] +|=== +| Backend | Status | Notes +| Metal | in_progress | ops-team; GH-1234 +|=== + +=== Notes + +TIP: *Metal*: ops-team + +TIP: *Metal*: GH-1234 + + diff --git a/docs/nav.adoc b/docs/nav.adoc new file mode 100644 index 00000000..f23df7ae --- /dev/null +++ b/docs/nav.adoc @@ -0,0 +1,50 @@ += SKaiNET Documentation Navigation + +[#main-nav] +== Main Navigation + +* xref:theory/index.adoc[Mathematical Theory] +** xref:theory/matmul.adoc[Matrix Multiplication] +* xref:examples/index.adoc[Usage Examples] +** xref:examples/matmul-examples.adoc[Matrix Multiplication Examples] +* xref:modules/operators/_generated_/index.adoc[Generated API Reference] + +[#quick-reference] +== Quick Reference + +=== Core Operations +* xref:theory/matmul.adoc#matmul-definition[Matrix Multiplication Theory] +* xref:examples/matmul-examples.adoc#basic-usage[Basic Matrix Multiplication] +* xref:examples/matmul-examples.adoc#neural-network[Neural Network Applications] + +=== Documentation Structure +* `docs/theory/` - Mathematical definitions and theoretical foundations +* `docs/examples/` - Practical usage examples and code samples +* `docs/modules/operators/_generated_/` - Auto-generated API reference + +[#toc-template] +== Table of Contents Template + +The following template can be used for generating table of contents in documentation pages: + +---- +[discrete] +== Table of Contents + +* <> +** <> +* <> +---- + +[#cross-reference-patterns] +== Cross-Reference Patterns + +=== Internal Links +* Theory to Examples: `xref:../examples/matmul-examples.adoc#basic-usage[Matrix Multiplication Examples]` +* Examples to Theory: `xref:../theory/matmul.adoc#matmul-definition[Mathematical Definition]` +* Generated to Human: `xref:../../theory/index.adoc[Theory Reference]` + +=== Anchor Naming Conventions +* Theory anchors: `#operation-definition`, `#operation-properties`, `#operation-complexity` +* Example anchors: `#basic-usage`, `#advanced-usage`, `#performance-tips` +* Generated anchors: `#operator-{name}`, `#function-{operator}-{function}` \ No newline at end of file diff --git a/docs/ops-docs.adoc b/docs/ops-docs.adoc new file mode 100644 index 00000000..642039c9 --- /dev/null +++ b/docs/ops-docs.adoc @@ -0,0 +1,267 @@ += Reflective Documentation for {FRAMEWORK_NAME} Operators + + +SYSTEM / ROLE +You are a senior Kotlin Multiplatform engineer and technical writer. You design build tooling and documentation systems for developer frameworks. You write with precision, provide runnable-ish snippets, and show architecture flows with diagrams. + +CONTEXT +- Framework: a new machine learning framework written in Kotlin Multiplatform (KMP). Name: {FRAMEWORK_NAME}. +- Key concept: Operators ā€“ Kotlin interfaces overridden in different contexts: + ā€¢ In model definition and shape handling, Operators provide "void" (no-op) implementations (no actual compute). + ā€¢ Real computation is provided later by backends (e.g., CPU, Metal, CUDA, WASM) during execution. +- Philosophy: Documentation is a first-class part of product design, not an afterthought. +- Goal: Describe a reflective documentation system that fuses machine-generated metadata (from code) with human-written scientific explanations (math, text, examples), so docs are always in sync with the codebase. + +OBJECTIVE +Write a comprehensive technical article / proposal that explains how to build this reflective documentation system in the Kotlin/JVM/Gradle ecosystem. Show how KSP-generated metadata and human-written content combine into AsciiDoc output, and how Dokka can integrate. Provide examples, sample schemas, and a small end-to-end demo with an example Operator interface. + +AUDIENCE +Senior Kotlin developers, ML framework contributors, technical writers, and tooling engineers. + +SCOPE & MUSTā€‘HAVES (STRUCTURE) +Your article must be written in AsciiDoc and include the following sections (use the exact outline and headings below): + +:toc: +:toclevels: 3 +:icons: font +:source-highlighter: highlight.js + +== 1. Introduction and Motivation +- Explain why documentation is part of product design for {FRAMEWORK_NAME}. +- Introduce the ā€œOperatorā€ abstraction and the need for reflective docs that track interface shape and backend coverage. +- State the pain points this solves (code drift, stale status matrices, ambiguous backend coverage). + +== 2. Reflective Documentation Concept +- Define ā€œreflective documentationā€: combining machine-generated metadata (from the codebase) with human-written narrative. +- Summarize how projects like Sphinx (Python) treat API docs and cross-links as a pipeline; use Sphinx only as an analogy (no Python implementation). +- Contrast traditional docgen vs. duplex-reflective docs (two-way awareness between code + docs). + +== 3. Implementation Plan (KSP-driven metadata) +- Describe using Kotlin Symbol Processing (KSP) to scan Operator interfaces and functions. +- Declare two annotations for backend implementation status: + [source,kotlin] + ---- + @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) + annotation class NotImplemented(vararg val backends: String) + + @Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION) + annotation class InProgress( + vararg val backends: String, + val owner: String = "", + val issue: String = "" // e.g., "GH-1234" + ) + ---- +- Semantics: + ā€¢ If neither annotation is present for a given backend, treat it as Implemented. + ā€¢ @InProgress overrides Implemented for listed backends. + ā€¢ @NotImplemented overrides both and marks listed backends as not available. +- KSP output: generate a single JSON metadata artifact per module per build (e.g., build/generated/{module}/operators.json). + +- Provide a concrete JSON Schema for the metadata (keep realistic but concise). Include at least: + ā€¢ frameworkVersion, commitSha, generatedAt + ā€¢ operators[] with: name, package, modality (interface/class), functions[] + ā€¢ functions[] with: signature, typeParameters, parameters (name:type:shape?), returnType, sinceVersion + ā€¢ statusByBackend: { "CPU": "Implemented|InProgress|NotImplemented", "Metal": ... } + ā€¢ notes (derived from annotation attributes), references (issue links) + + [source,json] + ---- + { + "$schema": "https://example.org/schemas/operator-docs.schema.json", + "frameworkVersion": "0.3.0", + "commitSha": "abc1234", + "generatedAt": "2025-10-22T12:00:00Z", + "backends": ["CPU", "Metal", "CUDA", "WASM"], + "operators": [ + { + "name": "TensorOps", + "package": "ai.{FRAMEWORK_NAME}.ops", + "modality": "interface", + "functions": [ + { + "name": "matmul", + "signature": "fun matmul(a: Tensor, b: Tensor): Tensor", + "parameters": [ + {"name": "a", "type": "Tensor"}, + {"name": "b", "type": "Tensor"} + ], + "returnType": "Tensor", + "sinceVersion": "0.1.0", + "statusByBackend": {"CPU": "Implemented", "Metal": "InProgress", "CUDA": "NotImplemented", "WASM": "Implemented"}, + "notes": [{"backend": "Metal", "owner": "ops-team", "issue": "GH-1234"}] + } + ] + } + ] + } + ---- + +== 4. Documentation Generation (AsciiDoc fragments) +- Explain a Gradle task that converts the KSP JSON into AsciiDoc fragments (one per Operator and optionally one per function). +- Show how fragments embed: + ā€¢ An API signature block + ā€¢ A status table by backend + ā€¢ Pointers (xref::) to human-written math/semantics sections + +- Provide example AsciiDoc fragment: + [source,adoc] + ---- + // generated: do not edit + == TensorOps.matmul + + [source,kotlin] + ---- + fun matmul(a: Tensor, b: Tensor): Tensor + ---- + + .Backend status + |=== + | Backend | Status | Notes + + | CPU | Implemented | + | Metal | InProgress | owner=ops-team, issue=GH-1234 + | CUDA | NotImplemented| + | WASM | Implemented | + |=== + + See xref:theory/matmul.adoc#definition[MatMul semantics] and xref:examples/matmul.adoc#examples[Examples]. + ---- + +- Demonstrate combining generated and human-written docs via include:: and xref::, with a small folder layout: + [source,text] + ---- + docs/ + modules/ + operators/ + _generated_/TensorOps.adoc + _generated_/TensorOps.matmul.adoc + theory/ + matmul.adoc + examples/ + matmul.adoc + index.adoc + ---- + +== 5. Duplex-Reflective Documentation +- Define patterns where human and generated content coexist: + ā€¢ Status tables generated; commentary rows (human) appended below a delimiter. + ā€¢ Human-written caveats that reference generated statuses via xref anchors. +- Show a synchronization flow as a Mermaid diagram: + + [source,mermaid] + ---- + flowchart LR + A[Operator Interfaces (KMP)] --> B[KSP Processor] + B --> C[operators.json] + C --> D[Doc Generator (JSON -> AsciiDoc)] + D --> E[_generated_ Fragments] + E --> F[Master Docs (AsciiDoc)] + F --> G[AsciiDoctorJ/Dokka Output] + G --> H[Website / PDF] + F -->|xref| E + ---- + +- Optionally include a PlantUML sequence to illustrate ā€œhuman + generatedā€ merge logic (optional). + +== 6. Integration & Tooling (Kotlin/JVM/Gradle) +- Keep the pipeline within Kotlin/JVM/Gradle: + ā€¢ KSP processor in :tools:ksp-ops + ā€¢ JSON->AsciiDoc generator in :tools:docgen + ā€¢ AsciiDoctorJ for final HTML/PDF + ā€¢ Dokka integration: link API docs to Operator pages and vice versa + +- Provide sample Gradle snippets: + [source,gradle] + ---- + plugins { + id("com.google.devtools.ksp") version "{KSP_VERSION}" + id("org.jetbrains.dokka") version "{DOKKA_VERSION}" + id("org.asciidoctor.jvm.convert") version "{ASCIIDOCTORJ_VERSION}" + } + + dependencies { + ksp(project(":tools:ksp-ops")) + implementation(project(":tools:docgen")) + } + + tasks.register("generateOperatorDocs") { + dependsOn("kspKotlin") + inputs.file(layout.buildDirectory.file("generated/ksp/operators.json")) + outputs.dir(layout.buildDirectory.dir("docs/_generated_")) + doLast { + // call JSON->AsciiDoc generator + } + } + + tasks.named("asciidoctor") { + dependsOn("generateOperatorDocs") + inputs.dir("$buildDir/docs/_generated_") + } + + tasks.named("dokkaHtml") { + dependsOn("generateOperatorDocs") + } + ---- + +- Explain how Dokka can: + ā€¢ Cross-link symbols to generated Operator pages. + ā€¢ Publish a ā€œSinceā€/ā€œDeprecatedā€ matrix aligned with KSP metadata. + ā€¢ Surface ā€œstatusByBackendā€ as badges in Dokka pages via a plugin. + +== 7. Example Section (TensorOps) +- Provide an example Operator interface and annotations: + + [source,kotlin] + ---- + package ai.{FRAMEWORK_NAME}.ops + + interface TensorOps { + fun matmul(a: Tensor, b: Tensor): Tensor + + @InProgress("Metal", owner="ops-team", issue="GH-1234") + fun relu(x: Tensor): Tensor + + @NotImplemented("CUDA") + fun conv2d(input: Tensor, kernel: Tensor, stride: Int = 1, padding: Int = 0): Tensor + } + ---- + +- Show the KSP-produced JSON excerpt and the corresponding generated AsciiDoc fragment for at least one function (e.g., relu). +- Give a minimal human-written math section for MatMul (dimensions, shapes, complexity), and show how itā€™s included via xref:: from the generated fragment. + +== 8. Summary and Benefits +- Summarize benefits: + ā€¢ Always in sync with code across KMP source sets. + ā€¢ Machine accuracy + human insight. + ā€¢ Scales to multiple backends; clear coverage view. + ā€¢ Reduces stale docs and manual status tables. +- Call out risks and mitigations: + ā€¢ Annotation misuse ā†’ add CI checks. + ā€¢ Cross-module drift ā†’ centralize schema versioning. + ā€¢ Backend sprawl ā†’ maintain a single source of truth for backend identifiers. + +APPENDIX (OPTIONAL BUT STRONGLY RECOMMENDED) +- CI integration (GitHub Actions) to fail builds when JSON schema changes or coverage drops. +- A ā€œDocs Previewā€ Gradle task (`docsPreview`) that launches a local server. +- A short migration guide from non-reflective docs. + +OUTPUT FORMAT REQUIREMENTS +- Write the entire article as **AsciiDoc**. +- Use code blocks with language tags: [source,kotlin], [source,gradle], [source,json], [source,adoc], [source,mermaid], [source,plantuml] (PlantUML optional). +- Use short paragraphs and bullet lists; avoid filler or marketing language. +- Include at least: + ā€¢ One Mermaid diagram (the pipeline). + ā€¢ One status table per example Operator/function. + ā€¢ One JSON Schema block and one JSON example. + ā€¢ Kotlin + Gradle snippets that would compile with minor stubs. +- Target length: ~1,300ā€“2,200 words. Itā€™s okay to exceed if content is substantive. + +ACCEPTANCE CHECKLIST (the output must satisfy all) +- [ ] AsciiDoc file with the 8 sections named above in order. +- [ ] Clear definition of ā€œreflective documentationā€ and how it differs from classic docgen. +- [ ] KSP plan, annotation semantics, and JSON Schema included. +- [ ] Example Operator (TensorOps) + generated metadata + generated AsciiDoc fragment. +- [ ] Demonstrated include:: and xref:: usage. +- [ ] Mermaid pipeline diagram present. +- [ ] Gradle/Dokka/AsciiDoctorJ integration details with code. +- [ ] Summary articulates benefits and risks. \ No newline at end of file diff --git a/docs/theory/index.adoc b/docs/theory/index.adoc new file mode 100644 index 00000000..7b4c94b2 --- /dev/null +++ b/docs/theory/index.adoc @@ -0,0 +1,36 @@ += Mathematical Theory Reference + +This section contains mathematical definitions and theoretical foundations for SKaiNET operators. + +[#operator-theory] +== Operator Theory + +=== Linear Algebra Operations + +include::matmul.adoc[leveloffset=+2] + +=== Activation Functions + +// TODO: Add theory for activation functions +// include::activations.adoc[leveloffset=+2] + +=== Convolution Operations + +// TODO: Add theory for convolution operations +// include::convolution.adoc[leveloffset=+2] + +=== Normalization Operations + +// TODO: Add theory for normalization operations +// include::normalization.adoc[leveloffset=+2] + +=== Loss Functions + +// TODO: Add theory for loss functions +// include::losses.adoc[leveloffset=+2] + +[#cross-references] +== Cross-References + +* xref:../examples/index.adoc[Usage Examples] +* xref:../modules/operators/_generated_/index.adoc[Generated API Reference] \ No newline at end of file diff --git a/docs/theory/matmul.adoc b/docs/theory/matmul.adoc new file mode 100644 index 00000000..c40ff179 --- /dev/null +++ b/docs/theory/matmul.adoc @@ -0,0 +1,41 @@ += Matrix Multiplication Theory + +[#matmul-definition] +== Mathematical Definition + +Matrix multiplication is a binary operation that produces a matrix from two matrices. +Given two matrices A āˆˆ ā„^(mƗk) and B āˆˆ ā„^(kƗn), the matrix product C = AB is defined as: + +[stem] +++++ +C_{ij} = \sum_{l=1}^{k} A_{il} \cdot B_{lj} +++++ + +Where: +- C āˆˆ ā„^(mƗn) is the resulting matrix +- i ranges from 1 to m (row index) +- j ranges from 1 to n (column index) +- l is the summation index over the shared dimension k + +[#matmul-properties] +== Properties + +* **Associativity**: (AB)C = A(BC) +* **Distributivity**: A(B + C) = AB + AC and (A + B)C = AC + BC +* **Non-commutativity**: Generally AB ā‰  BA +* **Identity element**: AI = IA = A where I is the identity matrix + +[#matmul-complexity] +== Computational Complexity + +* Standard algorithm: O(mnk) operations +* Strassen's algorithm: O(n^2.807) for square matrices +* Current best known: O(n^2.373) (theoretical) + +[#matmul-applications] +== Applications + +* Neural network forward pass computations +* Linear transformations in computer graphics +* Solving systems of linear equations +* Principal component analysis (PCA) \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index a6ae62b6..35c127d8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -18,12 +18,14 @@ POM_DEVELOPER_URL=https://github.com/sk-ai-net/ mavenCentralPublishing=true mavenCentralAutomaticPublishing=true -signAllPublications=true +signAllPublications=false #Gradle org.gradle.jvmargs=-Xmx2048M -Dfile.encoding=UTF-8 -Dkotlin.daemon.jvm.options\="-Xmx2048M" org.gradle.caching=true org.gradle.configuration-cache=true +org.gradle.parallel=true +org.gradle.configureondemand=true #Kotlin kotlin.code.style=official #MPP diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a3c69756..5dd751f0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,5 @@ [versions] -agp = "8.11.2" +agp = "8.12.3" kotlin = "2.2.20" kotlinxCoroutines = "1.10.2" android-minSdk = "24" @@ -13,8 +13,9 @@ binaryCompatibilityValidator = "0.18.1" ksp = "2.2.20-2.0.4" kotlinpoet = "2.2.0" kotlin-compile-testing = "1.6.0" -asciidoctorj = "2.5.13" -dokka = "1.9.20" +asciidoctorj = "3.0.0" +dokka = "2.1.0" +kotlinxCli = "0.3.6" [libraries] kotlinx-coroutines = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "kotlinxCoroutines" } @@ -23,6 +24,8 @@ kotlinx-coroutines-test = { group = "org.jetbrains.kotlinx", name = "kotlinx-cor kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotlin" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } +kotlinx-cli = { module = "org.jetbrains.kotlinx:kotlinx-cli", version.ref = "kotlinxCli" } + ktor-client-android = { module = "io.ktor:ktor-client-android", version.ref = "ktorClientCore" } ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktorClientCore" } @@ -43,6 +46,10 @@ kotlin-compile-testing-ksp = { module = "com.github.tschuchortdev:kotlin-compile asciidoctorj-core = { module = "org.asciidoctor:asciidoctorj", version.ref = "asciidoctorj" } asciidoctorj-pdf = { module = "org.asciidoctor:asciidoctorj-pdf", version.ref = "asciidoctorj" } +dokka-base = { module = "org.jetbrains.dokka:dokka-base", version.ref = "dokka" } +dokka-core = { module = "org.jetbrains.dokka:dokka-core", version.ref = "dokka" } +dokka-test-utils = { module = "org.jetbrains.dokka:dokka-test-utils", version.ref = "dokka" } + logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logbackClassic" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 7e3cf00d..146439b5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -20,3 +20,4 @@ include("skainet-lang:skainet-lang-models") include("skainet-lang:skainet-lang-ksp-annotations") include("skainet-lang:skainet-lang-ksp-processor") include("skainet-lang:skainet-lang-export-ops") +include("tools:docgen") diff --git a/skainet-lang/skainet-lang-core/build.gradle.kts b/skainet-lang/skainet-lang-core/build.gradle.kts index 40ecbe56..760f8483 100644 --- a/skainet-lang/skainet-lang-core/build.gradle.kts +++ b/skainet-lang/skainet-lang-core/build.gradle.kts @@ -8,6 +8,8 @@ plugins { alias(libs.plugins.vanniktech.mavenPublish) alias(libs.plugins.kover) alias(libs.plugins.binary.compatibility.validator) + alias(libs.plugins.ksp) + alias(libs.plugins.dokka) } kotlin { @@ -45,8 +47,14 @@ kotlin { } } +dependencies { + add("kspCommonMainMetadata", project(":skainet-lang:skainet-lang-ksp-processor")) + add("kspJvm", project(":skainet-lang:skainet-lang-ksp-processor")) + add("kspAndroid", project(":skainet-lang:skainet-lang-ksp-processor")) +} + android { - namespace = "sk.ai.net.lang.api" + namespace = "sk.ai.net.lang.core" compileSdk = libs.versions.android.compileSdk.get().toInt() defaultConfig { @@ -56,4 +64,8 @@ android { sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 } +} + +tasks.named("dokkaHtml") { + dependsOn("kspCommonMainKotlinMetadata") } \ No newline at end of file diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ComputeGraphProcessor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ComputeGraphProcessor.kt deleted file mode 100644 index 73ad3878..00000000 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ComputeGraphProcessor.kt +++ /dev/null @@ -1,235 +0,0 @@ -package sk.ainet.lang.ops.ksp - -import com.google.devtools.ksp.processing.* -import com.google.devtools.ksp.symbol.* -import com.google.devtools.ksp.validate -import com.squareup.kotlinpoet.* -import com.squareup.kotlinpoet.ksp.writeTo -import sk.ainet.lang.ops.ComputationMode -import sk.ainet.lang.ops.TensorOp -import java.io.File - -// KSP Processor -class ComputeGraphProcessor( - private val codeGenerator: CodeGenerator, - private val logger: KSPLogger -) : SymbolProcessor { - - override fun process(resolver: Resolver): List { - val symbols = resolver.getSymbolsWithAnnotation(TensorOp::class.qualifiedName!!) - logger.info("Found ${symbols.count()} symbols with @Mikrograd annotation") - val invalidSymbols = symbols.filter { !it.validate() }.toList() - logger.info("Found ${invalidSymbols.size} invalid symbols") - - symbols.filter { it is KSFunctionDeclaration && it.validate() } - .forEach { symbol -> - val function = symbol as KSFunctionDeclaration - logger.info("Processing function: ${function.simpleName.asString()}") - logger.info(" - Package: ${function.packageName.asString()}") - logger.info(" - File: ${function.containingFile?.fileName}") - logger.info(" - Parameters: ${function.parameters.map { it.name?.asString() to it.type.resolve().declaration.qualifiedName?.asString() }}") - logger.info(" - Return type: ${function.returnType?.resolve()?.declaration?.qualifiedName?.asString()}") - - // Extract the computation mode from the annotation - val annotation = function.annotations.find { - it.shortName.asString() == "Mikrograd" - } - - // Default to INFERENCE if the mode argument is not specified - val modeArgument = annotation?.arguments?.find { it.name?.asString() == "mode" } - val modeValue = modeArgument?.value?.toString() ?: "INFERENCE" - - // Extract just the enum constant name (INFERENCE or TRAINING) from the fully qualified name - val enumConstantName = modeValue.substringAfterLast('.', modeValue) - val mode = ComputationMode.valueOf(enumConstantName) - - logger.info(" - Computation mode: $mode") - - try { - generateComputeGraphCode(function, mode) - } catch (e: Exception) { - logger.error("Failed to process function ${function.simpleName.asString()}: ${e.message}", symbol) - } - } - - return invalidSymbols - } - - private fun generateComputeGraphCode(function: KSFunctionDeclaration, mode: ComputationMode) { - val packageName = function.packageName.asString() - val fileName = "${function.simpleName.asString()}Generated" - logger.info("Generating code for function: ${function.simpleName.asString()}") - logger.info(" - Output file: $packageName.$fileName") - logger.info(" - Computation mode: $mode") - - // Log AST details - logger.info(" - AST details:") - logger.info(" - Modifiers: ${function.modifiers.map { it.name }}") - logger.info(" - Documentation: ${function.docString}") - logger.info(" - Location: ${function.location}") - - // Extract the function body as a string - val functionBody = extractFunctionBody(function) - // If we couldn't extract the function body, use a default expression - val expressionString = functionBody ?: "3.0 * 8.0 + (7.0 + 3.0)" - logger.info(" - Extracted expression: $expressionString") - - // Parse the expression and generate code - val parser = ExpressionParser() - - // Use the appropriate visitor based on the mode - val visitor = DifferentiationVisitor(mode) - val codeBlock = parser.parseExpression(expressionString, visitor) - - // Get the last variable name from the code block - val lastVarName = extractLastVarName(codeBlock.toString()) - logger.info(" - Last variable name: $lastVarName") - - val fileSpec = FileSpec.builder(packageName, fileName) - - // Build the function using KotlinPoet, wrapping the entire code in a single context block - val funSpec = FunSpec.builder(function.simpleName.asString() + "Generated") - .returns(ClassName("org.mikrograd.diff", if (mode == ComputationMode.INFERENCE) "AutoDiffNode" else "BackpropNode")) - .addCode(codeBlock) - .addStatement("return $lastVarName") - .build() - - logger.info(" - Function spec created: ${funSpec.name}") - - // Add imports based on the computation mode - val imports = mutableListOf( - "org.mikrograd.core.ComputeNode", - "org.mikrograd.core.ValueNode", - "org.mikrograd.core.MultiplyNode", - "org.mikrograd.core.AddNode", - "org.mikrograd.diff.ForwardPassNode" - ) - - // Add mode-specific imports - if (mode == ComputationMode.INFERENCE) { - imports.add("org.mikrograd.diff.ForwardPassNode") - } else { - imports.add("org.mikrograd.diff.BackpropNode") - } - - // Add ValueInterface import - imports.add("org.mikrograd.diff.AutoDiffNode") - - // Write the file with imports - fileSpec.addFileComment("Generated by ComputeGraphProcessor") - .addFileComment(" - Mode: $mode") - - // Add imports - imports.forEach { importPath -> - val lastDot = importPath.lastIndexOf('.') - val packageName = importPath.substring(0, lastDot) - val className = importPath.substring(lastDot + 1) - fileSpec.addImport(packageName, className) - } - - fileSpec.addFunction(funSpec) - .build() - .writeTo(codeGenerator, Dependencies(false, function.containingFile!!)) - - logger.info(" - Code generation completed for ${function.simpleName.asString()}") - } - - /** - * Extract the variable name from the last statement in a code block. - * This is a simplistic implementation that assumes the last statement - * in the code block is a variable declaration. - * @param codeBlock The code block to extract from - * @return The variable name - */ - private fun extractLastVarName(codeBlock: String): String { - // Find the last variable declaration in the code block - val statements = codeBlock.trim().split("\n") - for (i in statements.indices.reversed()) { - val statement = statements[i] - val match = Regex("val (\\w+)").find(statement) - if (match != null) { - return match.groupValues[1] - } - } - - // If no variable declaration is found, return a default name - return "resultNode" - } - - /** - * Extract the function body as a string from a KSFunctionDeclaration. - * This method reads the source file directly and extracts the function body - * based on the function's location in the file. - * @param function The function declaration - * @return The function body as a string, or null if it couldn't be extracted - */ - private fun extractFunctionBody(function: KSFunctionDeclaration): String? { - try { - // Get the file path from the containing file - val filePath = function.containingFile?.filePath ?: return null - logger.info(" - Source file path: $filePath") - - // Read the file content - val fileContent = File(filePath).readText() - logger.info(" - File content length: ${fileContent.length}") - - // Get the function's location in the file - val location = function.location - logger.info(" - Function location: $location") - - // Extract the function body by finding the opening and closing braces - // or by finding the expression body after the equals sign - val functionName = function.simpleName.asString() - - // First try to match a function with a body enclosed in braces - val blockBodyPattern = - Regex("fun\\s+$functionName\\s*\\([^)]*\\)\\s*\\{([\\s\\S]*?)\\}", RegexOption.DOT_MATCHES_ALL) - val blockBodyMatch = blockBodyPattern.find(fileContent) - - if (blockBodyMatch != null && blockBodyMatch.groupValues.size > 1) { - val functionBody = blockBodyMatch.groupValues[1].trim() - logger.info(" - Extracted block body: $functionBody") - return functionBody - } - - // If that fails, try to match a function with an expression body - val exprBodyPattern = Regex( - "fun\\s+$functionName\\s*\\([^)]*\\)(?:\\s*:\\s*[^=]+)?\\s*=\\s*([^{]+)\\{([\\s\\S]*?)\\}", - RegexOption.DOT_MATCHES_ALL - ) - val exprBodyMatch = exprBodyPattern.find(fileContent) - - if (exprBodyMatch != null && exprBodyMatch.groupValues.size > 2) { - // For expression bodies, we're interested in the content inside the curly braces - val contextFunction = exprBodyMatch.groupValues[1].trim() - val functionBody = exprBodyMatch.groupValues[2].trim() - logger.info(" - Extracted expression body with context function $contextFunction: $functionBody") - return functionBody - } - - // Log the function declaration for debugging - logger.error("Function declaration not matched by regex patterns") - val functionDeclarationPattern = Regex("fun\\s+$functionName[^{]*", RegexOption.DOT_MATCHES_ALL) - val functionDeclarationMatch = functionDeclarationPattern.find(fileContent) - if (functionDeclarationMatch != null) { - logger.error("Function declaration: ${functionDeclarationMatch.value}") - } - - logger.error("Failed to extract function body for ${function.simpleName.asString()}") - return null - } catch (e: Exception) { - logger.error("Error extracting function body: ${e.message}") - return null - } - } - - companion object { - private val DOUBLE = ClassName("kotlin", "Double") - } -} - -class ComputeGraphProcessorProvider : SymbolProcessorProvider { - override fun create(environment: SymbolProcessorEnvironment): SymbolProcessor { - return ComputeGraphProcessor(environment.codeGenerator, environment.logger) - } -} diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionParser.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionParser.kt deleted file mode 100644 index 7375a19f..00000000 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionParser.kt +++ /dev/null @@ -1,168 +0,0 @@ -package sk.ainet.lang.ops.ksp - -import com.squareup.kotlinpoet.CodeBlock - -/** - * Parser for mathematical expressions. - * This class parses a simple expression and builds a compute graph using the visitor pattern. - */ -class ExpressionParser { - /** - * Parse an expression and generate code for the compute graph. - * @param expression The expression to parse - * @param visitor The visitor to use for code generation (defaults to CodeGeneratingVisitor) - * @return The code block representing the compute graph - */ - fun parseExpression(expression: String, visitor: ComputeNodeVisitor = CodeGeneratingVisitor()): CodeBlock { - val tokens = tokenize(expression) - val ast = buildAST(tokens) - return generateCode(ast, visitor) - } - - /** - * Tokenize an expression into a list of tokens. - * @param expression The expression to tokenize - * @return The list of tokens - */ - private fun tokenize(expression: String): List { - val tokens = mutableListOf() - var i = 0 - while (i < expression.length) { - val c = expression[i] - when { - c.isDigit() || c == '.' -> { - var j = i - while (j < expression.length && (expression[j].isDigit() || expression[j] == '.')) { - j++ - } - tokens.add(Token.Number(expression.substring(i, j).toDouble())) - i = j - } - c == '+' -> { - tokens.add(Token.Plus) - i++ - } - c == '*' -> { - tokens.add(Token.Times) - i++ - } - c == '(' -> { - tokens.add(Token.LeftParen) - i++ - } - c == ')' -> { - tokens.add(Token.RightParen) - i++ - } - c.isWhitespace() -> { - i++ - } - else -> { - throw IllegalArgumentException("Unexpected character: $c") - } - } - } - return tokens - } - - /** - * Build an abstract syntax tree (AST) from a list of tokens. - * @param tokens The list of tokens - * @return The root node of the AST - */ - private fun buildAST(tokens: List): ASTNode { - // This is a simple recursive descent parser for expressions - // It handles the following grammar: - // expr = term { "+" term } - // term = factor { "*" factor } - // factor = number | "(" expr ")" - - var pos = 0 - - // Forward declarations - lateinit var parseExpr: () -> ASTNode - lateinit var parseTerm: () -> ASTNode - lateinit var parseFactor: () -> ASTNode - - // Implementation - parseExpr = { - var left = parseTerm() - while (pos < tokens.size && tokens[pos] == Token.Plus) { - pos++ - val right = parseTerm() - left = ASTNode.Add(left, right) - } - left - } - - parseTerm = { - var left = parseFactor() - while (pos < tokens.size && tokens[pos] == Token.Times) { - pos++ - val right = parseFactor() - left = ASTNode.Multiply(left, right) - } - left - } - - parseFactor = { - when (val token = tokens[pos++]) { - is Token.Number -> ASTNode.Value(token.value) - Token.LeftParen -> { - val expr = parseExpr() - if (pos < tokens.size && tokens[pos] == Token.RightParen) { - pos++ - expr - } else { - throw IllegalArgumentException("Expected closing parenthesis") - } - } - else -> throw IllegalArgumentException("Unexpected token: $token") - } - } - - return parseExpr() - } - - /** - * Generate code for an AST using a visitor. - * @param ast The AST to generate code for - * @param visitor The visitor to use - * @return The generated code - */ - private fun generateCode(ast: ASTNode, visitor: ComputeNodeVisitor): CodeBlock { - return when (ast) { - is ASTNode.Value -> visitor.visitValueNode(ast.value, "const_${ast.value}") - is ASTNode.Add -> visitor.visitAddNode( - generateCode(ast.left, visitor), - generateCode(ast.right, visitor), - "add_${ast.left}_${ast.right}" - ) - is ASTNode.Multiply -> visitor.visitMultiplyNode( - generateCode(ast.left, visitor), - generateCode(ast.right, visitor), - "multiply_${ast.left}_${ast.right}" - ) - } - } - - /** - * Token types for the tokenizer. - */ - sealed class Token { - data class Number(val value: Double) : Token() - object Plus : Token() - object Times : Token() - object LeftParen : Token() - object RightParen : Token() - } - - /** - * AST node types for the parser. - */ - sealed class ASTNode { - data class Value(val value: Double) : ASTNode() - data class Add(val left: ASTNode, val right: ASTNode) : ASTNode() - data class Multiply(val left: ASTNode, val right: ASTNode) : ASTNode() - } -} diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionVisitor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionVisitor.kt deleted file mode 100644 index 6df6a085..00000000 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/ExpressionVisitor.kt +++ /dev/null @@ -1,206 +0,0 @@ -package sk.ainet.lang.ops.ksp - -import com.squareup.kotlinpoet.CodeBlock -import com.squareup.kotlinpoet.ClassName -import sk.ainet.lang.ops.ComputationMode - -/** - * Visitor for generating code that uses the differentiation context. - * This visitor generates code that uses either ForwardValue or BackwardValue - * based on the computation mode. - */ -class DifferentiationVisitor(private val mode: ComputationMode) : ComputeNodeVisitor { - // Counter for generating unique variable names - private var nodeCounter = 0 - - override fun visitValueNode(value: Double, id: String): CodeBlock { - val varName = generateNodeName("value") - return CodeBlock.builder() - .addStatement("val $varName = ${getConstructorByMode()}($value)") - .build() - } - - private fun getConstructorByMode(): String = - if (mode == ComputationMode.INFERENCE) { - "ForwardPassNode" - } else { - "BackpropNode" - } - - - override fun visitAddNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock { - val leftVarName = extractLastVarName(left) - val rightVarName = extractLastVarName(right) - val varName = generateNodeName("add") - - return CodeBlock.builder() - .add(left) - .add(right) - .addStatement("val $varName = $leftVarName + $rightVarName") - .build() - } - - override fun visitMultiplyNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock { - val leftVarName = extractLastVarName(left) - val rightVarName = extractLastVarName(right) - val varName = generateNodeName("multiply") - - return CodeBlock.builder() - .add(left) - .add(right) - .addStatement("val $varName = $leftVarName * $rightVarName") - .build() - } - - /** - * Generate a unique variable name for a node. - * @param prefix The prefix for the variable name - * @return The generated variable name - */ - private fun generateNodeName(prefix: String): String { - return "${prefix}${nodeCounter++}" - } - - /** - * Extract the variable name from the last statement in a code block. - * This is a simplistic implementation that assumes the last statement - * in the code block is a variable declaration. - * @param codeBlock The code block to extract from - * @return The variable name - */ - private fun extractLastVarName(codeBlock: CodeBlock): String { - // Find the last variable declaration in the code block - val statements = codeBlock.toString().trim().split("\n") - for (i in statements.indices.reversed()) { - val statement = statements[i] - val match = Regex("val (\\w+)").find(statement) - if (match != null) { - return match.groupValues[1] - } - } - - // If no variable declaration is found, return a default name - return "resultNode" - } -} - -/** - * Visitor interface for traversing and evaluating compute nodes. - * This interface defines methods for visiting different types of compute nodes - * and generating the corresponding code. - */ -interface ComputeNodeVisitor { - /** - * Visit a value node (leaf node with a constant value). - * @param value The value of the node - * @param id The ID of the node - * @return The code block representing the compute node - */ - fun visitValueNode(value: T, id: String): CodeBlock - - /** - * Visit an add node (node that adds two input values). - * @param left The left input node - * @param right The right input node - * @param id The ID of the node - * @return The code block representing the compute node - */ - fun visitAddNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock - - /** - * Visit a multiply node (node that multiplies two input values). - * @param left The left input node - * @param right The right input node - * @param id The ID of the node - * @return The code block representing the compute node - */ - fun visitMultiplyNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock -} - -/** - * Implementation of the ComputeNodeVisitor interface for generating code blocks. - */ -class CodeGeneratingVisitor : ComputeNodeVisitor { - // Counter for generating unique variable names - private var nodeCounter = 0 - - override fun visitValueNode(value: Double, id: String): CodeBlock { - val varName = generateNodeName("value") - return CodeBlock.builder() - .addStatement( - "val $varName = %T($value).withId(%S)", - ClassName("org.mikrograd.core", "ValueNode"), - id - ) - .build() - } - - override fun visitAddNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock { - val leftVarName = extractLastVarName(left) - val rightVarName = extractLastVarName(right) - val varName = generateNodeName("add") - - return CodeBlock.builder() - .add(left) - .add(right) - .addStatement( - "val $varName = %T<%T> { a, b -> a + b }.withId(%S)", - ClassName("org.mikrograd.core", "AddNode"), - ClassName("kotlin", "Double"), - id - ) - .addStatement("$varName.inputs.add($leftVarName)") - .addStatement("$varName.inputs.add($rightVarName)") - .build() - } - - override fun visitMultiplyNode(left: CodeBlock, right: CodeBlock, id: String): CodeBlock { - val leftVarName = extractLastVarName(left) - val rightVarName = extractLastVarName(right) - val varName = generateNodeName("multiply") - - return CodeBlock.builder() - .add(left) - .add(right) - .addStatement( - "val $varName = %T<%T> { a, b -> a * b }.withId(%S)", - ClassName("org.mikrograd.core", "MultiplyNode"), - ClassName("kotlin", "Double"), - id - ) - .addStatement("$varName.inputs.add($leftVarName)") - .addStatement("$varName.inputs.add($rightVarName)") - .build() - } - - /** - * Generate a unique variable name for a node. - * @param prefix The prefix for the variable name - * @return The generated variable name - */ - private fun generateNodeName(prefix: String): String { - return "${prefix}${nodeCounter++}" - } - - /** - * Extract the variable name from the last statement in a code block. - * This is a simplistic implementation that assumes the last statement - * in the code block is a variable declaration. - * @param codeBlock The code block to extract from - * @return The variable name - */ - private fun extractLastVarName(codeBlock: CodeBlock): String { - // Find the last variable declaration in the code block - val statements = codeBlock.toString().trim().split("\n") - for (i in statements.indices.reversed()) { - val statement = statements[i] - val match = Regex("val (\\w+)").find(statement) - if (match != null) { - return match.groupValues[1] - } - } - - // If no variable declaration is found, return a default name - return "resultNode" - } -} diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider b/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider index 379d91a2..67d335de 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/resources/META-INF/services/com.google.devtools.ksp.processing.SymbolProcessorProvider @@ -1,3 +1,2 @@ -sk.ainet.lang.ops.ksp.ComputeGraphProcessorProvider sk.ainet.lang.ops.ksp.OperatorDocProcessorProvider diff --git a/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt b/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt deleted file mode 100644 index f6d89bed..00000000 --- a/skainet-lang/skainet-lang-ksp-processor/src/test/kotlin/org/mikrograd/diff/ksp/ComputeGraphProcessorTest.kt +++ /dev/null @@ -1,125 +0,0 @@ -@file:OptIn(ExperimentalCompilerApi::class) - -package org.mikrograd.diff.ksp - -import com.tschuchort.compiletesting.KotlinCompilation -import com.tschuchort.compiletesting.SourceFile -import com.tschuchort.compiletesting.symbolProcessorProviders -import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi -import org.junit.Test -import sk.ainet.lang.ops.ksp.ComputeGraphProcessorProvider -import java.io.ByteArrayOutputStream -import java.io.PrintStream -import kotlin.test.assertTrue - -class ComputeGraphProcessorTest { - - @OptIn(ExperimentalCompilerApi::class) - @Test - fun testProcessorGeneratesCodeWithDefaultMode() { - // Create a test Kotlin source file with a function annotated with @Mikrograd - val sourceCode = """ - @org.mikrograd.diff.ksp.Mikrograd - fun testExpr() { - 3.0 * 4.0 + (7.0 + 3.0) - } - """ - val source = SourceFile.kotlin("test/TestFile.kt", sourceCode) - - // Capture the output - val outputStream = ByteArrayOutputStream() - val printStream = PrintStream(outputStream) - val originalOut = System.out - System.setOut(printStream) - - try { - // Compile the source file with the ComputeGraphProcessor - val compilation = KotlinCompilation().apply { - sources = listOf(source) - symbolProcessorProviders = listOf(ComputeGraphProcessorProvider()) - inheritClassPath = true - messageOutputStream = printStream - } - - // Run the compilation - compilation.compile() - - // Get the output - val output = outputStream.toString() - - // Print the output for debugging - System.setOut(originalOut) - println("[DEBUG_LOG] Compilation output:") - println(output) - - // Check that the KSP processor found and processed the function - assertTrue(output.contains("Found 1 symbols with @Mikrograd annotation"), - "KSP processor should find the annotated function") - assertTrue(output.contains("Processing function: testExpr"), - "KSP processor should process the testExpr function") - assertTrue(output.contains("Generating code for function: testExpr"), - "KSP processor should generate code for the testExpr function") - assertTrue(output.contains("Computation mode: INFERENCE"), - "KSP processor should use INFERENCE mode by default") - assertTrue(output.contains("Code generation completed for testExpr"), - "KSP processor should complete code generation for the testExpr function") - } finally { - // Restore the original output stream - System.setOut(originalOut) - } - } - - @Test - fun testProcessorGeneratesCodeWithTrainingMode() { - // Create a test Kotlin source file with a function annotated with @Mikrograd(mode = ComputationMode.TRAINING) - val sourceCode = """ - @org.mikrograd.diff.ksp.Mikrograd(mode = org.mikrograd.diff.ksp.ComputationMode.TRAINING) - fun testExprTraining() { - 3.0 * 4.0 + (7.0 + 3.0) - } - """ - val source = SourceFile.kotlin("test/TestFileTraining.kt", sourceCode) - - // Capture the output - val outputStream = ByteArrayOutputStream() - val printStream = PrintStream(outputStream) - val originalOut = System.out - System.setOut(printStream) - - try { - // Compile the source file with the ComputeGraphProcessor - val compilation = KotlinCompilation().apply { - sources = listOf(source) - symbolProcessorProviders = listOf(ComputeGraphProcessorProvider()) - inheritClassPath = true - messageOutputStream = printStream - } - - // Run the compilation - compilation.compile() - - // Get the output - val output = outputStream.toString() - - // Print the output for debugging - System.setOut(originalOut) - println("[DEBUG_LOG] Compilation output:") - println(output) - - // Check that the KSP processor found and processed the function - assertTrue(output.contains("Found 1 symbols with @Mikrograd annotation"), - "KSP processor should find the annotated function") - assertTrue(output.contains("Processing function: testExprTraining"), - "KSP processor should process the testExprTraining function") - assertTrue(output.contains("Generating code for function: testExprTraining"), - "KSP processor should generate code for the testExprTraining function") - assertTrue(output.contains("Computation mode: TRAINING"), - "KSP processor should use TRAINING mode as specified") - assertTrue(output.contains("Code generation completed for testExprTraining"), - "KSP processor should complete code generation for the testExprTraining function") - } finally { - // Restore the original output stream - System.setOut(originalOut) - } - } -} diff --git a/tools/docgen/build.gradle.kts b/tools/docgen/build.gradle.kts index 37d85767..a28e917a 100644 --- a/tools/docgen/build.gradle.kts +++ b/tools/docgen/build.gradle.kts @@ -1,14 +1,17 @@ plugins { kotlin("jvm") alias(libs.plugins.kotlinSerialization) + alias(libs.plugins.asciidoctorJvm) application } dependencies { implementation(kotlin("stdlib")) - implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.2") - implementation("org.jetbrains.kotlinx:kotlinx-cli:0.3.6") - + implementation(libs.kotlinx.serialization.json) + implementation(libs.kotlinx.cli) + + implementation(libs.asciidoctorj.core) + testImplementation(kotlin("test")) testImplementation("org.junit.jupiter:junit-jupiter:5.10.1") } @@ -17,6 +20,13 @@ application { mainClass.set("sk.ainet.tools.docgen.DocGenKt") } +tasks.named("run") { + args = listOf( + "--input", "${project.rootDir}/skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json", + "--output", "${project.rootDir}/docs/modules/operators/_generated_" + ) +} + tasks.test { useJUnitPlatform() } diff --git a/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt b/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt index 4280c601..e30f584f 100644 --- a/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt +++ b/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt @@ -15,7 +15,7 @@ data class OperatorDocModule( @Serializable data class OperatorDoc( val name: String, - val packageName: String, + @kotlinx.serialization.SerialName("package") val packageName: String, val modality: String, val functions: List ) @@ -41,5 +41,5 @@ data class ParameterDoc( data class Note( val type: String, val backend: String, - val content: String + val message: String ) \ No newline at end of file diff --git a/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt b/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt index 7c4d1395..3f026e6c 100644 --- a/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt +++ b/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt @@ -137,6 +137,12 @@ object DocGen { generateBackendStatusTable(function, this) } + // API reference link + appendLine("==== See also") + appendLine() + appendLine("* xref:api:${function.name}[API Reference (Dokka)]") + appendLine("* xref:theory:${function.name}.adoc[Mathematical Definition]") + appendLine("* xref:examples:${function.name}.adoc[Usage Examples]") appendLine() } } @@ -156,9 +162,9 @@ object DocGen { if (backendNotes.isNotEmpty()) { val notesText = backendNotes.joinToString(", ") { note -> when (note.type) { - "owner" -> "Owner: ${note.content}" - "issue" -> "Issue: ${note.content}" - else -> "${note.type}: ${note.content}" + "owner" -> "Owner: ${note.message}" + "issue" -> "Issue: ${note.message}" + else -> "${note.type}: ${note.message}" } } appendLine("| ${notesText}") From 1763c7c47e3d5e00cfdffdccda79f8fa0f3f0161 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Sun, 26 Oct 2025 12:22:19 +0100 Subject: [PATCH 09/11] Remove gradle module of logic moved to gradle plugin Related-To #139 --- build.gradle.kts | 9 +- settings.gradle.kts | 1 - .../sk/ainet/tools/docgen/DataModels.kt | 45 ---- .../kotlin/sk/ainet/tools/docgen/DocGen.kt | 220 ------------------ .../src/test/resources/test-operators.json | 73 ------ 5 files changed, 2 insertions(+), 346 deletions(-) delete mode 100644 tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt delete mode 100644 tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt delete mode 100644 tools/docgen/src/test/resources/test-operators.json diff --git a/build.gradle.kts b/build.gradle.kts index 784ab54e..147afad8 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -35,8 +35,6 @@ tasks.register("generateOperatorDocs") { // Configure inputs for incremental builds inputs.files("skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json") - inputs.files("tools/docgen/src/main/kotlin") - // Configure outputs for incremental builds outputs.dir("docs/modules/operators/_generated_") outputs.cacheIf { true } @@ -44,11 +42,8 @@ tasks.register("generateOperatorDocs") { // Depend on KSP processing dependsOn(":skainet-lang:skainet-lang-core:kspCommonMainKotlinMetadata") - // Depend on DocGen application - dependsOn(":tools:docgen:run") - - // Final step: process with AsciiDoctor - finalizedBy(":tools:docgen:asciidoctor") + // Run built-in documentation generation task (provided by sk.ainet.documentation plugin) + dependsOn("generateDocs") doLast { println("Operator documentation generation completed") diff --git a/settings.gradle.kts b/settings.gradle.kts index 146439b5..7e3cf00d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -20,4 +20,3 @@ include("skainet-lang:skainet-lang-models") include("skainet-lang:skainet-lang-ksp-annotations") include("skainet-lang:skainet-lang-ksp-processor") include("skainet-lang:skainet-lang-export-ops") -include("tools:docgen") diff --git a/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt b/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt deleted file mode 100644 index e30f584f..00000000 --- a/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DataModels.kt +++ /dev/null @@ -1,45 +0,0 @@ -package sk.ainet.tools.docgen - -import kotlinx.serialization.Serializable - -@Serializable -data class OperatorDocModule( - val schema: String = "https://skainet.ai/schemas/operator-doc/v1", - val version: String, - val commit: String, - val timestamp: String, - val module: String, - val operators: List -) - -@Serializable -data class OperatorDoc( - val name: String, - @kotlinx.serialization.SerialName("package") val packageName: String, - val modality: String, - val functions: List -) - -@Serializable -data class FunctionDoc( - val name: String, - val signature: String, - val parameters: List, - val returnType: String, - val statusByBackend: Map, - val notes: List -) - -@Serializable -data class ParameterDoc( - val name: String, - val type: String, - val description: String = "" -) - -@Serializable -data class Note( - val type: String, - val backend: String, - val message: String -) \ No newline at end of file diff --git a/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt b/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt deleted file mode 100644 index 3f026e6c..00000000 --- a/tools/docgen/src/main/kotlin/sk/ainet/tools/docgen/DocGen.kt +++ /dev/null @@ -1,220 +0,0 @@ -package sk.ainet.tools.docgen - -import kotlinx.cli.* -import kotlinx.serialization.json.Json -import java.io.File -import java.time.Instant -import java.time.format.DateTimeFormatter - -/** - * Main documentation generator that converts JSON operator documentation to AsciiDoc format. - * - * Usage: DocGen -i input.json -o output_directory - */ -object DocGen { - - private val json = Json { - ignoreUnknownKeys = true - prettyPrint = true - } - - fun generateDocumentation(inputFile: File, outputDir: File) { - println("Reading JSON from: ${inputFile.absolutePath}") - - val jsonContent = inputFile.readText() - val module = json.decodeFromString(jsonContent) - - println("Parsed module: ${module.module} with ${module.operators.size} operators") - - // Create output directory structure - outputDir.mkdirs() - val generatedDir = File(outputDir, "_generated_") - generatedDir.mkdirs() - - // Generate main index page - generateMainIndex(module, generatedDir) - - // Generate individual operator pages - module.operators.forEach { operator -> - generateOperatorPage(operator, module, generatedDir) - } - - println("Generated documentation in: ${generatedDir.absolutePath}") - } - - private fun generateMainIndex(module: OperatorDocModule, outputDir: File) { - val content = buildString { - appendLine("= ${module.module} Operators") - appendLine() - appendLine("// Generated on ${formatTimestamp(module.timestamp)}") - appendLine("// Version: ${module.version}") - appendLine("// Commit: ${module.commit}") - appendLine() - appendLine("This documentation is automatically generated from the codebase annotations.") - appendLine() - appendLine("== Operators") - appendLine() - - // Group operators by modality - val operatorsByModality = module.operators.groupBy { it.modality } - operatorsByModality.entries.sortedBy { it.key }.forEach { (modality, operators) -> - appendLine("=== ${modality.capitalize()} Operators") - appendLine() - operators.sortedBy { it.name }.forEach { operator -> - appendLine("* xref:${operator.name.lowercase()}.adoc[${operator.name}] - ${operator.packageName}") - } - appendLine() - } - } - - File(outputDir, "index.adoc").writeText(content) - } - - private fun generateOperatorPage(operator: OperatorDoc, module: OperatorDocModule, outputDir: File) { - val content = buildString { - appendLine("= ${operator.name}") - appendLine() - appendLine("// Generated on ${formatTimestamp(module.timestamp)}") - appendLine("// Package: ${operator.packageName}") - appendLine("// Modality: ${operator.modality}") - appendLine() - appendLine("Package: `${operator.packageName}`") - appendLine() - appendLine("Modality: *${operator.modality}*") - appendLine() - - if (operator.functions.isNotEmpty()) { - appendLine("== Functions") - appendLine() - - operator.functions.sortedBy { it.name }.forEach { function -> - generateFunctionSection(function, this) - } - } - } - - File(outputDir, "${operator.name.lowercase()}.adoc").writeText(content) - } - - private fun generateFunctionSection(function: FunctionDoc, builder: StringBuilder) { - builder.apply { - appendLine("=== ${function.name}") - appendLine() - appendLine("[source,kotlin]") - appendLine("----") - appendLine(function.signature) - appendLine("----") - appendLine() - - // Parameters table - if (function.parameters.isNotEmpty()) { - appendLine("==== Parameters") - appendLine() - appendLine("[cols=\"1,2,3\"]") - appendLine("|===") - appendLine("| Name | Type | Description") - appendLine() - function.parameters.forEach { param -> - appendLine("| ${param.name}") - appendLine("| `${param.type}`") - appendLine("| ${param.description.ifEmpty { "_No description_" }}") - appendLine() - } - appendLine("|===") - appendLine() - } - - // Return type - appendLine("==== Returns") - appendLine() - appendLine("`${function.returnType}`") - appendLine() - - // Backend status table - if (function.statusByBackend.isNotEmpty()) { - appendLine("==== Backend Status") - appendLine() - generateBackendStatusTable(function, this) - } - - // API reference link - appendLine("==== See also") - appendLine() - appendLine("* xref:api:${function.name}[API Reference (Dokka)]") - appendLine("* xref:theory:${function.name}.adoc[Mathematical Definition]") - appendLine("* xref:examples:${function.name}.adoc[Usage Examples]") - appendLine() - } - } - - private fun generateBackendStatusTable(function: FunctionDoc, builder: StringBuilder) { - builder.apply { - appendLine("[cols=\"1,1,2\"]") - appendLine("|===") - appendLine("| Backend | Status | Notes") - appendLine() - - function.statusByBackend.entries.sortedBy { it.key }.forEach { (backend, status) -> - appendLine("| ${backend}") - appendLine("| ${formatStatus(status)}") - - val backendNotes = function.notes.filter { it.backend == backend } - if (backendNotes.isNotEmpty()) { - val notesText = backendNotes.joinToString(", ") { note -> - when (note.type) { - "owner" -> "Owner: ${note.message}" - "issue" -> "Issue: ${note.message}" - else -> "${note.type}: ${note.message}" - } - } - appendLine("| ${notesText}") - } else { - appendLine("| _None_") - } - appendLine() - } - appendLine("|===") - appendLine() - } - } - - private fun formatStatus(status: String): String { - return when (status) { - "implemented" -> "āœ… Implemented" - "not_implemented" -> "āŒ Not Implemented" - "in_progress" -> "šŸš§ In Progress" - else -> status - } - } - - private fun formatTimestamp(timestamp: String): String { - return try { - val instant = Instant.parse(timestamp) - DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(instant.atZone(java.time.ZoneId.systemDefault())) - } catch (e: Exception) { - timestamp - } - } - - private fun String.capitalize(): String { - return this.replaceFirstChar { if (it.isLowerCase()) it.titlecase() else it.toString() } - } -} - -fun main(args: Array) { - val parser = ArgParser("docgen") - val input by parser.option(ArgType.String, shortName = "i", description = "Input JSON file").required() - val output by parser.option(ArgType.String, shortName = "o", description = "Output directory").required() - - parser.parse(args) - - val inputFile = File(input) - val outputDir = File(output) - - if (!inputFile.exists()) { - println("Error: Input file does not exist: $input") - return - } - - DocGen.generateDocumentation(inputFile, outputDir) -} \ No newline at end of file diff --git a/tools/docgen/src/test/resources/test-operators.json b/tools/docgen/src/test/resources/test-operators.json deleted file mode 100644 index d35bcfd8..00000000 --- a/tools/docgen/src/test/resources/test-operators.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "schema": "https://skainet.ai/schemas/operator-doc/v1", - "version": "1.0.0", - "commit": "abc123def", - "timestamp": "2025-10-22T23:30:00Z", - "module": "skainet-lang-core", - "operators": [ - { - "name": "TensorOps", - "packageName": "sk.ainet.lang.tensor.ops", - "modality": "core", - "functions": [ - { - "name": "matmul", - "signature": "fun matmul(a: Tensor, b: Tensor): Tensor", - "parameters": [ - { - "name": "a", - "type": "Tensor", - "description": "First tensor for matrix multiplication" - }, - { - "name": "b", - "type": "Tensor", - "description": "Second tensor for matrix multiplication" - } - ], - "returnType": "Tensor", - "statusByBackend": { - "cpu": "implemented", - "gpu": "in_progress", - "tpu": "not_implemented" - }, - "notes": [ - { - "type": "owner", - "backend": "gpu", - "content": "john.doe" - }, - { - "type": "issue", - "backend": "tpu", - "content": "#123" - } - ] - }, - { - "name": "add", - "signature": "fun add(a: Tensor, b: Tensor): Tensor", - "parameters": [ - { - "name": "a", - "type": "Tensor", - "description": "First tensor" - }, - { - "name": "b", - "type": "Tensor", - "description": "Second tensor" - } - ], - "returnType": "Tensor", - "statusByBackend": { - "cpu": "implemented", - "gpu": "implemented", - "tpu": "implemented" - }, - "notes": [] - } - ] - } - ] -} \ No newline at end of file From 4e34c79569cc38340c010489c48f8de26e2947b9 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Sun, 26 Oct 2025 15:32:16 +0100 Subject: [PATCH 10/11] Move json validation to gradle plugin too and make it running. Related-To #139 --- .github/workflows/schema-validation.yml | 8 +- README.md | 33 ++++ buildSrc/build.gradle.kts | 3 + .../src/main/kotlin/DocumentationPlugin.kt | 13 ++ .../src/main/kotlin/SchemaValidationTask.kt | 114 ++++++++++++ .../schemas/operator-doc-schema-v1.json | 166 ++++++++++++++++++ .../skainet-lang-export-ops/build.gradle.kts | 20 --- .../tools/export/ops}/SchemaValidationMain.kt | 2 +- .../lang/tools/export/ops}/SchemaValidator.kt | 3 +- .../lang/ops/ksp/OperatorDocProcessor.kt | 54 ++++-- 10 files changed, 375 insertions(+), 41 deletions(-) create mode 100644 buildSrc/src/main/kotlin/SchemaValidationTask.kt create mode 100644 buildSrc/src/main/resources/schemas/operator-doc-schema-v1.json rename skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/{org/mikrograd/diff/ksp => sk/ainet/lang/tools/export/ops}/SchemaValidationMain.kt (98%) rename skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/{org/mikrograd/diff/ksp => sk/ainet/lang/tools/export/ops}/SchemaValidator.kt (98%) diff --git a/.github/workflows/schema-validation.yml b/.github/workflows/schema-validation.yml index 8bfc6bde..66564a4b 100644 --- a/.github/workflows/schema-validation.yml +++ b/.github/workflows/schema-validation.yml @@ -36,11 +36,11 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - - name: Generate operator documentation - run: ./gradlew :skainet-lang:skainet-lang-export-ops:kspKotlinJvm + - name: Generate operator documentation (KSP) + run: ./gradlew :skainet-lang:skainet-lang-core:kspCommonMainKotlinMetadata - - name: Validate JSON schema - run: ./gradlew :skainet-lang:skainet-lang-export-ops:validateOperatorSchema + - name: Validate JSON schema via Gradle plugin + run: ./gradlew validateOperatorSchema - name: Upload validation artifacts if: failure() diff --git a/README.md b/README.md index 59109094..a6320942 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,36 @@ This project follows established development practices for maintaining code qual * **Branching Model**: We use [GitFlow](https://nvie.com/posts/a-successful-git-branching-model/) as our branching strategy for managing feature development, releases, and hotfixes. * **Versioning**: We follow [Semantic Versioning (SemVer)](https://semver.org/) for all releases, ensuring predictable version numbering based on the nature of changes. + +## Reflective Documentation (short overview) + +SKaiNET includes a reflective documentation system that keeps docs in sync with the code. During the build, a KSP processor extracts operator metadata (signatures, parameters, backend availability, implementation status) into a JSON file. A small DocGen tool then converts this JSON into AsciiDoc fragments and pages. + +- Source of truth (generated): skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json +- Generated docs output: docs/modules/operators/_generated_/ +- Asciidoctor site output: build/docs/asciidoc/ (if you run an Asciidoctor task locally) + +### Quick start: generate reflective docs + +Use any of the following Gradle tasks from the project root: + +1) Full pipeline (recommended) + ./gradlew generateDocs + - Runs KSP to produce operators.json (if needed) + - Generates AsciiDoc files under docs/modules/operators/_generated_ + - Optionally, you can run an Asciidoctor task to build an HTML site locally (output under build/docs/asciidoc) + +2) Operators documentation only + ./gradlew generateOperatorDocs + - Depends on KSP; runs the built-in generateDocs task and then Asciidoctor + +Open the generated AsciiDoc sources in docs/modules/operators/_generated_ with your preferred AsciiDoc viewer. If you build an HTML site locally with Asciidoctor, open build/docs/asciidoc. + +--- + +## Development Practices + +This project follows established development practices for maintaining code quality and release management: + +* Branching Model: We use GitFlow as our branching strategy for managing feature development, releases, and hotfixes. +* Versioning: We follow Semantic Versioning (SemVer) for all releases, ensuring predictable version numbering based on the nature of changes. diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 783603a2..8e6a4926 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -13,6 +13,9 @@ dependencies { implementation(kotlin("stdlib")) implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") implementation("org.asciidoctor:asciidoctorj:3.0.0") + // JSON schema validation dependencies for SchemaValidationTask + implementation("com.networknt:json-schema-validator:1.0.87") + implementation("com.fasterxml.jackson.core:jackson-databind:2.15.2") implementation(gradleApi()) } diff --git a/buildSrc/src/main/kotlin/DocumentationPlugin.kt b/buildSrc/src/main/kotlin/DocumentationPlugin.kt index 1b0ffed1..0a17efa1 100644 --- a/buildSrc/src/main/kotlin/DocumentationPlugin.kt +++ b/buildSrc/src/main/kotlin/DocumentationPlugin.kt @@ -1,6 +1,9 @@ import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.Action +import org.gradle.kotlin.dsl.named +import org.gradle.kotlin.dsl.register +import org.gradle.kotlin.dsl.configureEach class DocumentationPlugin : Plugin { override fun apply(project: Project) { @@ -19,5 +22,15 @@ class DocumentationPlugin : Plugin { task.generateIndex.set(extension.generateIndex) } }) + + // Register schema validation task in plugin (migrated from skainet-lang-export-ops) + val validateTaskProvider = project.tasks.register("validateOperatorSchema", SchemaValidationTask::class.java, object : Action { + override fun execute(task: SchemaValidationTask) { + task.group = "verification" + task.description = "Validate generated operators.json files against the JSON schema" + // By default search from the root project dir to find all operators.json + task.searchDirectory.set(project.rootProject.layout.projectDirectory) + } + }) } } \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/SchemaValidationTask.kt b/buildSrc/src/main/kotlin/SchemaValidationTask.kt new file mode 100644 index 00000000..5debe09a --- /dev/null +++ b/buildSrc/src/main/kotlin/SchemaValidationTask.kt @@ -0,0 +1,114 @@ +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.ObjectNode +import com.networknt.schema.JsonSchemaFactory +import com.networknt.schema.SpecVersion +import org.gradle.api.DefaultTask +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.tasks.* + +@CacheableTask +abstract class SchemaValidationTask : DefaultTask() { + @get:InputDirectory + @get:Optional + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val searchDirectory: DirectoryProperty + + init { + // Default to the root project directory to cover all subprojects by default + searchDirectory.convention(project.rootProject.layout.projectDirectory) + } + + private fun normalizeForSchema(root: JsonNode): JsonNode { + if (root is ObjectNode) { + // Normalize operators array + val operators = root.get("operators") + if (operators is ArrayNode) { + operators.forEach { opNode -> + if (opNode is ObjectNode) { + // Handle legacy "package" -> "packageName" + if (!opNode.has("packageName") && opNode.has("package")) { + opNode.set("packageName", opNode.get("package")) + opNode.remove("package") + } + // Normalize functions -> notes + val functions = opNode.get("functions") + if (functions is ArrayNode) { + functions.forEach { fnNode -> + if (fnNode is ObjectNode) { + val notes = fnNode.get("notes") + if (notes is ArrayNode) { + notes.forEach { noteNode -> + if (noteNode is ObjectNode) { + if (!noteNode.has("content") && noteNode.has("message")) { + noteNode.set("content", noteNode.get("message")) + noteNode.remove("message") + } + } + } + } + } + } + } + } + } + } + } + return root + } + + @TaskAction + fun validate() { + val buildDir = (if (searchDirectory.isPresent) searchDirectory.get() else project.rootProject.layout.projectDirectory).asFile + val schemaStream = this::class.java.classLoader.getResourceAsStream("schemas/operator-doc-schema-v1.json") + ?: throw IllegalStateException("Cannot find schema resource: schemas/operator-doc-schema-v1.json") + val schema = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012).getSchema(schemaStream) + + if (!buildDir.exists()) { + throw RuntimeException("Build directory does not exist: ${buildDir.absolutePath}") + } + + val operatorJsonFiles = buildDir.walkTopDown() + .filter { it.isFile && it.name == "operators.json" } + .toList() + + if (operatorJsonFiles.isEmpty()) { + logger.lifecycle("No operators.json files found under: ${buildDir.absolutePath}. Skipping schema validation.") + return + } + + var total = 0 + var valid = 0 + val errors = mutableListOf() + + // Create ObjectMapper locally to keep task configuration-cache friendly + val objectMapper = ObjectMapper() + + operatorJsonFiles.forEach { file -> + total++ + val original: JsonNode = objectMapper.readTree(file) + val node = normalizeForSchema(original) + val validationErrors = schema.validate(node) + if (validationErrors.isEmpty()) { + valid++ + logger.lifecycle("āœ“ VALID: ${file.relativeTo(buildDir)}") + } else { + logger.error("āœ— INVALID: ${file.relativeTo(buildDir)}") + validationErrors.forEach { err -> + val msg = " - ${err.path}: ${err.message}" + errors.add("${file.relativeTo(buildDir)}: ${err.message}") + logger.error(msg) + } + } + } + + logger.lifecycle("============================================") + logger.lifecycle("Schema Validation Summary") + logger.lifecycle("Total files: $total Valid: $valid Invalid: ${total - valid}") + + if (errors.isNotEmpty()) { + throw RuntimeException("Schema validation failed for ${errors.size} issue(s). See log for details.") + } + } +} diff --git a/buildSrc/src/main/resources/schemas/operator-doc-schema-v1.json b/buildSrc/src/main/resources/schemas/operator-doc-schema-v1.json new file mode 100644 index 00000000..34a7db39 --- /dev/null +++ b/buildSrc/src/main/resources/schemas/operator-doc-schema-v1.json @@ -0,0 +1,166 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://skainet.ai/schemas/operator-doc/v1", + "title": "SKaiNET Operator Documentation Schema", + "description": "JSON schema for SKaiNET operator documentation generated by KSP processor", + "type": "object", + "properties": { + "schema": { + "type": "string", + "format": "uri", + "description": "Schema URI identifier", + "const": "https://skainet.ai/schemas/operator-doc/v1" + }, + "version": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?$", + "description": "Semantic version of the framework" + }, + "commit": { + "type": "string", + "pattern": "^[a-f0-9]{7,40}$|^unknown$", + "description": "Git commit SHA or 'unknown'" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "ISO 8601 timestamp when documentation was generated" + }, + "module": { + "type": "string", + "minLength": 1, + "description": "Name of the module containing the operators" + }, + "operators": { + "type": "array", + "items": { + "$ref": "#/$defs/OperatorDoc" + }, + "description": "Array of operator documentation objects" + } + }, + "required": ["schema", "version", "commit", "timestamp", "module", "operators"], + "additionalProperties": false, + "$defs": { + "OperatorDoc": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the operator class" + }, + "packageName": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*(\\.[a-z][a-z0-9_]*)*$", + "description": "Fully qualified package name" + }, + "modality": { + "type": "string", + "enum": ["core", "vision", "nlp", "audio"], + "description": "Modality category of the operator" + }, + "functions": { + "type": "array", + "items": { + "$ref": "#/$defs/FunctionDoc" + }, + "description": "Array of function documentation objects" + } + }, + "required": ["name", "packageName", "modality", "functions"], + "additionalProperties": false + }, + "FunctionDoc": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the function" + }, + "signature": { + "type": "string", + "minLength": 1, + "description": "Full function signature string" + }, + "parameters": { + "type": "array", + "items": { + "$ref": "#/$defs/ParameterDoc" + }, + "description": "Array of parameter documentation objects" + }, + "returnType": { + "type": "string", + "minLength": 1, + "description": "Return type of the function" + }, + "statusByBackend": { + "type": "object", + "patternProperties": { + "^[a-zA-Z][a-zA-Z0-9_]*$": { + "type": "string", + "enum": ["implemented", "not_implemented", "in_progress"], + "description": "Implementation status for this backend" + } + }, + "additionalProperties": false, + "description": "Map of backend names to implementation status" + }, + "notes": { + "type": "array", + "items": { + "$ref": "#/$defs/Note" + }, + "description": "Array of notes associated with the function" + } + }, + "required": ["name", "signature", "parameters", "returnType", "statusByBackend", "notes"], + "additionalProperties": false + }, + "ParameterDoc": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the parameter" + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the parameter" + }, + "description": { + "type": "string", + "description": "Optional description of the parameter" + } + }, + "required": ["name", "type"], + "additionalProperties": false + }, + "Note": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["owner", "issue"], + "description": "Type of the note" + }, + "backend": { + "type": "string", + "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", + "description": "Backend this note applies to" + }, + "content": { + "type": "string", + "minLength": 1, + "description": "Content of the note" + } + }, + "required": ["type", "backend", "content"], + "additionalProperties": false + } + } +} diff --git a/skainet-lang/skainet-lang-export-ops/build.gradle.kts b/skainet-lang/skainet-lang-export-ops/build.gradle.kts index c0e21890..c8f8bb63 100644 --- a/skainet-lang/skainet-lang-export-ops/build.gradle.kts +++ b/skainet-lang/skainet-lang-export-ops/build.gradle.kts @@ -4,9 +4,6 @@ plugins { alias(libs.plugins.ksp) } - -group = "org.mikrograd.samples" - kotlin { compilerOptions { @@ -74,20 +71,3 @@ tasks.register("runKspMain") { mainClass.set("com.example.KspMainKt") } -// Add schema validation task -tasks.register("validateOperatorSchema") { - group = "verification" - description = "Validate generated operator.json files against the JSON schema" - classpath = files(kotlin.jvm().compilations["main"].output.allOutputs, configurations.getByName("jvmRuntimeClasspath")) - mainClass.set("org.mikrograd.diff.ksp.SchemaValidationMainKt") - - // Set build directory as argument - args(project.buildDir.absolutePath) - - // Depend on KSP compilation to ensure JSON files are generated first - dependsOn("kspKotlinJvm") - - doFirst { - println("Validating operator documentation JSON schema...") - } -} diff --git a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidationMain.kt b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidationMain.kt similarity index 98% rename from skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidationMain.kt rename to skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidationMain.kt index 1e7f2b09..e91905f5 100644 --- a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidationMain.kt +++ b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidationMain.kt @@ -1,4 +1,4 @@ -package org.mikrograd.diff.ksp +package sk.ainet.lang.tools.export.ops import java.io.File import kotlin.system.exitProcess diff --git a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidator.kt b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidator.kt similarity index 98% rename from skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidator.kt rename to skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidator.kt index a0db90bf..c563cad2 100644 --- a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/org/mikrograd/diff/ksp/SchemaValidator.kt +++ b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidator.kt @@ -1,11 +1,10 @@ -package org.mikrograd.diff.ksp +package sk.ainet.lang.tools.export.ops import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import com.networknt.schema.JsonSchema import com.networknt.schema.JsonSchemaFactory import com.networknt.schema.SpecVersion -import com.networknt.schema.ValidationMessage import java.io.File import java.io.InputStream diff --git a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt index 6c517a7f..04f750a6 100644 --- a/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt +++ b/skainet-lang/skainet-lang-ksp-processor/src/main/kotlin/sk/ainet/lang/ops/ksp/OperatorDocProcessor.kt @@ -236,36 +236,62 @@ class OperatorDocProcessor( return "unknown" } + private fun escapeJson(value: String): String = buildString { + value.forEach { ch -> + when (ch) { + '\\' -> append("\\\\") + '"' -> append("\\\"") + '\b' -> append("\\b") + '\u000C' -> append("\\f") // form feed + '\n' -> append("\\n") + '\r' -> append("\\r") + '\t' -> append("\\t") + else -> { + if (ch < ' ') { + append("\\u") + append(ch.code.toString(16).padStart(4, '0')) + } else append(ch) + } + } + } + } + private fun generateJsonOutput(module: OperatorDocModule) { try { // Simple JSON generation without external dependencies val jsonContent = buildString { append("{\n") - append(" \"schema\": \"${module.schema}\",\n") - append(" \"version\": \"${module.version}\",\n") - append(" \"commit\": \"${module.commit}\",\n") - append(" \"timestamp\": \"${module.timestamp}\",\n") - append(" \"module\": \"${module.module}\",\n") + append(" \"schema\": \"${escapeJson(module.schema)}\",\n") + append(" \"version\": \"${escapeJson(module.version)}\",\n") + append(" \"commit\": \"${escapeJson(module.commit)}\",\n") + append(" \"timestamp\": \"${escapeJson(module.timestamp)}\",\n") + append(" \"module\": \"${escapeJson(module.module)}\",\n") append(" \"operators\": [\n") module.operators.forEachIndexed { opIndex, operator -> append(" {\n") - append(" \"name\": \"${operator.name}\",\n") - append(" \"package\": \"${operator.packageName}\",\n") - append(" \"modality\": \"${operator.modality}\",\n") + append(" \"name\": \"${escapeJson(operator.name)}\",\n") + append(" \"packageName\": \"${escapeJson(operator.packageName)}\",\n") + append(" \"modality\": \"${escapeJson(operator.modality)}\",\n") append(" \"functions\": [\n") operator.functions.forEachIndexed { funcIndex, function -> append(" {\n") - append(" \"name\": \"${function.name}\",\n") - append(" \"signature\": \"${function.signature}\",\n") - append(" \"parameters\": [],\n") // Simplified for now - append(" \"returnType\": \"${function.returnType}\",\n") + append(" \"name\": \"${escapeJson(function.name)}\",\n") + append(" \"signature\": \"${escapeJson(function.signature)}\",\n") + // parameters + append(" \"parameters\": [") + function.parameters.forEachIndexed { pIndex, p -> + append("{\"name\": \"${escapeJson(p.name)}\", \"type\": \"${escapeJson(p.type)}\", \"description\": \"${escapeJson(p.description)}\"}") + if (pIndex < function.parameters.size - 1) append(", ") + } + append("],\n") + append(" \"returnType\": \"${escapeJson(function.returnType)}\",\n") // Generate statusByBackend JSON append(" \"statusByBackend\": {") function.statusByBackend.entries.forEachIndexed { statusIndex, (backend, status) -> - append("\"$backend\": \"$status\"") + append("\"${escapeJson(backend)}\": \"${escapeJson(status)}\"") if (statusIndex < function.statusByBackend.size - 1) append(", ") } append("},\n") @@ -273,7 +299,7 @@ class OperatorDocProcessor( // Generate notes JSON append(" \"notes\": [") function.notes.forEachIndexed { noteIndex, note -> - append("{\"type\": \"${note.type}\", \"backend\": \"${note.backend}\", \"message\": \"${note.content}\"}") + append("{\"type\": \"${escapeJson(note.type)}\", \"backend\": \"${escapeJson(note.backend)}\", \"content\": \"${escapeJson(note.content)}\"}") if (noteIndex < function.notes.size - 1) append(", ") } append("]\n") From 549655afede1aa198e24424776ed6f7f32b7dcf1 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Sun, 26 Oct 2025 16:17:11 +0100 Subject: [PATCH 11/11] Remove "skainet-leng-export-ops" because the logic belongs to gradle plugin now Related-To #139 --- README.md | 10 ++ .../main/kotlin/GenerateDocumentationTask.kt | 3 +- .../main/kotlin/models/DocumentationModels.kt | 18 +- .../_generated_/_generated_/index.adoc | 14 -- .../_generated_/voidtensorops.adoc | 70 -------- docs/modules/operators/_generated_/index.adoc | 35 ---- .../_generated_plugin_test/index.adoc | 10 -- .../_generated_plugin_test/voidtensorops.adoc | 62 ------- .../skainet-lang-export-ops/build.gradle.kts | 73 -------- .../schemas/operator-doc-schema-v1.json | 166 ------------------ .../tools/export/ops/SchemaValidationMain.kt | 67 ------- .../lang/tools/export/ops/SchemaValidator.kt | 155 ---------------- 12 files changed, 22 insertions(+), 661 deletions(-) delete mode 100644 docs/modules/operators/_generated_/_generated_/index.adoc delete mode 100644 docs/modules/operators/_generated_/_generated_/voidtensorops.adoc delete mode 100644 docs/modules/operators/_generated_/index.adoc delete mode 100644 docs/modules/operators/_generated_plugin_test/index.adoc delete mode 100644 docs/modules/operators/_generated_plugin_test/voidtensorops.adoc delete mode 100644 skainet-lang/skainet-lang-export-ops/build.gradle.kts delete mode 100644 skainet-lang/skainet-lang-export-ops/src/commonMain/resources/schemas/operator-doc-schema-v1.json delete mode 100644 skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidationMain.kt delete mode 100644 skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidator.kt diff --git a/README.md b/README.md index a6320942..b271ca4a 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,16 @@ Use any of the following Gradle tasks from the project root: Open the generated AsciiDoc sources in docs/modules/operators/_generated_ with your preferred AsciiDoc viewer. If you build an HTML site locally with Asciidoctor, open build/docs/asciidoc. +### Documentation tooling + +We now use the Gradle plugin (buildSrc) only. The former skainet-lang-export-ops module has been removed. All everyday workflows are covered by: +- generateDocs ā€” converts KSP JSON to AsciiDoc +- validateOperatorSchema ā€” validates generated operators.json against the JSON schema + +Run from the project root, for example: +- ./gradlew generateDocs +- ./gradlew validateOperatorSchema + --- ## Development Practices diff --git a/buildSrc/src/main/kotlin/GenerateDocumentationTask.kt b/buildSrc/src/main/kotlin/GenerateDocumentationTask.kt index 85365871..0d29fcfe 100644 --- a/buildSrc/src/main/kotlin/GenerateDocumentationTask.kt +++ b/buildSrc/src/main/kotlin/GenerateDocumentationTask.kt @@ -44,7 +44,8 @@ abstract class GenerateDocumentationTask : DefaultTask() { logger.lifecycle("šŸ“‚ Output directory: ${output.absolutePath}") val jsonContent = input.readText() - val module = Json.decodeFromString(jsonContent) + val json = Json { ignoreUnknownKeys = true } + val module = json.decodeFromString(jsonContent) when (format.get()) { DocumentationFormat.ASCIIDOC -> generateAsciidoc(module, output) diff --git a/buildSrc/src/main/kotlin/models/DocumentationModels.kt b/buildSrc/src/main/kotlin/models/DocumentationModels.kt index ba7698c3..fd176a33 100644 --- a/buildSrc/src/main/kotlin/models/DocumentationModels.kt +++ b/buildSrc/src/main/kotlin/models/DocumentationModels.kt @@ -15,7 +15,9 @@ data class OperatorDocModule( @Serializable data class OperatorDoc( val name: String, - @kotlinx.serialization.SerialName("package") val packageName: String, + @kotlinx.serialization.SerialName("package") + @kotlinx.serialization.json.JsonNames("packageName") + val packageName: String, val modality: String, val functions: List ) @@ -24,10 +26,10 @@ data class OperatorDoc( data class FunctionDoc( val name: String, val signature: String, - val parameters: List, + val parameters: List = emptyList(), val returnType: String, - val statusByBackend: Map, - val notes: List + val statusByBackend: Map = emptyMap(), + val notes: List = emptyList() ) @Serializable @@ -38,10 +40,10 @@ data class ParameterDoc( ) @Serializable -data class Note( - val type: String, - val backend: String, - val message: String + data class Note( + val type: String = "", + val backend: String = "", + val message: String = "" ) enum class DocumentationFormat { diff --git a/docs/modules/operators/_generated_/_generated_/index.adoc b/docs/modules/operators/_generated_/_generated_/index.adoc deleted file mode 100644 index 11e245e4..00000000 --- a/docs/modules/operators/_generated_/_generated_/index.adoc +++ /dev/null @@ -1,14 +0,0 @@ -= skainet-lang-core Operators - -// Generated on 2025-10-23T12:05:07.383567 -// Version: 1.0.0 -// Commit: unknown - -This documentation is automatically generated from the codebase annotations. - -== Operators - -=== Core Operators - -* xref:voidtensorops.adoc[VoidTensorOps] - sk.ainet.lang.tensor.ops - diff --git a/docs/modules/operators/_generated_/_generated_/voidtensorops.adoc b/docs/modules/operators/_generated_/_generated_/voidtensorops.adoc deleted file mode 100644 index 3b834d32..00000000 --- a/docs/modules/operators/_generated_/_generated_/voidtensorops.adoc +++ /dev/null @@ -1,70 +0,0 @@ -= VoidTensorOps - -// Generated on 2025-10-23T12:05:07.383567 -// Package: sk.ainet.lang.tensor.ops -// Modality: core - -Package: `sk.ainet.lang.tensor.ops` - -Modality: *core* - -== Functions - -=== matmul - -[source,kotlin] ----- -fun matmul(a:Tensor, b:Tensor): Tensor ----- - -==== Returns - -`Tensor` - -==== Backend Status - -[cols="1,1,2"] -|=== -| Backend | Status | Notes - -| Metal -| šŸš§ In Progress -| Owner: ops-team, Issue: GH-1234 - -|=== - -==== See also - -* xref:api:matmul[API Reference (Dokka)] -* xref:theory:matmul.adoc[Mathematical Definition] -* xref:examples:matmul.adoc[Usage Examples] - -=== transpose - -[source,kotlin] ----- -fun transpose(tensor:Tensor): Tensor ----- - -==== Returns - -`Tensor` - -==== Backend Status - -[cols="1,1,2"] -|=== -| Backend | Status | Notes - -| Metal -| šŸš§ In Progress -| Owner: ops-team, Issue: GH-1234 - -|=== - -==== See also - -* xref:api:transpose[API Reference (Dokka)] -* xref:theory:transpose.adoc[Mathematical Definition] -* xref:examples:transpose.adoc[Usage Examples] - diff --git a/docs/modules/operators/_generated_/index.adoc b/docs/modules/operators/_generated_/index.adoc deleted file mode 100644 index 9fc3b629..00000000 --- a/docs/modules/operators/_generated_/index.adoc +++ /dev/null @@ -1,35 +0,0 @@ -= Generated Operator Reference - -This section contains automatically generated API reference documentation for SKaiNET operators. - -[NOTE] -==== -This content is automatically generated from source code annotations and should not be edited manually. -Generated on: {docdate} -==== - -[#operator-index] -== Operator Index - -// Generated content will be inserted here by the documentation generation system -// The DocGen tool will populate this with operator listings from operators.json - -[#cross-references] -== Cross-References to Human-Authored Content - -=== Theory References -* xref:../../theory/index.adoc[Mathematical Theory Reference] -* xref:../../theory/matmul.adoc#matmul-definition[Matrix Multiplication Theory] - -=== Usage Examples -* xref:../../examples/index.adoc[Usage Examples] -* xref:../../examples/matmul-examples.adoc#basic-usage[Matrix Multiplication Examples] - -[#anchors] -== Anchor Points for Cross-Linking - -The following anchors are available for cross-referencing from generated content: - -* `#operator-index` - Main operator index -* Individual operator anchors will be generated with pattern: `#operator-{operatorName}` -* Function anchors will follow pattern: `#function-{operatorName}-{functionName}` \ No newline at end of file diff --git a/docs/modules/operators/_generated_plugin_test/index.adoc b/docs/modules/operators/_generated_plugin_test/index.adoc deleted file mode 100644 index f2eb2c9a..00000000 --- a/docs/modules/operators/_generated_plugin_test/index.adoc +++ /dev/null @@ -1,10 +0,0 @@ -= AI-NET Operators Reference - -Generated from version `1.0.0` on 2025-10-23 - -== Operators by Modality - -=== Core - -* xref:voidtensorops.adoc[VoidTensorOps] - diff --git a/docs/modules/operators/_generated_plugin_test/voidtensorops.adoc b/docs/modules/operators/_generated_plugin_test/voidtensorops.adoc deleted file mode 100644 index 2defe80c..00000000 --- a/docs/modules/operators/_generated_plugin_test/voidtensorops.adoc +++ /dev/null @@ -1,62 +0,0 @@ -= VoidTensorOps - -Package: `sk.ainet.lang.tensor.ops` - -Modality: Core - -== matmul - -=== Signature - -[source,kotlin] ----- -fun matmul(a:Tensor, b:Tensor): Tensor ----- - -=== Return Type - -`Tensor` - -=== Backend Support - -[cols="1,1,3", options="header"] -|=== -| Backend | Status | Notes -| Metal | in_progress | ops-team; GH-1234 -|=== - -=== Notes - -TIP: *Metal*: ops-team - -TIP: *Metal*: GH-1234 - - -== transpose - -=== Signature - -[source,kotlin] ----- -fun transpose(tensor:Tensor): Tensor ----- - -=== Return Type - -`Tensor` - -=== Backend Support - -[cols="1,1,3", options="header"] -|=== -| Backend | Status | Notes -| Metal | in_progress | ops-team; GH-1234 -|=== - -=== Notes - -TIP: *Metal*: ops-team - -TIP: *Metal*: GH-1234 - - diff --git a/skainet-lang/skainet-lang-export-ops/build.gradle.kts b/skainet-lang/skainet-lang-export-ops/build.gradle.kts deleted file mode 100644 index c8f8bb63..00000000 --- a/skainet-lang/skainet-lang-export-ops/build.gradle.kts +++ /dev/null @@ -1,73 +0,0 @@ -plugins { - alias(libs.plugins.kotlinMultiplatform) - alias(libs.plugins.kotlinSerialization) - alias(libs.plugins.ksp) -} - -kotlin { - - compilerOptions { - // Common compiler options applied to all Kotlin source sets - freeCompilerArgs.add("-Xexpect-actual-classes") - freeCompilerArgs.add("-Xmulti-platform") - } - - jvmToolchain(17) - - jvm() - - - sourceSets { - commonMain.dependencies { - implementation(project(":skainet-lang:skainet-lang-core")) - implementation(libs.kotlinx.serialization.json) - - } - - commonTest.dependencies { - implementation(kotlin("test-common")) - implementation(kotlin("test-annotations-common")) - } - - val jvmMain by getting { - kotlin.srcDir("build/generated/ksp/jvm/jvmMain/kotlin") - dependencies { - implementation(project(":skainet-lang:skainet-lang-ksp-annotations")) - implementation("com.networknt:json-schema-validator:1.0.87") - implementation("com.fasterxml.jackson.core:jackson-databind:2.15.2") - } - } - - - - jvmTest.dependencies { - implementation(kotlin("test-junit")) - } - } -} - -dependencies { - // add("kspCommonMainMetadata", project(":test-processor")) - add("kspJvm", project(":skainet-lang:skainet-lang-ksp-processor")) -} - -ksp { - arg("ksp.verbose", "true") -} - -// Add a run task for the JVM application -tasks.register("runJvm") { - group = "application" - description = "Run the JVM application" - classpath = files(kotlin.jvm().compilations["main"].output.allOutputs, configurations.getByName("jvmRuntimeClasspath")) - mainClass.set("com.example.MainKt") -} - -// Add a run task for the KspMain application -tasks.register("runKspMain") { - group = "application" - description = "Run the KspMain application" - classpath = files(kotlin.jvm().compilations["main"].output.allOutputs, configurations.getByName("jvmRuntimeClasspath")) - mainClass.set("com.example.KspMainKt") -} - diff --git a/skainet-lang/skainet-lang-export-ops/src/commonMain/resources/schemas/operator-doc-schema-v1.json b/skainet-lang/skainet-lang-export-ops/src/commonMain/resources/schemas/operator-doc-schema-v1.json deleted file mode 100644 index 4ae3d234..00000000 --- a/skainet-lang/skainet-lang-export-ops/src/commonMain/resources/schemas/operator-doc-schema-v1.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://skainet.ai/schemas/operator-doc/v1", - "title": "SKaiNET Operator Documentation Schema", - "description": "JSON schema for SKaiNET operator documentation generated by KSP processor", - "type": "object", - "properties": { - "schema": { - "type": "string", - "format": "uri", - "description": "Schema URI identifier", - "const": "https://skainet.ai/schemas/operator-doc/v1" - }, - "version": { - "type": "string", - "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9.-]+)?$", - "description": "Semantic version of the framework" - }, - "commit": { - "type": "string", - "pattern": "^[a-f0-9]{7,40}$|^unknown$", - "description": "Git commit SHA or 'unknown'" - }, - "timestamp": { - "type": "string", - "format": "date-time", - "description": "ISO 8601 timestamp when documentation was generated" - }, - "module": { - "type": "string", - "minLength": 1, - "description": "Name of the module containing the operators" - }, - "operators": { - "type": "array", - "items": { - "$ref": "#/$defs/OperatorDoc" - }, - "description": "Array of operator documentation objects" - } - }, - "required": ["schema", "version", "commit", "timestamp", "module", "operators"], - "additionalProperties": false, - "$defs": { - "OperatorDoc": { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the operator class" - }, - "packageName": { - "type": "string", - "pattern": "^[a-z][a-z0-9_]*(\\.[a-z][a-z0-9_]*)*$", - "description": "Fully qualified package name" - }, - "modality": { - "type": "string", - "enum": ["core", "vision", "nlp", "audio"], - "description": "Modality category of the operator" - }, - "functions": { - "type": "array", - "items": { - "$ref": "#/$defs/FunctionDoc" - }, - "description": "Array of function documentation objects" - } - }, - "required": ["name", "packageName", "modality", "functions"], - "additionalProperties": false - }, - "FunctionDoc": { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the function" - }, - "signature": { - "type": "string", - "minLength": 1, - "description": "Full function signature string" - }, - "parameters": { - "type": "array", - "items": { - "$ref": "#/$defs/ParameterDoc" - }, - "description": "Array of parameter documentation objects" - }, - "returnType": { - "type": "string", - "minLength": 1, - "description": "Return type of the function" - }, - "statusByBackend": { - "type": "object", - "patternProperties": { - "^[a-zA-Z][a-zA-Z0-9_]*$": { - "type": "string", - "enum": ["implemented", "not_implemented", "in_progress"], - "description": "Implementation status for this backend" - } - }, - "additionalProperties": false, - "description": "Map of backend names to implementation status" - }, - "notes": { - "type": "array", - "items": { - "$ref": "#/$defs/Note" - }, - "description": "Array of notes associated with the function" - } - }, - "required": ["name", "signature", "parameters", "returnType", "statusByBackend", "notes"], - "additionalProperties": false - }, - "ParameterDoc": { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the parameter" - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the parameter" - }, - "description": { - "type": "string", - "description": "Optional description of the parameter" - } - }, - "required": ["name", "type"], - "additionalProperties": false - }, - "Note": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["owner", "issue"], - "description": "Type of the note" - }, - "backend": { - "type": "string", - "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$", - "description": "Backend this note applies to" - }, - "content": { - "type": "string", - "minLength": 1, - "description": "Content of the note" - } - }, - "required": ["type", "backend", "content"], - "additionalProperties": false - } - } -} \ No newline at end of file diff --git a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidationMain.kt b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidationMain.kt deleted file mode 100644 index e91905f5..00000000 --- a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidationMain.kt +++ /dev/null @@ -1,67 +0,0 @@ -package sk.ainet.lang.tools.export.ops - -import java.io.File -import kotlin.system.exitProcess - -/** - * Main entry point for schema validation task. - * - * This is executed by the Gradle validateOperatorSchema task to validate - * generated operator.json files against the JSON schema. - */ -fun main(args: Array) { - if (args.isEmpty()) { - println("Error: Build directory path required as argument") - exitProcess(1) - } - - val buildDirPath = args[0] - val buildDir = File(buildDirPath) - - println("Starting schema validation for operator documentation...") - println("Build directory: ${buildDir.absolutePath}") - - val validationResults = SchemaValidator.validateBuildOutput(buildDir) - - if (validationResults.isEmpty()) { - println("Warning: No validation results returned") - exitProcess(1) - } - - var hasErrors = false - var totalFiles = 0 - var validFiles = 0 - - for (result in validationResults) { - totalFiles++ - - if (result.result.isValid) { - validFiles++ - println("āœ“ VALID: ${result.file.relativeTo(buildDir)}") - } else { - hasErrors = true - println("āœ— INVALID: ${result.file.relativeTo(buildDir)}") - println(" Errors:") - for (error in result.result.errors) { - println(" - $error") - } - } - } - - println("\n" + "=".repeat(60)) - println("Schema Validation Summary") - println("=".repeat(60)) - println("Total files validated: $totalFiles") - println("Valid files: $validFiles") - println("Invalid files: ${totalFiles - validFiles}") - - if (hasErrors) { - println("\nāŒ Schema validation FAILED") - println("Please fix the validation errors above and run again.") - exitProcess(1) - } else { - println("\nāœ… All operator documentation files are valid!") - println("Schema validation PASSED") - exitProcess(0) - } -} \ No newline at end of file diff --git a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidator.kt b/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidator.kt deleted file mode 100644 index c563cad2..00000000 --- a/skainet-lang/skainet-lang-export-ops/src/jvmMain/kotlin/sk/ainet/lang/tools/export/ops/SchemaValidator.kt +++ /dev/null @@ -1,155 +0,0 @@ -package sk.ainet.lang.tools.export.ops - -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.ObjectMapper -import com.networknt.schema.JsonSchema -import com.networknt.schema.JsonSchemaFactory -import com.networknt.schema.SpecVersion -import java.io.File -import java.io.InputStream - -/** - * Utility class for validating operator documentation JSON against the JSON schema. - */ -object SchemaValidator { - - private val objectMapper = ObjectMapper() - private val schemaFactory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012) - - /** - * Validates a JSON file against the operator documentation schema. - * - * @param jsonFile The JSON file to validate - * @return ValidationResult containing success status and any errors - */ - fun validateFile(jsonFile: File): ValidationResult { - return try { - if (!jsonFile.exists()) { - return ValidationResult(false, listOf("File does not exist: ${jsonFile.absolutePath}")) - } - - val jsonNode = objectMapper.readTree(jsonFile) - validate(jsonNode) - } catch (e: Exception) { - ValidationResult(false, listOf("Error reading JSON file: ${e.message}")) - } - } - - /** - * Validates a JSON string against the operator documentation schema. - * - * @param jsonContent The JSON content as a string - * @return ValidationResult containing success status and any errors - */ - fun validateContent(jsonContent: String): ValidationResult { - return try { - val jsonNode = objectMapper.readTree(jsonContent) - validate(jsonNode) - } catch (e: Exception) { - ValidationResult(false, listOf("Error parsing JSON content: ${e.message}")) - } - } - - /** - * Validates a JsonNode against the operator documentation schema. - * - * @param jsonNode The JsonNode to validate - * @return ValidationResult containing success status and any errors - */ - private fun validate(jsonNode: JsonNode): ValidationResult { - return try { - val schema = loadSchema() - val errors = schema.validate(jsonNode) - - if (errors.isEmpty()) { - ValidationResult(true, emptyList()) - } else { - val errorMessages = errors.map { error -> - "${error.path}: ${error.message}" - } - ValidationResult(false, errorMessages) - } - } catch (e: Exception) { - ValidationResult(false, listOf("Schema validation error: ${e.message}")) - } - } - - /** - * Loads the JSON schema from resources. - * - * @return JsonSchema instance - */ - private fun loadSchema(): JsonSchema { - val schemaStream = getSchemaStream() - ?: throw IllegalStateException("Cannot find schema resource: schemas/operator-doc-schema-v1.json") - - return schemaFactory.getSchema(schemaStream) - } - - /** - * Gets the schema file as an InputStream from resources. - * - * @return InputStream for the schema file or null if not found - */ - private fun getSchemaStream(): InputStream? { - return this::class.java.classLoader.getResourceAsStream("schemas/operator-doc-schema-v1.json") - } - - /** - * Validates all operator.json files in the given directory recursively. - * - * @param buildDir The build directory to search for operator.json files - * @return List of ValidationResult for each file found - */ - fun validateBuildOutput(buildDir: File): List { - val results = mutableListOf() - - if (!buildDir.exists()) { - return listOf(FileValidationResult(buildDir, ValidationResult(false, listOf("Build directory does not exist")))) - } - - val operatorJsonFiles = buildDir.walkTopDown() - .filter { it.isFile && it.name == "operators.json" } - .toList() - - if (operatorJsonFiles.isEmpty()) { - return listOf(FileValidationResult(buildDir, ValidationResult(false, listOf("No operators.json files found in build directory")))) - } - - for (file in operatorJsonFiles) { - val result = validateFile(file) - results.add(FileValidationResult(file, result)) - } - - return results - } -} - -/** - * Result of JSON schema validation. - * - * @param isValid Whether the validation passed - * @param errors List of validation error messages - */ -data class ValidationResult( - val isValid: Boolean, - val errors: List -) { - /** - * Returns a formatted string of all errors. - */ - fun getErrorsAsString(): String { - return errors.joinToString("\n") - } -} - -/** - * Result of validating a specific file. - * - * @param file The file that was validated - * @param result The validation result - */ -data class FileValidationResult( - val file: File, - val result: ValidationResult -) \ No newline at end of file