Reflective docs for Ops

.github/workflows/documentation.yml

@@ -0,0 +1,142 @@
+name: Documentation Build and Preview
+
+on:
+  push:
+    branches: [ main, develop ]
+    paths:
+      - 'skainet-lang/**'
+      - 'tools/docgen/**'
+      - 'docs/**'
+      - '.github/workflows/documentation.yml'
+  pull_request:
+    branches: [ main, develop ]
+    paths:
+      - 'skainet-lang/**'
+      - 'tools/docgen/**'
+      - 'docs/**'
+      - '.github/workflows/documentation.yml'
+
+jobs:
+  build-documentation:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      - name: Cache Gradle packages
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+
+      - name: Grant execute permission for gradlew
+        run: chmod +x gradlew
+
+      - name: Copy CI gradle.properties
+        run: mkdir -p ~/.gradle ; cp .github/ci-gradle.properties ~/.gradle/gradle.properties
+
+      - name: Generate operator documentation
+        run: ./gradlew generateDocs --stacktrace
+
+      - name: Upload generated documentation
+        uses: actions/upload-artifact@v4
+        with:
+          name: operator-documentation
+          path: |
+            docs/modules/operators/_generated_/**
+            skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json
+          retention-days: 30
+
+      - name: Upload documentation preview (PR only)
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@v4
+        with:
+          name: documentation-preview-${{ github.event.number }}
+          path: |
+            docs/**
+            tools/docgen/build/docs/asciidoc/**
+          retention-days: 7
+
+  # Job for documentation preview generation on PRs
+  preview-documentation:
+    if: github.event_name == 'pull_request'
+    needs: build-documentation
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Download documentation artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: documentation-preview-${{ github.event.number }}
+          path: ./docs-preview
+
+      - name: Setup Node.js for preview server
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+
+      - name: Install serve package
+        run: npm install -g serve
+
+      - name: Start preview server
+        run: |
+          cd docs-preview
+          serve -s . -l 3000 &
+          sleep 5
+          echo "Preview server started at http://localhost:3000"
+
+      - name: Create PR comment with preview link
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const botComment = comments.find(comment => 
+              comment.user.type === 'Bot' && 
+              comment.body.includes('📖 Documentation Preview')
+            );
+
+            const commentBody = `📖 **Documentation Preview**
+
+            The documentation has been built successfully for this PR.
+
+            **Generated Files:**
+            - Operator documentation: \`docs/modules/operators/_generated_/\`
+            - JSON schema output: \`operators.json\`
+
+            **Artifacts:** 
+            - Download the \`documentation-preview-${{ github.event.number }}\` artifact to view the complete documentation locally.
+
+            _This comment will be updated automatically when the PR is updated._`;
+
+            if (botComment) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: commentBody
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: commentBody
+              });
+            }

.github/workflows/schema-validation.yml

@@ -0,0 +1,53 @@
+name: Schema Validation
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+    paths:
+      - 'skainet-lang/**'
+      - '.github/workflows/schema-validation.yml'
+
+jobs:
+  validate-schema:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up JDK 17
+      uses: actions/setup-java@v4
+      with:
+        java-version: '17'
+        distribution: 'temurin'
+
+    - name: Cache Gradle packages
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+        restore-keys: |
+          ${{ runner.os }}-gradle-
+
+    - name: Grant execute permission for gradlew
+      run: chmod +x gradlew
+
+    - name: Generate operator documentation (KSP)
+      run: ./gradlew :skainet-lang:skainet-lang-core:kspCommonMainKotlinMetadata
+
+    - name: Validate JSON schema via Gradle plugin
+      run: ./gradlew validateOperatorSchema
+
+    - name: Upload validation artifacts
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: validation-logs
+        path: |
+          **/build/generated/**/*.json
+          **/build/logs/
+        retention-days: 7

.gitignore

@@ -21,3 +21,15 @@ out/
 
 ### Mac OS ###
 .DS_Store
+
+.java-version
+
+### BROKK'S CONFIGURATION ###
+.brokk/**
+/.brokk/workspace.properties
+/.brokk/sessions/
+/.brokk/dependencies/
+/.brokk/history.zip
+!.brokk/style.md
+!.brokk/review.md
+!.brokk/project.properties

README.md

@@ -11,3 +11,46 @@ This project follows established development practices for maintaining code qual
 
 * **Branching Model**: We use [GitFlow](https://nvie.com/posts/a-successful-git-branching-model/) as our branching strategy for managing feature development, releases, and hotfixes.
 * **Versioning**: We follow [Semantic Versioning (SemVer)](https://semver.org/) for all releases, ensuring predictable version numbering based on the nature of changes.
+
+## Reflective Documentation (short overview)
+
+SKaiNET includes a reflective documentation system that keeps docs in sync with the code. During the build, a KSP processor extracts operator metadata (signatures, parameters, backend availability, implementation status) into a JSON file. A small DocGen tool then converts this JSON into AsciiDoc fragments and pages.
+
+- Source of truth (generated): skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json
+- Generated docs output: docs/modules/operators/_generated_/
+- Asciidoctor site output: build/docs/asciidoc/ (if you run an Asciidoctor task locally)
+
+### Quick start: generate reflective docs
+
+Use any of the following Gradle tasks from the project root:
+
+1) Full pipeline (recommended)
+   ./gradlew generateDocs
+   - Runs KSP to produce operators.json (if needed)
+   - Generates AsciiDoc files under docs/modules/operators/_generated_
+   - Optionally, you can run an Asciidoctor task to build an HTML site locally (output under build/docs/asciidoc)
+
+2) Operators documentation only
+   ./gradlew generateOperatorDocs
+   - Depends on KSP; runs the built-in generateDocs task and then Asciidoctor
+
+Open the generated AsciiDoc sources in docs/modules/operators/_generated_ with your preferred AsciiDoc viewer. If you build an HTML site locally with Asciidoctor, open build/docs/asciidoc.
+
+### Documentation tooling
+
+We now use the Gradle plugin (buildSrc) only. The former skainet-lang-export-ops module has been removed. All everyday workflows are covered by:
+- generateDocs — converts KSP JSON to AsciiDoc
+- validateOperatorSchema — validates generated operators.json against the JSON schema
+
+Run from the project root, for example:
+- ./gradlew generateDocs
+- ./gradlew validateOperatorSchema
+
+---
+
+## Development Practices
+
+This project follows established development practices for maintaining code quality and release management:
+
+* Branching Model: We use GitFlow as our branching strategy for managing feature development, releases, and hotfixes.
+* Versioning: We follow Semantic Versioning (SemVer) for all releases, ensuring predictable version numbering based on the nature of changes.

build.gradle.kts

@@ -5,6 +5,10 @@ plugins {
     alias(libs.plugins.vanniktech.mavenPublish) apply false
     alias(libs.plugins.kover)
     alias(libs.plugins.binary.compatibility.validator) apply false
+    alias(libs.plugins.ksp) apply false
+    alias(libs.plugins.asciidoctorJvm) apply false
+    alias(libs.plugins.dokka) apply false
+    id("sk.ainet.documentation")
 }
 
 allprojects {
@@ -22,4 +26,38 @@ kover {
             }
         }
     }
+}
+
+// Custom task to generate operator documentation
+tasks.register("generateOperatorDocs") {
+    group = "documentation"
+    description = "Generate operator documentation from KSP-generated JSON files"
+
+    // Configure inputs for incremental builds
+    inputs.files("skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json")
+    // Configure outputs for incremental builds
+    outputs.dir("docs/modules/operators/_generated_")
+    outputs.cacheIf { true }
+
+    // Depend on KSP processing
+    dependsOn(":skainet-lang:skainet-lang-core:kspCommonMainKotlinMetadata")
+
+    // Run built-in documentation generation task (provided by sk.ainet.documentation plugin)
+    dependsOn("generateDocs")
+
+    doLast {
+        println("Operator documentation generation completed")
+    }
+}
+
+// Documentation plugin configuration
+documentation {
+    inputFile.set(file("skainet-lang/skainet-lang-core/build/generated/ksp/metadata/commonMain/resources/operators.json"))
+    outputDirectory.set(file("docs/modules/operators/_generated_"))
+    includeBackendStatus.set(true)
+    generateIndex.set(true)
+}
+
+tasks.named("generateDocs") {
+    dependsOn(":skainet-lang:skainet-lang-core:kspCommonMainKotlinMetadata")
 }

buildSrc/build.gradle.kts

@@ -0,0 +1,24 @@
+plugins {
+    `kotlin-dsl`
+    kotlin("jvm") version "2.2.20"
+    kotlin("plugin.serialization") version "2.2.20"
+}
+
+repositories {
+    gradlePluginPortal()
+    mavenCentral()
+}
+
+dependencies {
+    implementation(kotlin("stdlib"))
+    implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0")
+    implementation("org.asciidoctor:asciidoctorj:3.0.0")
+    // JSON schema validation dependencies for SchemaValidationTask
+    implementation("com.networknt:json-schema-validator:1.0.87")
+    implementation("com.fasterxml.jackson.core:jackson-databind:2.15.2")
+    implementation(gradleApi())
+}
+
+kotlin {
+    jvmToolchain(17)
+}

buildSrc/src/main/kotlin/DocumentationExtension.kt

@@ -0,0 +1,17 @@
+import models.DocumentationFormat
+import org.gradle.api.Project
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.provider.Property
+
+open class DocumentationExtension(project: Project) {
+    val inputFile: RegularFileProperty = project.objects.fileProperty()
+    val outputDirectory: DirectoryProperty = project.objects.directoryProperty()
+    val templateDirectory: DirectoryProperty = project.objects.directoryProperty()
+    val format: Property<DocumentationFormat> = project.objects.property(DocumentationFormat::class.java)
+        .convention(DocumentationFormat.ASCIIDOC)
+    val includeBackendStatus: Property<Boolean> = project.objects.property(Boolean::class.java)
+        .convention(true)
+    val generateIndex: Property<Boolean> = project.objects.property(Boolean::class.java)
+        .convention(true)
+}

buildSrc/src/main/kotlin/DocumentationPlugin.kt

@@ -0,0 +1,36 @@
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.Action
+import org.gradle.kotlin.dsl.named
+import org.gradle.kotlin.dsl.register
+import org.gradle.kotlin.dsl.configureEach
+
+class DocumentationPlugin : Plugin<Project> {
+    override fun apply(project: Project) {
+        val extension = project.extensions.create("documentation", DocumentationExtension::class.java, project)
+
+        project.tasks.register("generateDocs", GenerateDocumentationTask::class.java, object : Action<GenerateDocumentationTask> {
+            override fun execute(task: GenerateDocumentationTask) {
+                task.group = "documentation"
+                task.description = "Generate documentation from KSP metadata"
+
+                task.inputFile.set(extension.inputFile)
+                task.outputDirectory.set(extension.outputDirectory)
+                task.templateDirectory.set(extension.templateDirectory)
+                task.format.set(extension.format)
+                task.includeBackendStatus.set(extension.includeBackendStatus)
+                task.generateIndex.set(extension.generateIndex)
+            }
+        })
+
+        // Register schema validation task in plugin (migrated from skainet-lang-export-ops)
+        val validateTaskProvider = project.tasks.register("validateOperatorSchema", SchemaValidationTask::class.java, object : Action<SchemaValidationTask> {
+            override fun execute(task: SchemaValidationTask) {
+                task.group = "verification"
+                task.description = "Validate generated operators.json files against the JSON schema"
+                // By default search from the root project dir to find all operators.json
+                task.searchDirectory.set(project.rootProject.layout.projectDirectory)
+            }
+        })
+    }
+}