diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 30a58262d..eca7f2913 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -60,6 +60,15 @@ tasks: - test build_targets: - //... + examples-dagger: + name: "Example - Dagger" + platform: ubuntu1804 + working_directory: examples/dagger + include_json_profile: + - build + - test + build_targets: + - //... examples-nodejs: name: Example - Node platform: ubuntu1804 diff --git a/.bazelignore b/.bazelignore index f1cffcd7d..471232e09 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,2 +1,2 @@ # Exclude examples from //...:all -examples/* +examples diff --git a/.bazelproject b/.bazelproject index 6ce17f63c..1df98e7c7 100644 --- a/.bazelproject +++ b/.bazelproject @@ -14,16 +14,11 @@ directories: # Add the directories you want added as source here # By default, we've added your entire workspace ('.') - -examples/node/bazel-node - -examples/node/bazel-bin - -examples/node/bazel-genfiles - -examples/node/bazel-out - -examples/node/bazel-testlogs + -examples/* . targets: //:all_local_tests - //examples/dagger/... # These targets are built for the ide only. Primary purpose is to ensure the builder can build the targets, but it's # also a good way of testing the intellij plugin. //src/main/kotlin/io/bazel/kotlin/builder/tasks:tasks_for_ide diff --git a/scripts/release.sh b/scripts/release.sh index a5d20a002..3a8de2930 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -47,6 +47,8 @@ if test ! -d examples; then fail "unable to find example directory: $PWD" fi +bazel test //...:all || fail "tests failed" + # build release bazel build //:rules_kotlin_release || fail "release archive failed" diff --git a/src/main/kotlin/io/bazel/worker/ContextLog.kt b/src/main/kotlin/io/bazel/worker/ContextLog.kt index 5ace2c934..cfd225824 100644 --- a/src/main/kotlin/io/bazel/worker/ContextLog.kt +++ b/src/main/kotlin/io/bazel/worker/ContextLog.kt @@ -22,19 +22,31 @@ import java.nio.charset.StandardCharsets.UTF_8 import java.util.logging.Level /** Log encapsulates standard out and error of execution. */ -data class ContextLog(val out: CharSequence, val profiles: List = emptyList()) : +data class ContextLog( + val out: CharSequence, + val profiles: List = emptyList() +) : CharSequence by out { - constructor(bytes: ByteArray, profiles: List) : this(String(bytes, UTF_8), profiles) + constructor( + bytes: ByteArray, + profiles: List + ) : this(String(bytes, UTF_8), profiles) enum class Granularity(val level: Level) { - INFO(Level.INFO), ERROR(Level.SEVERE), DEBUG(Level.FINEST) + INFO(Level.INFO), + ERROR(Level.SEVERE), + DEBUG(Level.FINEST) } /** Logging runtime messages lazily */ interface Logging { fun debug(msg: () -> String) fun info(msg: () -> String) - fun error(t: Throwable, msg: () -> String) + fun error( + t: Throwable, + msg: () -> String + ) + fun error(msg: () -> String) } diff --git a/src/main/kotlin/io/bazel/worker/CpuTimeBasedGcScheduler.kt b/src/main/kotlin/io/bazel/worker/CpuTimeBasedGcScheduler.kt index 1acc0f4fc..9ee06ba75 100644 --- a/src/main/kotlin/io/bazel/worker/CpuTimeBasedGcScheduler.kt +++ b/src/main/kotlin/io/bazel/worker/CpuTimeBasedGcScheduler.kt @@ -1,6 +1,7 @@ package io.bazel.worker import com.sun.management.OperatingSystemMXBean +import src.main.kotlin.io.bazel.worker.GcScheduler import java.lang.management.ManagementFactory import java.time.Duration import java.util.concurrent.atomic.AtomicReference @@ -12,7 +13,7 @@ class CpuTimeBasedGcScheduler( * disable. */ private val cpuUsageBeforeGc: Duration -) { +) : GcScheduler { /** The total process CPU time at the last GC run (or from the start of the worker). */ private val cpuTime: Duration @@ -20,13 +21,13 @@ class CpuTimeBasedGcScheduler( private val cpuTimeAtLastGc: AtomicReference = AtomicReference(cpuTime) /** Call occasionally to perform a GC if enough CPU time has been used. */ - fun maybePerformGc() { + override fun maybePerformGc() { if (!cpuUsageBeforeGc.isZero) { val currentCpuTime = cpuTime val lastCpuTime = cpuTimeAtLastGc.get() // Do GC when enough CPU time has been used, but only if nobody else beat us to it. - if (currentCpuTime.minus(lastCpuTime).compareTo(cpuUsageBeforeGc) > 0 - && cpuTimeAtLastGc.compareAndSet(lastCpuTime, currentCpuTime) + if (currentCpuTime.minus(lastCpuTime).compareTo(cpuUsageBeforeGc) > 0 && + cpuTimeAtLastGc.compareAndSet(lastCpuTime, currentCpuTime) ) { System.gc() // Avoid counting GC CPU time against CPU time before next GC. diff --git a/src/main/kotlin/io/bazel/worker/GcScheduler.kt b/src/main/kotlin/io/bazel/worker/GcScheduler.kt new file mode 100644 index 000000000..4e3d9beb4 --- /dev/null +++ b/src/main/kotlin/io/bazel/worker/GcScheduler.kt @@ -0,0 +1,6 @@ +package src.main.kotlin.io.bazel.worker + +/** GcScheduler for invoking garbage collection in a persistent worker. */ +fun interface GcScheduler { + fun maybePerformGc() +} diff --git a/src/main/kotlin/io/bazel/worker/IO.kt b/src/main/kotlin/io/bazel/worker/IO.kt index a341ed25c..7d03d7ccd 100644 --- a/src/main/kotlin/io/bazel/worker/IO.kt +++ b/src/main/kotlin/io/bazel/worker/IO.kt @@ -40,6 +40,7 @@ class IO( * the same console output multiple times **/ fun readCapturedAsUtf8String(): String { + captured.flush() val out = captured.toByteArray().toString(StandardCharsets.UTF_8) captured.reset() return out diff --git a/src/main/kotlin/io/bazel/worker/InvocationWorker.kt b/src/main/kotlin/io/bazel/worker/InvocationWorker.kt index 0a95ee0da..3f1ede78c 100644 --- a/src/main/kotlin/io/bazel/worker/InvocationWorker.kt +++ b/src/main/kotlin/io/bazel/worker/InvocationWorker.kt @@ -20,10 +20,14 @@ package io.bazel.worker /** InvocationWorker executes a single unit of work. */ class InvocationWorker(private val arguments: Iterable) : Worker { override fun start(execute: Work): Int = - WorkerContext.run { - doTask("invocation") { ctx -> execute(ctx, arguments) }.run { - println(log.out.toString()) - status.exit + runCatching { + WorkerContext.run { + doTask("invocation") { ctx -> execute(ctx, arguments) }.run { + println(log.out.toString()) + status.exit + } } - } + }.recover { + 1 + }.getOrThrow() } diff --git a/src/main/kotlin/io/bazel/worker/PersistentWorker.kt b/src/main/kotlin/io/bazel/worker/PersistentWorker.kt index 06cf338a0..8cb067c4d 100644 --- a/src/main/kotlin/io/bazel/worker/PersistentWorker.kt +++ b/src/main/kotlin/io/bazel/worker/PersistentWorker.kt @@ -19,30 +19,18 @@ package io.bazel.worker import com.google.devtools.build.lib.worker.WorkerProtocol import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.ExecutorCoroutineDispatcher import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.async -import kotlinx.coroutines.flow.asFlow -import kotlinx.coroutines.flow.buffer -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.flowOn -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.Job import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED +import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.consumeAsFlow -import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext +import src.main.kotlin.io.bazel.worker.GcScheduler import java.io.PrintStream -import java.nio.charset.StandardCharsets.UTF_8 import java.time.Duration -import java.util.concurrent.Executors import kotlin.coroutines.CoroutineContext /** @@ -57,27 +45,34 @@ import kotlin.coroutines.CoroutineContext class PersistentWorker( private val coroutineContext: CoroutineContext, private val captureIO: () -> IO, - private val cpuTimeBasedGcScheduler: CpuTimeBasedGcScheduler, + private val cpuTimeBasedGcScheduler: GcScheduler ) : Worker { - constructor() : this(Dispatchers.IO, IO.Companion::capture, CpuTimeBasedGcScheduler(Duration.ofSeconds(10))) + constructor( + coroutineContext: CoroutineContext, + captureIO: () -> IO + ) : this(coroutineContext, captureIO, GcScheduler {}) + + constructor() : this( + Dispatchers.IO, IO.Companion::capture, CpuTimeBasedGcScheduler(Duration.ofSeconds(10)) + ) @ExperimentalCoroutinesApi override fun start(execute: Work) = WorkerContext.run { - //Use channel to serialize writing output + // Use channel to serialize writing output val writeChannel = Channel(UNLIMITED) captureIO().use { io -> runBlocking { - //Parent coroutine to track all of children and close channel on completion + // Parent coroutine to track all of children and close channel on completion launch(Dispatchers.Default) { - generateSequence { WorkRequest.parseDelimitedFrom(io.input) } - .forEach { request -> - launch { - compileWork(request, io, writeChannel, execute) - //Be a friendly worker by performing a GC between compilation requests - cpuTimeBasedGcScheduler.maybePerformGc() - } + generateSequence { WorkRequest.parseDelimitedFrom(io.input) } + .forEach { request -> + launch { + compileWork(request, io, writeChannel, execute) + // Be a friendly worker by performing a GC between compilation requests + cpuTimeBasedGcScheduler.maybePerformGc() } + } }.invokeOnCompletion { writeChannel.close() } writeChannel.consumeAsFlow() @@ -96,30 +91,34 @@ class PersistentWorker( chan: Channel, execute: Work ) = withContext(Dispatchers.Default) { - val result = doTask("request ${request.requestId}") { ctx -> - request.argumentsList.run { - execute(ctx, toList()) - } - } - info { "task result ${result.status}" } - val response = WorkerProtocol.WorkResponse.newBuilder().apply { - output = listOf( - result.log.out.toString(), - io.readCapturedAsUtf8String() - ).filter { it.isNotBlank() }.joinToString("\n") - exitCode = result.status.exit - requestId = request.requestId - }.build() - info { - response.toString() + val result = doTask("request ${request.requestId}") { ctx -> + request.argumentsList.run { + execute(ctx, toList()) } - chan.send(response) + } + info { "task result ${result.status}" } + val response = WorkerProtocol.WorkResponse.newBuilder().apply { + val cap = io.readCapturedAsUtf8String() + output = listOf( + result.log.out.toString(), + cap + ).joinToString("\n") + exitCode = result.status.exit + requestId = request.requestId + }.build() + + info { + response.toString() + } + chan.send(response) } - private suspend fun writeOutput(response: WorkerProtocol.WorkResponse, output: PrintStream) = + private suspend fun writeOutput( + response: WorkerProtocol.WorkResponse, + output: PrintStream + ) = withContext(Dispatchers.IO) { response.writeDelimitedTo(output) output.flush() } - } diff --git a/src/main/kotlin/io/bazel/worker/TaskResult.kt b/src/main/kotlin/io/bazel/worker/TaskResult.kt index 2fc8c4c2c..214c84a68 100644 --- a/src/main/kotlin/io/bazel/worker/TaskResult.kt +++ b/src/main/kotlin/io/bazel/worker/TaskResult.kt @@ -17,4 +17,7 @@ package io.bazel.worker -data class TaskResult(val status: Status, val log: ContextLog) +data class TaskResult( + val status: Status, + val log: ContextLog +) diff --git a/src/main/kotlin/io/bazel/worker/Work.kt b/src/main/kotlin/io/bazel/worker/Work.kt index db78ab73d..91e54c0cd 100644 --- a/src/main/kotlin/io/bazel/worker/Work.kt +++ b/src/main/kotlin/io/bazel/worker/Work.kt @@ -21,5 +21,8 @@ import io.bazel.worker.WorkerContext.TaskContext /** Task for Worker execution. */ fun interface Work { - operator fun invoke(ctx: TaskContext, args: Iterable): Status + operator fun invoke( + ctx: TaskContext, + args: Iterable + ): Status } diff --git a/src/main/kotlin/io/bazel/worker/Worker.kt b/src/main/kotlin/io/bazel/worker/Worker.kt index 5d4aeb004..ba990284c 100644 --- a/src/main/kotlin/io/bazel/worker/Worker.kt +++ b/src/main/kotlin/io/bazel/worker/Worker.kt @@ -20,7 +20,10 @@ package io.bazel.worker /** Worker executes a unit of Work */ interface Worker { companion object { - fun from(args: Iterable, then: Worker.(Iterable) -> Int): Int { + fun from( + args: Iterable, + then: Worker.(Iterable) -> Int + ): Int { val worker = when { "--persistent_worker" in args -> PersistentWorker() else -> InvocationWorker(args) diff --git a/src/main/kotlin/io/bazel/worker/WorkerContext.kt b/src/main/kotlin/io/bazel/worker/WorkerContext.kt index 4596b0655..54c1fa2d7 100644 --- a/src/main/kotlin/io/bazel/worker/WorkerContext.kt +++ b/src/main/kotlin/io/bazel/worker/WorkerContext.kt @@ -86,7 +86,10 @@ class WorkerContext private constructor( logger.logp(Level.INFO, sourceName, name, msg) } - override fun error(t: Throwable, msg: () -> String) { + override fun error( + t: Throwable, + msg: () -> String + ) { logger.logp(Level.SEVERE, sourceName, name, t, msg) } diff --git a/src/test/kotlin/io/bazel/kotlin/BUILD b/src/test/kotlin/io/bazel/kotlin/BUILD index 6679cff46..f2b461b71 100644 --- a/src/test/kotlin/io/bazel/kotlin/BUILD +++ b/src/test/kotlin/io/bazel/kotlin/BUILD @@ -51,11 +51,6 @@ kt_rules_e2e_test( data = ["//src/test/data/jvm/kapt"], ) -kt_rules_e2e_test( - name = "KotlinJvmDaggerExampleTest", - srcs = ["KotlinJvmDaggerExampleTest.kt"], - data = ["//examples/dagger:coffee_app"], -) kt_rules_e2e_test( name = "KotlinJvmAssociatesBasicVisibilityTest", @@ -74,7 +69,6 @@ test_suite( "KotlinJvm13Test", "KotlinJvmAssociatesBasicVisibilityTest", "KotlinJvmBasicAssertionTest", - "KotlinJvmDaggerExampleTest", "KotlinJvmKaptAssertionTest", ], ) diff --git a/src/test/kotlin/io/bazel/kotlin/KotlinJvmDaggerExampleTest.kt b/src/test/kotlin/io/bazel/kotlin/KotlinJvmDaggerExampleTest.kt deleted file mode 100644 index 836d20f78..000000000 --- a/src/test/kotlin/io/bazel/kotlin/KotlinJvmDaggerExampleTest.kt +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2018 The Bazel Authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.bazel.kotlin - -import org.junit.Test - -/** - * These tests verify properties of the example. - */ -class KotlinJvmDaggerExampleTest : BasicAssertionTestCase() { - @Test - fun daggerExampleIsRunnable() { - assertExecutableRunfileSucceeds( - "//examples/dagger/coffee_app", - description = "the dagger coffee_app should execute successfully" - ) - } -} diff --git a/src/test/kotlin/io/bazel/worker/PersistentWorkerTest.kt b/src/test/kotlin/io/bazel/worker/PersistentWorkerTest.kt index d5f1d744c..f4ceaea59 100644 --- a/src/test/kotlin/io/bazel/worker/PersistentWorkerTest.kt +++ b/src/test/kotlin/io/bazel/worker/PersistentWorkerTest.kt @@ -75,7 +75,6 @@ class PersistentWorkerTest { closeStdIn() return@inProcess generateSequence { readStdOut() } }.associateBy { wr -> wr.requestId } - assertThat(String(captured.toByteArray(), UTF_8)).contains("Squeek!") assertThat(actualResponses.keys).isEqualTo(expectedResponses.keys)