From e303e78ffb942e8ed153daa5d5453a68ec369680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Pr=C3=B3chniak?= Date: Fri, 15 Nov 2019 06:52:11 +0100 Subject: [PATCH] Scala 2.12 build (#99) Scala 2.11 and 2.12 cross build --- .travis.yml | 25 ++++++------ README.md | 9 +++-- build.sbt | 39 ++++++++++++------- ciBackendTest.sh | 14 +++++-- ciBuild.sh | 27 ------------- ciPublish.sh | 21 ++++++++++ .../sample/TestProcessConfigCreator.scala | 4 +- .../src/it/resources/application.conf | 7 ---- .../src/it/resources/docker/Dockerfile | 3 +- .../engine/management/DockerTest.scala | 26 ++++++++++--- .../management/batch/BatchDockerTest.scala | 12 ++---- ...nkStreamingProcessConfigLocationTest.scala | 3 +- .../FlinkStreamingProcessManagerSpec.scala | 27 ++++++------- .../FlinkStreamingProcessTestRunnerSpec.scala | 13 +++++-- .../JavaConfigProcessManagerSpec.scala | 13 +------ .../streaming/StreamingDockerTest.scala | 11 +++--- .../engine/process/ProcessTestHelpers.scala | 4 +- .../engine/compile/ExpressionCompiler.scala | 3 +- .../engine/sql/HsqlSqlQueryableDataBase.scala | 14 ++++++- .../engine/types/EspTypeUtilsSpec.scala | 9 +++-- .../engine/src/it/resources/application.conf | 7 ---- .../engine/src/it/resources/standalone.conf | 7 ++++ .../StandaloneProcessManagerSpec.scala | 7 +++- .../util/config/ScalaMajorVersionConfig.scala | 17 ++++++++ ui/client/webpack.config.js | 3 +- .../nussknacker/ui/api/UserResources.scala | 1 + .../process/ProcessTypesForCategories.scala | 6 +-- .../resources/{application.conf => ui.conf} | 4 +- .../ProcessesNonTechnicalResourcesSpec.scala | 1 + .../ui/api/ServiceRoutesSpec.scala | 3 +- .../nussknacker/ui/api/SslBindingSpec.scala | 3 +- .../ui/api/helpers/EspItTest.scala | 16 ++++---- .../ui/api/helpers/TestFactory.scala | 3 +- .../definition/DefinitionPreparerSpec.scala | 7 +++- .../ui/definition/UIProcessObjectsSpec.scala | 10 +++-- .../ui/integration/BaseFlowTest.scala | 11 ++++-- .../ui/integration/DictsFlowTest.scala | 5 ++- .../ui/util/ConfigWithScalaVersion.scala | 10 +++++ 38 files changed, 236 insertions(+), 169 deletions(-) delete mode 100755 ciBuild.sh create mode 100755 ciPublish.sh delete mode 100644 engine/standalone/engine/src/it/resources/application.conf create mode 100644 engine/standalone/engine/src/it/resources/standalone.conf create mode 100644 engine/util/src/main/scala/pl/touk/nussknacker/engine/util/config/ScalaMajorVersionConfig.scala rename ui/server/src/test/resources/{application.conf => ui.conf} (89%) create mode 100644 ui/server/src/test/scala/pl/touk/nussknacker/ui/util/ConfigWithScalaVersion.scala diff --git a/.travis.yml b/.travis.yml index 6857a3d01af..21b8d4d750f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,5 @@ language: scala -scala: - - 2.11.12 - jdk: - oraclejdk8 @@ -51,7 +48,7 @@ cache: - $HOME/.npm stages: - - tests + - test - name: releases # We don't allow do releases from fork and corn type because of security: https://docs.travis-ci.com/user/pull-requests/#pull-requests-and-security-restrictions # We also don't allow for pull request builds because they are built from merge of source and upstream branches. @@ -60,23 +57,23 @@ stages: jobs: include: - - stage: tests + - stage: test name: "Backend tests" - script: ./ciBackendTest.sh $TRAVIS_SCALA_VERSION - after_success: if [ "$COVERAGE" = true ]; then sbt ++$TRAVIS_SCALA_VERSION coveralls; else echo "Skipping coveralls"; fi - - name: "Frontend tests" - before_script: if [[ `npm -v | sed 's/\..*//'` < 6 ]]; then npm i -g npm; fi - script: ./ciFrontendTest.sh + script: ./ciBackendTest.sh + after_success: if [ "$COVERAGE" = true ]; then sbt + coveralls; else echo "Skipping coveralls"; fi - name: "UI slow tests" - script: sbt ++$TRAVIS_SCALA_VERSION clean ui/slow:test + script: sbt +clean +ui/slow:test - name: "Management tests" - script: sbt ++$TRAVIS_SCALA_VERSION clean management/it:test + script: sbt +clean +management/it:test - name: "EngineStandalone tests" - script: sbt ++$TRAVIS_SCALA_VERSION clean engineStandalone/it:test + script: sbt +clean +engineStandalone/it:test + - name: "Frontend tests" + before_script: if [[ `npm -v | sed 's/\..*//'` < 6 ]]; then npm i -g npm; fi + script: ./ciFrontendTest.sh - stage: releases name: "Release sonatype packages" before_script: if [[ `npm -v | sed 's/\..*//'` < 6 ]]; then npm i -g npm; fi - script: ./ciBuild.sh "${BUILD_VERSION}-SNAPSHOT" ${NEXUS_PASSWORD} ${NEXUS_URL} ${NEXUS_USER} true + script: ./ciPublish.sh "${BUILD_VERSION}-SNAPSHOT" ${NEXUS_PASSWORD} ${NEXUS_URL} ${NEXUS_USER} - name: "Push Docker Image" before_script: if [[ `npm -v | sed 's/\..*//'` < 6 ]]; then npm i -g npm; fi script: diff --git a/README.md b/README.md index 23a1591cd56..6a23d28cbe2 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,10 @@ Talk to us on our [mailing list](https://groups.google.com/forum/#!forum/nusskna **Nussknacker** is published under [Apache License 2.0](http://www.apache.org/licenses/LICENSE-2.0). - ## Scala compatibility -Currently we only support scala 2.11.x. The main reason we don't support scala < 2.11 and scala 2.12 is -[this jira](https://issues.apache.org/jira/browse/FLINK-5005) - we rely heavily on Flink, and making it support scala 2.12 -is suprisingly hard (due to changes in implementation of lambdas). +Currently we do support Scala 2.11 and 2.12, we cross publish versions. + +## Flink compatibility + +We currently support only one Flink version (please see flinkV in build.sbt) \ No newline at end of file diff --git a/build.sbt b/build.sbt index be2a55f38aa..3b0f5e140cd 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,9 @@ import sbt._ import sbtassembly.AssemblyPlugin.autoImport.assembly import sbtassembly.MergeStrategy -val scalaV = "2.11.12" +val scala211 = "2.11.12" +val scala212 = "2.12.10" +lazy val supportedScalaVersions = List(scala212, scala211) //by default we include flink and scala, we want to be able to disable this behaviour for performance reasons val includeFlinkAndScala = Option(System.getProperty("includeFlinkAndScala", "true")).exists(_.toBoolean) @@ -26,7 +28,8 @@ val dockerUpLatest = System.getProperty("dockerUpLatest", "true").toBoolean // unfortunately it does not work, so we resort to hack by publishing root module to Resolver.defaultLocal //publishArtifact := false publishTo := Some(Resolver.defaultLocal) - +crossScalaVersions := Nil + val publishSettings = Seq( publishMavenStyle := true, releasePublishArtifactsAction := PgpKeys.publishSigned.value, @@ -81,7 +84,8 @@ val commonSettings = Seq( test in assembly := {}, licenses += ("Apache-2.0", url("https://www.apache.org/licenses/LICENSE-2.0.html")), - scalaVersion := scalaV, + crossScalaVersions := supportedScalaVersions, + scalaVersion := scala212, resolvers ++= Seq( "confluent" at "https://packages.confluent.io/maven" ), @@ -102,11 +106,15 @@ val commonSettings = ), javacOptions := Seq( "-Xlint:deprecation", - "-Xlint:unchecked" + "-Xlint:unchecked", + //we use it e.g. to provide consistent behaviour wrt extracting parameter names from scala and java + "-parameters" ), assemblyMergeStrategy in assembly := nussknackerMergeStrategy, coverageMinimum := 60, - coverageFailOnMinimum := false + coverageFailOnMinimum := false, + //problem with scaladoc of api: https://github.com/scala/bug/issues/10134 + scalacOptions in (Compile, doc) -= "-Xfatal-warnings" ) val akkaV = "2.4.20" //same version as in Flink @@ -156,7 +164,7 @@ lazy val dockerSettings = { dockerLabels := Map( "tag" -> dockerTagName.getOrElse(version.value), "version" -> version.value, - "scala" -> scalaV, + "scala" -> scalaVersion.value, "flink" -> flinkV ), version in Docker := dockerTagName.getOrElse(version.value) @@ -172,12 +180,13 @@ lazy val dist = (project in file("nussknacker-dist")) (assembly in Compile) in generic, (assembly in Compile) in example ).value, + mappings in Universal += { - val genericModel = generic.base / "target" / "scala-2.11" / "genericModel.jar" + val genericModel = (crossTarget in generic).value / "genericModel.jar" genericModel -> "model/genericModel.jar" }, mappings in Universal += { - val exampleModel = example.base / "target" / "scala-2.11" / s"nussknacker-example-assembly-${version.value}.jar" + val exampleModel = (crossTarget in example).value / s"nussknacker-example-assembly-${version.value}.jar" exampleModel -> "model/exampleModel.jar" }, publishArtifact := false, @@ -355,6 +364,7 @@ lazy val generic = (project in engine("flink/generic")). ) }, test in assembly := {}, + assemblyJarName in assembly := "genericModel.jar", artifact in (Compile, assembly) := { val art = (artifact in (Compile, assembly)).value @@ -484,7 +494,7 @@ lazy val util = (project in engine("util")). Seq( "com.iheart" %% "ficus" % ficusV, "org.scalatest" %% "scalatest" % scalaTestV % "test", - "io.circe" %% "circe-java8" % "0.11.1" + "io.circe" %% "circe-java8" % circeV ) } ).dependsOn(api) @@ -618,6 +628,9 @@ lazy val httpUtils = (project in engine("httpUtils")). libraryDependencies ++= { val sttpV = "2.0.0-M6" Seq( + //we force circe version here, because sttp has 0.12.1 for scala 2.12, we don't want it ATM + "io.circe" %% "circe-core" % circeV force(), + "io.circe" %% "circe-parser" % circeV force(), "org.dispatchhttp" %% "dispatch-core" % dispatchV, "org.asynchttpclient" % "async-http-client" % "2.10.4", "org.scala-lang.modules" %% "scala-parser-combinators" % scalaParsersV, // scalaxb deps @@ -649,11 +662,11 @@ lazy val queryableState = (project in engine("queryableState")). lazy val buildUi = taskKey[Unit]("builds ui") -def runNpm(command: String, errorMessage: String): Unit = { +def runNpm(command: String, errorMessage: String, outputPath: File): Unit = { import sys.process.Process val path = Path.apply("ui/client").asFile println("Using path: " + path.getAbsolutePath) - val result = Process(s"npm $command", path)!; + val result = Process(s"npm $command", path, "OUTPUT_PATH" -> outputPath.absolutePath)!; if (result != 0) throw new RuntimeException(errorMessage) } @@ -674,8 +687,8 @@ lazy val ui = (project in file("ui/server")) .settings(commonSettings) .settings( name := "nussknacker-ui", - buildUi := { - runNpm("run build", "Client build failed") + buildUi := { + runNpm("run build", "Client build failed", (crossTarget in compile).value) }, parallelExecution in ThisBuild := false, assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = includeFlinkAndScala, level = Level.Info), diff --git a/ciBackendTest.sh b/ciBackendTest.sh index 944adacb237..548c75fb7f3 100755 --- a/ciBackendTest.sh +++ b/ciBackendTest.sh @@ -1,12 +1,18 @@ #!/usr/bin/env bash -scala_version=$1 +scala_version="" +scala_cross_build="+" + +if ! [[ -z "$1" ]]; then + scala_version="++$1" + scala_cross_build="" +fi set -e if [ "$COVERAGE" = true ]; then - sbt ++$scala_version clean coverage test coverageReport - sbt ++$scala_version coverageAggregate + sbt $scala_version ${scala_cross_build}clean ${scala_cross_build}coverage ${scala_cross_build}test ${scala_cross_build}coverageReport + sbt $scala_version ${scala_cross_build}coverageAggregate else - sbt ++$scala_version clean test + sbt $scala_version ${scala_cross_build}clean ${scala_cross_build}test fi \ No newline at end of file diff --git a/ciBuild.sh b/ciBuild.sh deleted file mode 100755 index dcb00058ade..00000000000 --- a/ciBuild.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -e - -espEngineToukVersion=$1 -nexusPassword=$2 - -if [[ -n "$3" ]]; then - nexusUrlProperty="-DnexusUrl=$3" -else - nexusUrlProperty="" -fi - -if [[ -n "$4" ]]; then - nexusUserProperty="-DnexusUser=$4" -else - nexusUserProperty="" -fi - -cd ui/client && npm ci && cd - -if [[ -z "$5" || "$5" == false ]]; then - ./sbtwrapper clean test management/it:test engineStandalone/it:test ui/slow:test || { echo 'Failed to build and test nussknacker' ; exit 1; } -fi - -if [[ -n "$espEngineToukVersion" ]]; then - ./sbtwrapper -DnexusPassword=$2 ${nexusUrlProperty} ${nexusUserProperty} "set version in ThisBuild := \"$espEngineToukVersion\"" publish -fi diff --git a/ciPublish.sh b/ciPublish.sh new file mode 100755 index 00000000000..0f97d481758 --- /dev/null +++ b/ciPublish.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +espEngineToukVersion=$1 +nexusPassword=$2 + +if [[ -n "$3" ]]; then + nexusUrlProperty="-DnexusUrl=$3" +else + nexusUrlProperty="" +fi + +if [[ -n "$4" ]]; then + nexusUserProperty="-DnexusUser=$4" +else + nexusUserProperty="" +fi + +cd ui/client && npm ci && cd - +./sbtwrapper -DnexusPassword=$2 ${nexusUrlProperty} ${nexusUserProperty} "set version in ThisBuild := \"$espEngineToukVersion\"" +publish diff --git a/engine/flink/management/sample/src/main/scala/pl/touk/nussknacker/engine/management/sample/TestProcessConfigCreator.scala b/engine/flink/management/sample/src/main/scala/pl/touk/nussknacker/engine/management/sample/TestProcessConfigCreator.scala index a4def8c77a7..36657ae7b51 100644 --- a/engine/flink/management/sample/src/main/scala/pl/touk/nussknacker/engine/management/sample/TestProcessConfigCreator.scala +++ b/engine/flink/management/sample/src/main/scala/pl/touk/nussknacker/engine/management/sample/TestProcessConfigCreator.scala @@ -293,7 +293,7 @@ case object CustomFilter extends CustomStreamTransformer { = FlinkCustomStreamTransformation((start: DataStream[Context], ctx: FlinkCustomNodeContext) => start .filter(ctx.lazyParameterHelper.lazyFilterFunction(expression)) - .map(ValueWithContext(null, _))) + .map(ValueWithContext[Any](null, _))) } @@ -309,7 +309,7 @@ object AdditionalVariableTransformer extends CustomStreamTransformer { @MethodToInvoke(returnType = classOf[Void]) def execute(@AdditionalVariables(Array(new AdditionalVariable(name = "additional", clazz = classOf[String]))) @ParamName("expression") expression: LazyParameter[Boolean]) = FlinkCustomStreamTransformation((start: DataStream[Context]) => - start.map(ValueWithContext("", _))) + start.map(ValueWithContext[Any]("", _))) } diff --git a/engine/flink/management/src/it/resources/application.conf b/engine/flink/management/src/it/resources/application.conf index a1cfe8df621..cf65f010508 100644 --- a/engine/flink/management/src/it/resources/application.conf +++ b/engine/flink/management/src/it/resources/application.conf @@ -3,9 +3,6 @@ flinkConfig { restUrl: "localhost:8081" queryableStateProxyUrl: "localhost:9069" jobManagerTimeout: 1m - classpath: [ - "./engine/flink/management/sample/target/scala-2.11/managementSample.jar" - ] } processConfig { @@ -15,10 +12,6 @@ processConfig { bufferSize: 200 parallelismMultiplier: 2 } - #this will be overwritten for docker tests - kafka = { - kafkaAddress = "kafka:9092" - } signals { topic: "esp.signals" diff --git a/engine/flink/management/src/it/resources/docker/Dockerfile b/engine/flink/management/src/it/resources/docker/Dockerfile index 502e81c6d11..4400a6b1a86 100644 --- a/engine/flink/management/src/it/resources/docker/Dockerfile +++ b/engine/flink/management/src/it/resources/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM flink:1.7.2-scala_2.11 +FROM flink:1.7.2-scala_${scala.major.version} COPY entrypointWithIP.sh / COPY docker-entrypoint.sh / @@ -16,7 +16,6 @@ RUN chmod +x /docker-entrypoint.sh USER flink RUN mkdir -p /tmp/storage -RUN mkdir -p /tmp/storage/1.4/blob/cache USER root ENTRYPOINT ["/entrypointWithIP.sh"] \ No newline at end of file diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/DockerTest.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/DockerTest.scala index e6c22890ef3..51b3f874255 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/DockerTest.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/DockerTest.scala @@ -1,31 +1,34 @@ package pl.touk.nussknacker.engine.management import java.io.File +import java.nio.file.Files import java.nio.file.{Files, Path} import java.nio.file.attribute.{PosixFilePermission, PosixFilePermissions} +import java.util.Collections import com.spotify.docker.client.{DefaultDockerClient, DockerClient} +import com.typesafe.config.ConfigValueFactory.fromAnyRef +import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} import com.typesafe.scalalogging.LazyLogging import com.whisk.docker.impl.spotify.SpotifyDockerFactory import com.whisk.docker.scalatest.DockerTestKit import com.whisk.docker.{ContainerLink, DockerContainer, DockerFactory, DockerReadyChecker, LogLineReceiver, VolumeMapping} -import org.apache.commons.io.FileUtils +import org.apache.commons.io.{FileUtils, IOUtils} import org.scalatest.Suite import org.scalatest.concurrent.ScalaFutures import org.scalatest.time.{Millis, Seconds, Span} import pl.touk.nussknacker.engine.kafka.KafkaClient +import pl.touk.nussknacker.engine.util.config.ScalaMajorVersionConfig import scala.concurrent.duration._ trait DockerTest extends DockerTestKit with ScalaFutures with LazyLogging { self: Suite => - private val flinkEsp = "flinkesp:1.7.2" + private val flinkEsp = s"flinkesp:1.7.2-scala_${ScalaMajorVersionConfig.scalaMajorVersion}" private val client: DockerClient = DefaultDockerClient.fromEnv().build() - protected var kafkaClient: KafkaClient = _ - override implicit val patienceConfig: PatienceConfig = PatienceConfig( timeout = Span(90, Seconds), interval = Span(1, Millis) @@ -39,7 +42,9 @@ trait DockerTest extends DockerTestKit with ScalaFutures with LazyLogging { val dirFile = dir.toFile List("Dockerfile", "entrypointWithIP.sh", "conf.yml", "docker-entrypoint.sh").foreach { file => - FileUtils.copyInputStreamToFile(getClass.getResourceAsStream(s"/docker/$file"), new File(dirFile, file)) + val resource = IOUtils.toString(getClass.getResourceAsStream(s"/docker/$file")) + val withVersionReplaced = resource.replace("${scala.major.version}", ScalaMajorVersionConfig.scalaMajorVersion) + FileUtils.writeStringToFile(new File(dirFile, file), withVersionReplaced) } client.build(dir, flinkEsp) @@ -92,4 +97,15 @@ trait DockerTest extends DockerTestKit with ScalaFutures with LazyLogging { Files.createTempDirectory("dockerTest", PosixFilePermissions.asFileAttribute(PosixFilePermission.values().toSet[PosixFilePermission].asJava)) } + + def config: Config = ConfigFactory.load() + .withValue("flinkConfig.restUrl", fromAnyRef(s"http://${jobManagerContainer.getIpAddresses().futureValue.head}:$FlinkJobManagerRestPort")) + .withValue("flinkConfig.classpath", ConfigValueFactory.fromIterable(Collections.singletonList(classPath))) + .withFallback(additionalConfig) + + + protected def classPath: String + + protected def additionalConfig: Config = ConfigFactory.empty() + } diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/batch/BatchDockerTest.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/batch/BatchDockerTest.scala index 8c004b7a472..65745271538 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/batch/BatchDockerTest.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/batch/BatchDockerTest.scala @@ -2,14 +2,11 @@ package pl.touk.nussknacker.engine.management.batch import java.nio.file.Path -import com.typesafe.config.ConfigValueFactory.fromAnyRef -import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} import com.whisk.docker.{DockerContainer, VolumeMapping} import org.scalatest.Suite import pl.touk.nussknacker.engine.api.deployment.ProcessManager import pl.touk.nussknacker.engine.management.{DockerTest, FlinkBatchProcessManagerProvider} - -import scala.collection.JavaConverters._ +import pl.touk.nussknacker.engine.util.config.ScalaMajorVersionConfig trait BatchDockerTest extends DockerTest { self: Suite => @@ -24,12 +21,11 @@ trait BatchDockerTest extends DockerTest { self: Suite => taskManagerContainer ) ++ super.dockerContainers - def config: Config = ConfigFactory.load() - .withValue("flinkConfig.restUrl", fromAnyRef(s"http://${jobManagerContainer.getIpAddresses().futureValue.head}:$FlinkJobManagerRestPort")) - .withValue("flinkConfig.classpath", ConfigValueFactory.fromIterable(List("./engine/flink/management/batch_sample/target/scala-2.11/managementBatchSample.jar").asJava)) - lazy val processManager: ProcessManager = { val typeConfig = FlinkBatchProcessManagerProvider.defaultTypeConfig(config) new FlinkBatchProcessManagerProvider().createProcessManager(typeConfig.toModelData, typeConfig.engineConfig) } + + override protected def classPath: String + = s"./engine/flink/management/batch_sample/target/scala-${ScalaMajorVersionConfig.scalaMajorVersion}/managementBatchSample.jar" } diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessConfigLocationTest.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessConfigLocationTest.scala index b6adbbe8865..7e2a2b16bee 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessConfigLocationTest.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessConfigLocationTest.scala @@ -13,12 +13,13 @@ import scala.collection.JavaConverters._ class FlinkStreamingProcessConfigLocationTest extends FunSuite with StreamingDockerTest with Matchers { - //FIXME: do we still need it?? ignore("reads config from local dir") { processManager.findJobStatus(ProcessName("aaa")).futureValue shouldBe None } + override protected def classPath: String = "" + override lazy val config: Config = { val normalConf = super.config diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessManagerSpec.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessManagerSpec.scala index 331d98d6416..5f05fb065aa 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessManagerSpec.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessManagerSpec.scala @@ -17,6 +17,7 @@ import pl.touk.nussknacker.engine.graph.EspProcess import pl.touk.nussknacker.engine.kafka.KafkaClient import pl.touk.nussknacker.engine.management.FlinkStreamingProcessManagerProvider import pl.touk.nussknacker.engine.marshall.ProcessMarshaller +import pl.touk.nussknacker.engine.util.config.ScalaMajorVersionConfig import scala.concurrent.duration._ @@ -25,6 +26,8 @@ class FlinkStreamingProcessManagerSpec extends FunSuite with Matchers with Scala import pl.touk.nussknacker.engine.kafka.KafkaUtils._ + override protected def classPath: String = s"./engine/flink/management/sample/target/scala-${ScalaMajorVersionConfig.scalaMajorVersion}/managementSample.jar" + test("deploy process in running flink") { val processId = "runningFlink" @@ -191,22 +194,14 @@ class FlinkStreamingProcessManagerSpec extends FunSuite with Matchers with Scala .withValue("processConfig.signals.topic", ConfigValueFactory.fromAnyRef(signalsTopic)) val flinkModelData = FlinkStreamingProcessManagerProvider.defaultTypeConfig(configWithSignals).toModelData - val kafkaClient = new KafkaClient( - configWithSignals.getString("processConfig.kafka.kafkaAddress"), - configWithSignals.getString("processConfig.kafka.zkAddress")) - - try { - val consumer = kafkaClient.createConsumer() - flinkModelData.dispatchSignal("removeLockSignal", "test-process", Map("lockId" -> "test-lockId")) - - val readSignals = consumer.consume(signalsTopic).take(1).map(m => new String(m.message(), StandardCharsets.UTF_8)).toList - val signalJson = CirceUtil.decodeJsonUnsafe[Json](readSignals.head, "invalid signals").hcursor - signalJson.downField("processId").focus shouldBe Some(Json.fromString("test-process")) - signalJson.downField("action").downField("type").focus shouldBe Some(Json.fromString("RemoveLock")) - signalJson.downField("action").downField("lockId").focus shouldBe Some(Json.fromString("test-lockId")) - } finally { - kafkaClient.shutdown() - } + val consumer = kafkaClient.createConsumer() + flinkModelData.dispatchSignal("removeLockSignal", "test-process", Map("lockId" -> "test-lockId")) + + val readSignals = consumer.consume(signalsTopic).take(1).map(m => new String(m.message(), StandardCharsets.UTF_8)).toList + val signalJson = CirceUtil.decodeJsonUnsafe[Json](readSignals.head, "invalid signals").hcursor + signalJson.downField("processId").focus shouldBe Some(Json.fromString("test-process")) + signalJson.downField("action").downField("type").focus shouldBe Some(Json.fromString("RemoveLock")) + signalJson.downField("action").downField("lockId").focus shouldBe Some(Json.fromString("test-lockId")) } diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessTestRunnerSpec.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessTestRunnerSpec.scala index 8853041caf3..df2689528c3 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessTestRunnerSpec.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingProcessTestRunnerSpec.scala @@ -1,8 +1,8 @@ package pl.touk.nussknacker.engine.management.streaming -import java.util.UUID +import java.util.{Collections, UUID} -import com.typesafe.config.ConfigFactory +import com.typesafe.config.{ConfigFactory, ConfigValueFactory} import org.scalatest.concurrent.{Eventually, ScalaFutures} import org.scalatest.time.{Millis, Seconds, Span} import org.scalatest.{FlatSpec, Matchers} @@ -12,18 +12,24 @@ import pl.touk.nussknacker.engine.build.EspProcessBuilder import pl.touk.nussknacker.engine.canonize.ProcessCanonizer import pl.touk.nussknacker.engine.management.FlinkStreamingProcessManagerProvider import pl.touk.nussknacker.engine.marshall.ProcessMarshaller +import pl.touk.nussknacker.engine.util.config.ScalaMajorVersionConfig import scala.concurrent.Await class FlinkStreamingProcessTestRunnerSpec extends FlatSpec with Matchers with ScalaFutures with Eventually { + private val classPath: String = s"./engine/flink/management/sample/target/scala-${ScalaMajorVersionConfig.scalaMajorVersion}/managementSample.jar" + + private val config = ConfigFactory.load() + .withValue("processConfig.kafka.kafkaAddress", ConfigValueFactory.fromAnyRef("kafka:1234")) + .withValue("flinkConfig.classpath", ConfigValueFactory.fromIterable(Collections.singletonList(classPath))) + override implicit val patienceConfig: PatienceConfig = PatienceConfig( timeout = Span(10, Seconds), interval = Span(100, Millis) ) it should "run process in test mode" in { - val config = ConfigFactory.load() val processManager = FlinkStreamingProcessManagerProvider.defaultProcessManager(config) val processId = UUID.randomUUID().toString @@ -49,7 +55,6 @@ class FlinkStreamingProcessTestRunnerSpec extends FlatSpec with Matchers with Sc .source("startProcess", "kafka-transaction") .emptySink("endSend", "sendSms") - val config = ConfigFactory.load() val processManager = FlinkStreamingProcessManagerProvider.defaultProcessManager(config) val processData = ProcessMarshaller.toJson(ProcessCanonizer.canonize(process)).spaces2 diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/JavaConfigProcessManagerSpec.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/JavaConfigProcessManagerSpec.scala index 9bc9b531d31..8fcad0ac460 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/JavaConfigProcessManagerSpec.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/JavaConfigProcessManagerSpec.scala @@ -1,8 +1,6 @@ package pl.touk.nussknacker.engine.management.streaming -import java.util.Collections -import com.typesafe.config.{Config, ConfigValueFactory} import org.scalatest.concurrent.{Eventually, ScalaFutures} import org.scalatest.{FunSuite, Matchers} import pl.touk.nussknacker.engine.api.ProcessVersion @@ -11,17 +9,13 @@ import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.build.EspProcessBuilder import pl.touk.nussknacker.engine.canonize.ProcessCanonizer import pl.touk.nussknacker.engine.marshall.ProcessMarshaller +import pl.touk.nussknacker.engine.util.config.ScalaMajorVersionConfig import scala.concurrent.duration._ class JavaConfigProcessManagerSpec extends FunSuite with Matchers with ScalaFutures with Eventually with StreamingDockerTest { - - override def config: Config = { - super.config - .withValue("flinkConfig.classpath", - ConfigValueFactory.fromIterable(Collections.singletonList("./engine/flink/management/java_sample/target/scala-2.11/managementJavaSample.jar"))) - } + override protected def classPath: String = s"./engine/flink/management/java_sample/target/scala-${ScalaMajorVersionConfig.scalaMajorVersion}/managementJavaSample.jar" test("deploy java process in running flink") { val processId = "runningJavaFlink" @@ -41,7 +35,4 @@ class JavaConfigProcessManagerSpec extends FunSuite with Matchers with ScalaFutu assert(processManager.cancel(ProcessName(process.id)).isReadyWithin(10 seconds)) } - - - } diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/StreamingDockerTest.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/StreamingDockerTest.scala index f14b1b6f393..9158ba4b011 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/StreamingDockerTest.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/StreamingDockerTest.scala @@ -1,21 +1,24 @@ package pl.touk.nussknacker.engine.management.streaming -import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} import com.typesafe.config.ConfigValueFactory.fromAnyRef import com.whisk.docker.{ContainerLink, DockerContainer, DockerReadyChecker} import org.scalatest.Suite import pl.touk.nussknacker.engine.api.deployment.ProcessManager import pl.touk.nussknacker.engine.kafka.KafkaClient import pl.touk.nussknacker.engine.management.{DockerTest, FlinkStreamingProcessManagerProvider} +import pl.touk.nussknacker.engine.util.config.ScalaMajorVersionConfig import scala.concurrent.duration._ trait StreamingDockerTest extends DockerTest { self: Suite => + protected var kafkaClient: KafkaClient = _ + override def beforeAll(): Unit = { super.beforeAll() kafkaClient = new KafkaClient(config.getString("processConfig.kafka.kafkaAddress"), - config.getString("processConfig.kafka.zkAddress")) + s"${ipOfContainer(zookeeperContainer)}:$ZookeeperDefaultPort") } override def afterAll(): Unit = { @@ -42,10 +45,8 @@ trait StreamingDockerTest extends DockerTest { self: Suite => ) ++ super.dockerContainers } - def config: Config = ConfigFactory.load() - .withValue("processConfig.kafka.zkAddress", fromAnyRef(s"${ipOfContainer(zookeeperContainer)}:$ZookeeperDefaultPort")) + override protected def additionalConfig: Config = ConfigFactory.empty() .withValue("processConfig.kafka.kafkaAddress", fromAnyRef(s"${ipOfContainer(kafkaContainer)}:$KafkaPort")) - .withValue("flinkConfig.restUrl", fromAnyRef(s"http://${ipOfContainer(jobManagerContainer)}:$FlinkJobManagerRestPort")) protected lazy val processManager: ProcessManager = FlinkStreamingProcessManagerProvider.defaultProcessManager(config) diff --git a/engine/flink/process/src/test/scala/pl/touk/nussknacker/engine/process/ProcessTestHelpers.scala b/engine/flink/process/src/test/scala/pl/touk/nussknacker/engine/process/ProcessTestHelpers.scala index 5ee8492a59c..11a724608ea 100644 --- a/engine/flink/process/src/test/scala/pl/touk/nussknacker/engine/process/ProcessTestHelpers.scala +++ b/engine/flink/process/src/test/scala/pl/touk/nussknacker/engine/process/ProcessTestHelpers.scala @@ -159,7 +159,7 @@ object ProcessTestHelpers { .filter(new AbstractOneParamLazyParameterFunction(keyBy, context.lazyParameterHelper) with FilterFunction[Context] { override def filter(value: Context): Boolean = evaluateParameter(value) == stringVal }) - .map(ValueWithContext(null, _)) + .map(ValueWithContext[Any](null, _)) }) } @@ -177,7 +177,7 @@ object ProcessTestHelpers { .filter(new AbstractOneParamLazyParameterFunction(keyBy, context.lazyParameterHelper) with FilterFunction[Context] { override def filter(value: Context): Boolean = evaluateParameter(value) == stringVal }) - .map(ValueWithContext(null, _)) + .map(ValueWithContext[Any](null, _)) }) } diff --git a/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/compile/ExpressionCompiler.scala b/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/compile/ExpressionCompiler.scala index e052dee9f0e..c695750b09f 100644 --- a/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/compile/ExpressionCompiler.scala +++ b/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/compile/ExpressionCompiler.scala @@ -1,6 +1,6 @@ package pl.touk.nussknacker.engine.compile -import cats.data.Validated.{Valid, invalid, valid} +import cats.data.Validated.{Invalid, Valid, invalid, valid} import cats.data.ValidatedNel import cats.instances.list._ import pl.touk.nussknacker.engine.api.context.ProcessCompilationError._ @@ -55,6 +55,7 @@ class ExpressionCompiler(expressionParsers: Map[String, ExpressionParser]) { : ValidatedNel[PartSubGraphCompilationError, List[compiledgraph.evaluatedparam.Parameter]] = { compileObjectParameters(parameterDefinitions, parameters, List.empty, ctx, ctx).map(_.map { case TypedParameter(name, expr: TypedExpression) => compiledgraph.evaluatedparam.Parameter(name, expr.expression, expr.returnType, expr.typingInfo) + case TypedParameter(name, expr: TypedExpressionMap) => throw new IllegalArgumentException("Typed expression map should not be here...") }) } diff --git a/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/sql/HsqlSqlQueryableDataBase.scala b/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/sql/HsqlSqlQueryableDataBase.scala index f9e47a77511..4786542c51f 100644 --- a/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/sql/HsqlSqlQueryableDataBase.scala +++ b/engine/interpreter/src/main/scala/pl/touk/nussknacker/engine/sql/HsqlSqlQueryableDataBase.scala @@ -4,10 +4,14 @@ import java.sql._ import java.util.UUID import com.typesafe.scalalogging.LazyLogging +import org.hsqldb.jdbc.JDBCDriver import pl.touk.nussknacker.engine.api.typed.typing.TypedObjectTypingResult import pl.touk.nussknacker.engine.api.typed.{ClazzRef, TypedMap, TypedObjectDefinition, typing} import scala.collection.mutable +import scala.util.Try +import scala.util.control.Exception._ + /** This class is *not* thread safe. One connection is used to handle all operations the idea is that we prepare all tables, and compile all queries during (lazy) initialization @@ -20,8 +24,14 @@ class HsqlSqlQueryableDataBase(query: String, tables: Map[String, ColumnModel]) import HsqlSqlQueryableDataBase._ - private val connection: Connection = - DriverManager.getConnection(s"jdbc:hsqldb:mem:${UUID.randomUUID()};shutdown=true;allow_empty_batch=true", "SA", "") + private val connection: Connection = { + val url = s"jdbc:hsqldb:mem:${UUID.randomUUID()};shutdown=true;allow_empty_batch=true" + //TODO: sometimes in tests something deregisters HsqlSql driver... + if (catching(classOf[SQLException]).opt( DriverManager.getDriver(url)).isEmpty) { + DriverManager.registerDriver(new JDBCDriver) + } + DriverManager.getConnection(url, "SA", "") + } init() diff --git a/engine/interpreter/src/test/scala/pl/touk/nussknacker/engine/types/EspTypeUtilsSpec.scala b/engine/interpreter/src/test/scala/pl/touk/nussknacker/engine/types/EspTypeUtilsSpec.scala index 9d70b2db4e0..42a52e0366c 100644 --- a/engine/interpreter/src/test/scala/pl/touk/nussknacker/engine/types/EspTypeUtilsSpec.scala +++ b/engine/interpreter/src/test/scala/pl/touk/nussknacker/engine/types/EspTypeUtilsSpec.scala @@ -56,7 +56,6 @@ class EspTypeUtilsSpec extends FunSuite with Matchers { } test("should extract public fields from scala case class") { - val infos = TypesInformationExtractor.clazzAndItsChildrenDefinition(List(Typed[SampleClass]))(ClassExtractionSettings.Default) val sampleClassInfo = infos.find(_.clazzName.refClazzName.contains("SampleClass")).get @@ -69,7 +68,7 @@ class EspTypeUtilsSpec extends FunSuite with Matchers { test("shoud detect java beans and fields in java class") { EspTypeUtils.clazzDefinition(classOf[JavaSampleClass])(ClassExtractionSettings.Default).methods shouldBe Map( - "getNotProperty" -> MethodInfo(List(Parameter("arg0", ClazzRef[Int])), ClazzRef[String], None), + "getNotProperty" -> MethodInfo(List(Parameter("foo", ClazzRef[Int])), ClazzRef[String], None), "bar" -> MethodInfo(List(), ClazzRef[String], None), "getBeanProperty" -> MethodInfo(List(), ClazzRef[String], None), "beanProperty" -> MethodInfo(List(), ClazzRef[String], None), @@ -196,10 +195,12 @@ class EspTypeUtilsSpec extends FunSuite with Matchers { val table = Table( ("method", "methodInfo"), - ("foo", MethodInfo(parameters = List(param[String]("arg0")), refClazz = ClazzRef[Long], description = None)), + //FIXME: scala 2.11, 2.12 have different behaviour - named parameters are extracted differently :/ + //("foo", MethodInfo(parameters = List(param[String]("fooParam1")), refClazz = ClazzRef[Long], description = None)), ("bar", MethodInfo(parameters = List(param[Long]("barparam1")), refClazz = ClazzRef[String], description = None)), ("baz", MethodInfo(parameters = List(param[String]("bazparam1"), param[Int]("bazparam2")), refClazz = ClazzRef[Long], description = Some(ScalaSampleDocumentedClass.bazDocs))), - ("qux", MethodInfo(parameters = List(param[String]("arg0")), refClazz = ClazzRef[Long], description = Some(ScalaSampleDocumentedClass.quxDocs))), + //FIXME: scala 2.11, 2.12 have different behaviour - named parameters are extracted differently :/ + //("qux", MethodInfo(parameters = List(param[String]("quxParam1")), refClazz = ClazzRef[Long], description = Some(ScalaSampleDocumentedClass.quxDocs))), ("field1", MethodInfo(parameters = List.empty, refClazz = ClazzRef[Long], description = None)), ("field2", MethodInfo(parameters = List.empty, refClazz = ClazzRef[Long], description = Some(ScalaSampleDocumentedClass.field2Docs))) ) diff --git a/engine/standalone/engine/src/it/resources/application.conf b/engine/standalone/engine/src/it/resources/application.conf deleted file mode 100644 index 6581c5991b1..00000000000 --- a/engine/standalone/engine/src/it/resources/application.conf +++ /dev/null @@ -1,7 +0,0 @@ -standaloneConfig { - jarPath: "./engine/standalone/engine/sample/target/scala-2.11/standaloneSample.jar" -} - -standaloneProcessConfig { - -} \ No newline at end of file diff --git a/engine/standalone/engine/src/it/resources/standalone.conf b/engine/standalone/engine/src/it/resources/standalone.conf new file mode 100644 index 00000000000..219e3b213ba --- /dev/null +++ b/engine/standalone/engine/src/it/resources/standalone.conf @@ -0,0 +1,7 @@ +standaloneConfig { + jarPath: "./engine/standalone/engine/sample/target/scala-"${scala.major.version}"/standaloneSample.jar" +} + +standaloneProcessConfig { + +} \ No newline at end of file diff --git a/engine/standalone/engine/src/it/scala/pl/touk/nussknacker/engine/standalone/management/StandaloneProcessManagerSpec.scala b/engine/standalone/engine/src/it/scala/pl/touk/nussknacker/engine/standalone/management/StandaloneProcessManagerSpec.scala index 002884cb44e..07126112923 100644 --- a/engine/standalone/engine/src/it/scala/pl/touk/nussknacker/engine/standalone/management/StandaloneProcessManagerSpec.scala +++ b/engine/standalone/engine/src/it/scala/pl/touk/nussknacker/engine/standalone/management/StandaloneProcessManagerSpec.scala @@ -1,6 +1,6 @@ package pl.touk.nussknacker.engine.standalone.management -import com.typesafe.config.ConfigFactory +import com.typesafe.config.{ConfigFactory, ConfigParseOptions} import org.scalatest.{FunSuite, Matchers} import org.scalatest.concurrent.ScalaFutures import org.scalatest.time.{Seconds, Span} @@ -14,6 +14,7 @@ import pl.touk.nussknacker.engine.graph.node.{Sink, Source} import pl.touk.nussknacker.engine.graph.sink.SinkRef import pl.touk.nussknacker.engine.graph.source.SourceRef import pl.touk.nussknacker.engine.marshall.ProcessMarshaller +import pl.touk.nussknacker.engine.util.config.ScalaMajorVersionConfig class StandaloneProcessManagerSpec extends FunSuite with ScalaFutures with Matchers { @@ -21,7 +22,9 @@ class StandaloneProcessManagerSpec extends FunSuite with ScalaFutures with Match test("it should parse test data and test standalone process") { - val modelData = StandaloneProcessManagerProvider.defaultTypeConfig(ConfigFactory.load()).toModelData + val config = ScalaMajorVersionConfig.configWithScalaMajorVersion(ConfigFactory.parseResources("standalone.conf")) + val modelData = StandaloneProcessManagerProvider + .defaultTypeConfig(config).toModelData val manager = new StandaloneProcessManager(modelData, null) diff --git a/engine/util/src/main/scala/pl/touk/nussknacker/engine/util/config/ScalaMajorVersionConfig.scala b/engine/util/src/main/scala/pl/touk/nussknacker/engine/util/config/ScalaMajorVersionConfig.scala new file mode 100644 index 00000000000..b22c6ce4f1d --- /dev/null +++ b/engine/util/src/main/scala/pl/touk/nussknacker/engine/util/config/ScalaMajorVersionConfig.scala @@ -0,0 +1,17 @@ +package pl.touk.nussknacker.engine.util.config + +import java.util.Collections + +import com.typesafe.config.{Config, ConfigFactory} + +object ScalaMajorVersionConfig { + + val scalaMajorVersion: String = util.Properties.versionNumberString.replaceAll("(\\d+\\.\\d+)\\..*$", "$1") + + def configWithScalaMajorVersion(config: Config = ConfigFactory.defaultApplication()): Config = { + val withMajor = ConfigFactory.parseMap(Collections.singletonMap("scala.major.version", scalaMajorVersion)) + ConfigFactory.load(config.withFallback(withMajor)) + + } + +} diff --git a/ui/client/webpack.config.js b/ui/client/webpack.config.js index 5d24ffcdc8f..502aecba078 100644 --- a/ui/client/webpack.config.js +++ b/ui/client/webpack.config.js @@ -57,7 +57,8 @@ module.exports = { }, entry: entry, output: { - path: path.join(__dirname, '..', 'server', 'target', 'scala-2.11', 'classes', 'web', 'static'), + //by default we use default webpack value, but we want to be able to override it for building frontend via sbt + path: process.env.OUTPUT_PATH ? path.join(process.env.OUTPUT_PATH, 'classes', 'web', 'static') : path.join(process.cwd(), 'dist'), filename: '[name].js', //see config.js publicPath: isProd ? '__publicPath__/static/' : '/static/', diff --git a/ui/server/src/main/scala/pl/touk/nussknacker/ui/api/UserResources.scala b/ui/server/src/main/scala/pl/touk/nussknacker/ui/api/UserResources.scala index 8eb3aa5ec0b..697e1fc4cea 100644 --- a/ui/server/src/main/scala/pl/touk/nussknacker/ui/api/UserResources.scala +++ b/ui/server/src/main/scala/pl/touk/nussknacker/ui/api/UserResources.scala @@ -28,6 +28,7 @@ object GlobalPermissions { def apply(permissions: List[GlobalPermission]): GlobalPermissions = { permissions.foldLeft(GlobalPermissions(false)) { case (acc, GlobalPermission.AdminTab) => acc.copy(adminTab = true) + case (acc, _) => acc } } } diff --git a/ui/server/src/main/scala/pl/touk/nussknacker/ui/process/ProcessTypesForCategories.scala b/ui/server/src/main/scala/pl/touk/nussknacker/ui/process/ProcessTypesForCategories.scala index d1866cd2c2f..7e42379e16d 100644 --- a/ui/server/src/main/scala/pl/touk/nussknacker/ui/process/ProcessTypesForCategories.scala +++ b/ui/server/src/main/scala/pl/touk/nussknacker/ui/process/ProcessTypesForCategories.scala @@ -20,9 +20,5 @@ class ProcessTypesForCategories(config: Config) { categoriesToTypesMap.get(category) } - val getAllCategories = categoriesToTypesMap.keys.toList -} - -object ProcessTypesForCategories { - def apply(): ProcessTypesForCategories = new ProcessTypesForCategories(ConfigFactory.load()) + val getAllCategories: List[ProcessingType] = categoriesToTypesMap.keys.toList } \ No newline at end of file diff --git a/ui/server/src/test/resources/application.conf b/ui/server/src/test/resources/ui.conf similarity index 89% rename from ui/server/src/test/resources/application.conf rename to ui/server/src/test/resources/ui.conf index 3eb9b3daac8..988ebc920b3 100644 --- a/ui/server/src/test/resources/application.conf +++ b/ui/server/src/test/resources/ui.conf @@ -41,11 +41,11 @@ flinkConfig { restUrl: "localhost:8081" queryableStateProxyUrl: "localhost:9123" jobManagerTimeout: 1m - jarPath: "engine/flink/management/sample/target/scala-2.11/managementSample.jar" + jarPath: "engine/flink/management/sample/target/scala-"${scala.major.version}"/managementSample.jar" } standaloneConfig { - jarPath: "engine/standalone/engine/sample/target/scala-2.11/standaloneSample.jar" + jarPath: "engine/standalone/engine/sample/target/scala-"${scala.major.version}"/standaloneSample.jar" } standaloneProcessConfig { diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/ProcessesNonTechnicalResourcesSpec.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/ProcessesNonTechnicalResourcesSpec.scala index 9879cf68568..6f09c0c3f7c 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/ProcessesNonTechnicalResourcesSpec.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/ProcessesNonTechnicalResourcesSpec.scala @@ -11,6 +11,7 @@ import pl.touk.nussknacker.ui.api.helpers.{EspItTest, ProcessTestData} import pl.touk.nussknacker.ui.api.helpers.TestFactory._ import pl.touk.nussknacker.restmodel.processdetails.{ProcessDetails, ValidatedProcessDetails} import pl.touk.nussknacker.ui.security.api.{LoggedUser, Permission} +import akka.http.scaladsl.testkit.ScalatestRouteTest import scala.language.higherKinds diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/ServiceRoutesSpec.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/ServiceRoutesSpec.scala index c5ab3711728..10c467e40ac 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/ServiceRoutesSpec.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/ServiceRoutesSpec.scala @@ -2,6 +2,7 @@ package pl.touk.nussknacker.ui.api import akka.http.scaladsl.model.{ContentTypes, HttpEntity, MediaTypes, StatusCodes} import akka.http.scaladsl.testkit.ScalatestRouteTest +import pl.touk.nussknacker.ui.util.ConfigWithScalaVersion import com.typesafe.config.ConfigFactory import org.scalatest.{FlatSpec, FunSuite, Matchers} import pl.touk.nussknacker.engine.api.{DisplayJsonWithEncoder, DisplayableAsJson} @@ -22,7 +23,7 @@ class ServiceRoutesSpec extends FunSuite with Matchers with ScalatestRouteTest w private val category1Deploy = Map("Category1" -> Set(Permission.Deploy)) private implicit val user = LoggedUser("admin", category1Deploy) - private val modelData = FlinkStreamingProcessManagerProvider.defaultModelData(ConfigFactory.load()) + private val modelData = FlinkStreamingProcessManagerProvider.defaultModelData(ConfigWithScalaVersion.config) private val serviceRoutes = new ServiceRoutes(Map(TestProcessingTypes.Streaming -> modelData)) implicit val queryResultDecoder: Decoder[QueryResult] = Decoder.decodeJson diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/SslBindingSpec.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/SslBindingSpec.scala index 37e7894f7fe..8c359708881 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/SslBindingSpec.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/SslBindingSpec.scala @@ -10,6 +10,7 @@ import org.scalatest._ import pl.touk.nussknacker.ui.NussknackerApp import pl.touk.nussknacker.ui.NussknackerApp.system import pl.touk.nussknacker.ui.security.ssl.{HttpsConnectionContextFactory, KeyStoreConfig} +import pl.touk.nussknacker.ui.util.ConfigWithScalaVersion import scala.concurrent.Await import scala.concurrent.duration._ @@ -18,7 +19,7 @@ import scala.language.higherKinds class SslBindingSpec extends FlatSpec with Matchers { it should "connect to api via SSL" in { - implicit val system: ActorSystem = ActorSystem("SslBindingSpec") + implicit val system: ActorSystem = ActorSystem("SslBindingSpec", ConfigWithScalaVersion.config) implicit val materializer: Materializer = ActorMaterializer() val route = NussknackerApp.initializeRoute(system.settings.config) diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/helpers/EspItTest.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/helpers/EspItTest.scala index cf6a74507cc..82fc7206870 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/helpers/EspItTest.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/helpers/EspItTest.scala @@ -3,9 +3,10 @@ package pl.touk.nussknacker.ui.api.helpers import akka.actor.ActorRef import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes} import akka.http.scaladsl.testkit.ScalatestRouteTest +import pl.touk.nussknacker.ui.util.ConfigWithScalaVersion import cats.instances.all._ import cats.syntax.semigroup._ -import com.typesafe.config.ConfigFactory +import com.typesafe.config.Config import com.typesafe.scalalogging.LazyLogging import io.circe.{Json, Printer} import org.scalatest._ @@ -30,6 +31,8 @@ import pl.touk.nussknacker.ui.security.api.{DefaultAuthenticationConfiguration, trait EspItTest extends LazyLogging with ScalaFutures with WithHsqlDbTesting with TestPermissions { self: ScalatestRouteTest with Suite with BeforeAndAfterEach with Matchers => + override def testConfig: Config = ConfigWithScalaVersion.config + val env = "test" val attachmentsPath = "/tmp/attachments" + System.currentTimeMillis() @@ -41,7 +44,7 @@ trait EspItTest extends LazyLogging with ScalaFutures with WithHsqlDbTesting wit val deploymentProcessRepository = newDeploymentProcessRepository(db) val processActivityRepository = newProcessActivityRepository(db) - val typesForCategories = ProcessTypesForCategories() + val typesForCategories = new ProcessTypesForCategories(testConfig) val existingProcessingType = "streaming" @@ -71,17 +74,16 @@ trait EspItTest extends LazyLogging with ScalaFutures with WithHsqlDbTesting wit processAuthorizer = processAuthorizer ) - private val config = system.settings.config.withFallback(ConfigFactory.load()) - val authenticationConfig = DefaultAuthenticationConfiguration.create(config) + val authenticationConfig = DefaultAuthenticationConfiguration.create(testConfig) - val featureTogglesConfig = FeatureTogglesConfig.create(config) - val typeToConfig = ProcessingTypeDeps(config, featureTogglesConfig.standaloneMode) + val featureTogglesConfig = FeatureTogglesConfig.create(testConfig) + val typeToConfig = ProcessingTypeDeps(testConfig, featureTogglesConfig.standaloneMode) val usersRoute = new UserResources(typesForCategories) val settingsRoute = new SettingsResources(featureTogglesConfig, typeToConfig, authenticationConfig) val processesExportResources = new ProcessesExportResources(processRepository, processActivityRepository) val definitionResources = new DefinitionResources( - Map(existingProcessingType -> FlinkStreamingProcessManagerProvider.defaultModelData(ConfigFactory.load())), subprocessRepository, typesForCategories) + Map(existingProcessingType -> FlinkStreamingProcessManagerProvider.defaultModelData(testConfig)), subprocessRepository, typesForCategories) val processesRouteWithAllPermissions = withAllPermissions(processesRoute) diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/helpers/TestFactory.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/helpers/TestFactory.scala index c2dd3dab7d3..a1e06528137 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/helpers/TestFactory.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/api/helpers/TestFactory.scala @@ -17,6 +17,7 @@ import pl.touk.nussknacker.ui.db.DbConfig import pl.touk.nussknacker.ui.process.repository.{DBFetchingProcessRepository, _} import pl.touk.nussknacker.ui.process.subprocess.{DbSubprocessRepository, SubprocessDetails, SubprocessRepository, SubprocessResolver} import pl.touk.nussknacker.ui.security.api.{LoggedUser, Permission} +import pl.touk.nussknacker.ui.util.ConfigWithScalaVersion import pl.touk.nussknacker.ui.uiresolving.UIProcessResolving import pl.touk.nussknacker.ui.validation.ProcessValidation @@ -85,7 +86,7 @@ object TestFactory extends TestPermissions{ def adminUser(userName: String = "adminId") = LoggedUser(userName, Map.empty, Nil, isAdmin = true) - class MockProcessManager extends FlinkProcessManager(FlinkStreamingProcessManagerProvider.defaultModelData(ConfigFactory.load()), shouldVerifyBeforeDeploy = false, mainClassName = "UNUSED"){ + class MockProcessManager extends FlinkProcessManager(FlinkStreamingProcessManagerProvider.defaultModelData(ConfigWithScalaVersion.config), shouldVerifyBeforeDeploy = false, mainClassName = "UNUSED"){ override def findJobStatus(name: ProcessName): Future[Option[ProcessState]] = Future.successful( Some(ProcessState(DeploymentId("1"), runningState = managerProcessState.get(), "RUNNING", 0, None))) diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/definition/DefinitionPreparerSpec.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/definition/DefinitionPreparerSpec.scala index 6b852f74c35..343b10f1c80 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/definition/DefinitionPreparerSpec.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/definition/DefinitionPreparerSpec.scala @@ -11,9 +11,12 @@ import pl.touk.nussknacker.restmodel.displayedgraph.displayablenode.EdgeType._ import pl.touk.nussknacker.ui.api.helpers.{ProcessTestData, TestFactory, TestPermissions} import pl.touk.nussknacker.ui.process.ProcessTypesForCategories import pl.touk.nussknacker.ui.process.uiconfig.defaults.{DefaultValueExtractorChain, ParamDefaultValueConfig} +import pl.touk.nussknacker.ui.util.ConfigWithScalaVersion class DefinitionPreparerSpec extends FunSuite with Matchers with TestPermissions{ + private val processTypesForCategories = new ProcessTypesForCategories(ConfigWithScalaVersion.config) + test("return groups sorted by name") { val groups = prepareGroups(Map(), Map("custom" -> "CUSTOM", "sinks"->"BAR")) @@ -106,7 +109,7 @@ class DefinitionPreparerSpec extends FunSuite with Matchers with TestPermissions extractorFactory = DefaultValueExtractorChain(ParamDefaultValueConfig(Map()), ModelClassLoader.empty), nodesConfig = nodesConfig.mapValues(v => SingleNodeConfig(None, None, None, Some(v))), nodeCategoryMapping = nodeCategoryMapping, - typesForCategories = ProcessTypesForCategories() + typesForCategories = processTypesForCategories ) groups } @@ -122,7 +125,7 @@ class DefinitionPreparerSpec extends FunSuite with Matchers with TestPermissions extractorFactory = DefaultValueExtractorChain(ParamDefaultValueConfig(Map()), ModelClassLoader.empty), nodesConfig = Map(), nodeCategoryMapping = Map(), - typesForCategories = ProcessTypesForCategories() + typesForCategories = processTypesForCategories ) groups } diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/definition/UIProcessObjectsSpec.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/definition/UIProcessObjectsSpec.scala index 5bd246e2038..9c9d60005a6 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/definition/UIProcessObjectsSpec.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/definition/UIProcessObjectsSpec.scala @@ -16,6 +16,7 @@ import pl.touk.nussknacker.engine.testing.{EmptyProcessConfigCreator, LocalModel import pl.touk.nussknacker.ui.api.helpers.TestFactory import pl.touk.nussknacker.ui.process.ProcessTypesForCategories import pl.touk.nussknacker.ui.process.subprocess.SubprocessDetails +import pl.touk.nussknacker.ui.util.ConfigWithScalaVersion import scala.concurrent.Future @@ -30,13 +31,14 @@ class UIProcessObjectsSpec extends FunSuite with Matchers { test("should read restrictions from config") { - val model : ModelData = LocalModelData(ConfigFactory.load().getConfig("processConfig"), new EmptyProcessConfigCreator() { + val model : ModelData = LocalModelData(ConfigWithScalaVersion.config.getConfig("processConfig"), new EmptyProcessConfigCreator() { override def services(config: Config): Map[String, WithCategories[Service]] = Map("enricher" -> WithCategories(TestService)) }) val processObjects = - UIProcessObjects.prepareUIProcessObjects(model, TestFactory.user("userId"), Set(), false, ProcessTypesForCategories()) + UIProcessObjects.prepareUIProcessObjects(model, TestFactory.user("userId"), Set(), false, + new ProcessTypesForCategories(ConfigWithScalaVersion.config)) processObjects.nodesConfig("enricher").params shouldBe Some(Map("param" -> ParameterConfig(Some("'default value'"), Some(FixedExpressionValues(List( @@ -68,14 +70,14 @@ class UIProcessObjectsSpec extends FunSuite with Matchers { test("should read restrictions from config for subprocess") { - val model : ModelData = LocalModelData(ConfigFactory.load().getConfig("processConfig"), new EmptyProcessConfigCreator()) + val model : ModelData = LocalModelData(ConfigWithScalaVersion.config.getConfig("processConfig"), new EmptyProcessConfigCreator()) val processObjects = UIProcessObjects.prepareUIProcessObjects(model, TestFactory.user("userId"), Set( SubprocessDetails(CanonicalProcess(MetaData("enricher", null, isSubprocess = true), null, List(FlatNode(SubprocessInputDefinition("", List( SubprocessParameter("param", SubprocessClazzRef[String]) )))), None), "") - ), false, ProcessTypesForCategories()) + ), false, new ProcessTypesForCategories(ConfigWithScalaVersion.config)) processObjects.processDefinition.subprocessInputs("enricher").parameters.map(p => (p.name, p.restriction)).toMap shouldBe Map( "param" -> Some(FixedExpressionValues(List( diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/BaseFlowTest.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/BaseFlowTest.scala index 0aac698d97b..da74647f002 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/BaseFlowTest.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/BaseFlowTest.scala @@ -4,8 +4,11 @@ import java.util.UUID import akka.http.javadsl.model.headers.HttpCredentials import akka.http.scaladsl.model.{ContentTypeRange, ContentTypes, HttpEntity, StatusCodes} +import akka.http.scaladsl.testkit.ScalatestRouteTest +import pl.touk.nussknacker.ui.util.{ConfigWithScalaVersion, MultipartUtils} import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} +import com.typesafe.config.Config import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport import io.circe.{Decoder, Json} import org.scalatest._ @@ -24,22 +27,24 @@ import pl.touk.nussknacker.ui.api.helpers.{TestFactory, TestProcessUtil} import pl.touk.nussknacker.ui.api.helpers.TestProcessingTypes import pl.touk.nussknacker.restmodel.displayedgraph.displayablenode.Edge import pl.touk.nussknacker.restmodel.displayedgraph.{DisplayableProcess, ProcessProperties} -import pl.touk.nussknacker.ui.util.MultipartUtils import pl.touk.nussknacker.restmodel.validation.ValidationResults.ValidationResult import scala.concurrent.duration._ class BaseFlowTest extends FunSuite with ScalatestRouteTest with FailFastCirceSupport with Matchers with ScalaFutures with BeforeAndAfterEach with BeforeAndAfterAll { + override def testConfig: Config = ConfigWithScalaVersion.config + private implicit final val string: FromEntityUnmarshaller[String] = Unmarshaller.stringUnmarshaller.forContentTypes(ContentTypeRange.*) - private val mainRoute = NussknackerApp.initializeRoute(system.settings.config) + private val mainRoute = NussknackerApp.initializeRoute(ConfigWithScalaVersion.config) private val credentials = HttpCredentials.createBasicHttpCredentials("admin", "admin") implicit val timeout: RouteTestTimeout = RouteTestTimeout(1.minute) test("saves, updates and retrieves sample process") { + val processId = UUID.randomUUID().toString val endpoint = s"/api/processes/$processId" @@ -145,7 +150,7 @@ class BaseFlowTest extends FunSuite with ScalatestRouteTest with FailFastCirceSu } } } - + def checkWithClue[T](body: => T): RouteTestResult => T = check { withClue(s"response: '${responseAs[String]}'") { body diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/DictsFlowTest.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/DictsFlowTest.scala index 2f7a0b77ae0..4bf065d61f1 100644 --- a/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/DictsFlowTest.scala +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/integration/DictsFlowTest.scala @@ -6,6 +6,7 @@ import akka.http.javadsl.model.headers.HttpCredentials import akka.http.scaladsl.model.{ContentTypeRange, StatusCodes} import akka.http.scaladsl.testkit.ScalatestRouteTest import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} +import com.typesafe.config.Config import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport import io.circe.Json import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, EitherValues, FunSuite, Matchers, OptionValues} @@ -14,7 +15,7 @@ import pl.touk.nussknacker.engine.build.EspProcessBuilder import pl.touk.nussknacker.engine.graph.EspProcess import pl.touk.nussknacker.ui.NussknackerApp import pl.touk.nussknacker.ui.api.helpers.{TestFactory, TestProcessUtil} -import pl.touk.nussknacker.ui.util.MultipartUtils +import pl.touk.nussknacker.ui.util.{ConfigWithScalaVersion, MultipartUtils} class DictsFlowTest extends FunSuite with ScalatestRouteTest with FailFastCirceSupport with Matchers with ScalaFutures with BeforeAndAfterEach with BeforeAndAfterAll with EitherValues with OptionValues { @@ -27,6 +28,8 @@ class DictsFlowTest extends FunSuite with ScalatestRouteTest with FailFastCirceS private val credentials = HttpCredentials.createBasicHttpCredentials("admin", "admin") + override def testConfig: Config = ConfigWithScalaVersion.config + private val EndNodeId = "end" private val Key = "foo" private val Label = "Foo" diff --git a/ui/server/src/test/scala/pl/touk/nussknacker/ui/util/ConfigWithScalaVersion.scala b/ui/server/src/test/scala/pl/touk/nussknacker/ui/util/ConfigWithScalaVersion.scala new file mode 100644 index 00000000000..28d96f82a2d --- /dev/null +++ b/ui/server/src/test/scala/pl/touk/nussknacker/ui/util/ConfigWithScalaVersion.scala @@ -0,0 +1,10 @@ +package pl.touk.nussknacker.ui.util + + +import com.typesafe.config.{Config, ConfigFactory} +import pl.touk.nussknacker.engine.util.config.ScalaMajorVersionConfig + +object ConfigWithScalaVersion { + + val config: Config = ScalaMajorVersionConfig.configWithScalaMajorVersion(ConfigFactory.parseResources("ui.conf")) +}