From 88005494c6685a979ec9e43f17a8e1111c864de5 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Tue, 14 Jan 2020 14:39:49 -0500 Subject: [PATCH 1/2] Update PDAL deps, make the build cross publish across 2.13, 2.12 and 2.11 --- .gitignore | 3 + .sbtopts | 2 + .travis.yml | 21 +- .travis/run.sh | 2 +- .travis/test-all.sh | 2 +- .travis/test.sh | 4 +- HOWTORELEASE.txt | 3 +- README.md | 12 +- build.sbt | 28 +- .../pipeline/PipelineExpressionsSpec.scala | 6 +- core/src/main/scala/io/pdal/Native.scala | 2 +- core/src/main/scala/io/pdal/Pipeline.scala | 2 +- core/src/main/scala/io/pdal/PointCloud.scala | 16 +- core/src/main/scala/io/pdal/PointLayout.scala | 8 +- .../src/test/scala/io/pdal/PipelineSpec.scala | 2 +- .../test/scala/io/pdal/PointCloudSpec.scala | 6 +- .../scala/io/pdal/TestEnvironmentSpec.scala | 9 +- examples/pdal-jni/build.sbt | 8 +- examples/pdal-jni/sbt | 436 ++++++++++-------- native/src/CMakeLists.txt | 6 +- project/Dependencies.scala | 20 +- project/build.properties | 2 +- project/plugins.sbt | 10 +- sbt | 436 ++++++++++-------- scripts/crosscompile.sh | 2 +- scripts/docker/2.0.0/alpine/Dockerfile | 72 +++ scripts/docker/2.0.0/crossbuild/Dockerfile | 19 + scripts/docker/2.0.0/debian/Dockerfile | 14 + scripts/docker/2.0.0/debian/mbio/Dockerfile | 328 +++++++++++++ scripts/merge-native.sh | 8 +- scripts/publish-211.sh | 32 ++ scripts/publish-all.sh | 1 + scripts/publish-javastyle.sh | 2 +- scripts/publish-local-211.sh | 21 + 34 files changed, 1063 insertions(+), 482 deletions(-) create mode 100644 scripts/docker/2.0.0/alpine/Dockerfile create mode 100644 scripts/docker/2.0.0/crossbuild/Dockerfile create mode 100644 scripts/docker/2.0.0/debian/Dockerfile create mode 100644 scripts/docker/2.0.0/debian/mbio/Dockerfile create mode 100755 scripts/publish-211.sh create mode 100755 scripts/publish-local-211.sh diff --git a/.gitignore b/.gitignore index 3e36596..aa88398 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,9 @@ README.html *.sublime-workspace +# VSCode files +.vscode + # Test data files # java/data diff --git a/.sbtopts b/.sbtopts index 82c41e4..d690174 100644 --- a/.sbtopts +++ b/.sbtopts @@ -2,3 +2,5 @@ -J-Xss512M -Djava.awt.headless=true -Dsun.io.serialization.extendedDebugInfo=true +-Dsbt.color=always +-Dsbt.supershell=false diff --git a/.travis.yml b/.travis.yml index 3ca22b7..f553d67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,28 +3,28 @@ sudo: false matrix: include: - os: linux - jdk: oraclejdk8 + jdk: openjdk8 language: scala scala: - - 2.11.12 + - 2.13.1 before_install: - - docker pull daunnc/pdal-debian:1.8.0 + - docker pull daunnc/pdal-debian:2.0.0 services: - docker - os: linux - jdk: oraclejdk8 + jdk: openjdk8 language: scala scala: - - 2.12.8 + - 2.12.10 before_install: - - docker pull daunnc/pdal-debian:1.8.0 + - docker pull daunnc/pdal-debian:2.0.0 services: - docker - os: osx scala: - - 2.11.12 + - 2.13.1 compiler: - gcc - clang @@ -34,7 +34,7 @@ matrix: - os: osx scala: - - 2.12.8 + - 2.12.10 compiler: - gcc - clang @@ -47,8 +47,11 @@ addons: cache: directories: - - $HOME/.ivy2 + - $HOME/.m2 + - $HOME/.ivy2/cache - $HOME/.sbt + - $HOME/.cache + - $HOME/.cache/coursier - $HOME/Library/Caches/Homebrew script: diff --git a/.travis/run.sh b/.travis/run.sh index 6242ed0..1b6abe0 100755 --- a/.travis/run.sh +++ b/.travis/run.sh @@ -9,5 +9,5 @@ else -v $TRAVIS_BUILD_DIR:/pdal-java \ -e TRAVIS_SCALA_VERSION=$TRAVIS_SCALA_VERSION \ -e TRAVIS_COMMIT=$TRAVIS_COMMIT \ - -e TRAVIS_JDK_VERSION=$TRAVIS_JDK_VERSION daunnc/pdal-debian:1.8.0 /bin/bash -c "cd /pdal-java; .travis/test-all.sh" + -e TRAVIS_JDK_VERSION=$TRAVIS_JDK_VERSION daunnc/pdal-debian:2.0.0 /bin/bash -c "cd /pdal-java; .travis/test-all.sh" fi diff --git a/.travis/test-all.sh b/.travis/test-all.sh index 6308699..f2b21ee 100755 --- a/.travis/test-all.sh +++ b/.travis/test-all.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -if [ `echo $TRAVIS_SCALA_VERSION | cut -f1-2 -d "."` = "2.11" ]; then +if [ `echo $TRAVIS_SCALA_VERSION | cut -f1-2 -d "."` = "2.13" ]; then .travis/test.sh; else .travis/test-212.sh; diff --git a/.travis/test.sh b/.travis/test.sh index 172747b..9e9fe31 100755 --- a/.travis/test.sh +++ b/.travis/test.sh @@ -4,6 +4,6 @@ ./sbt "project core-scala" test || { exit 1; } cd examples/pdal-jni -./sbt "-211" "runMain com.azavea.Main" || { exit 1; } -./sbt "-211" "runMain com.azavea.MainScala" || { exit 1; } +./sbt "runMain com.azavea.Main" || { exit 1; } +./sbt "runMain com.azavea.MainScala" || { exit 1; } cd ~- diff --git a/HOWTORELEASE.txt b/HOWTORELEASE.txt index 4d89aa7..5223898 100644 --- a/HOWTORELEASE.txt +++ b/HOWTORELEASE.txt @@ -26,8 +26,9 @@ Release Process - Publishing snaphots: Snapshot can be published without PGP sign, it is published to a snapshot repo and allows immediate snaphot updates. To publish everything in a local repo use command: - - ./scripts/publish-local.sh (publishes scala 2.11 version) + - ./scripts/publish-local.sh (publishes scala 2.13 version) - ./scripts/publish-local-212.sh (publishes scala 2.12 version) + - ./scripts/publish-local-211.sh (publishes scala 2.11 version) - ./scripts/publish-local-native.sh (publishes native dependencies) To publish everything into sonatype snapshot repo use: - ./scripts/publish-all.sh diff --git a/README.md b/README.md index c38a8a1..3a819e0 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://api.travis-ci.org/PDAL/java.svg)](http://travis-ci.org/PDAL/java) [![Join the chat at https://gitter.im/PDAL/PDAL](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/PDAL/PDAL?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.pdal/pdal/badge.svg)](https://maven-badges.herokuapp.com/maven-central/io.pdal/pdal) -Java bindings to use PDAL on JVM (supports PDAL >= 1.4). +Java bindings to use PDAL on JVM (supports PDAL >= 2.0). Mac users can experience some issues with bindings that were build against a different PDAL version, so try to use a consistent PDAL version. @@ -18,8 +18,8 @@ resolvers ++= Seq( ) libraryDependencies ++= Seq( - "io.pdal" %% "pdal" % "1.9.0", // core library - "io.pdal" % "pdal-native" % "1.9.0" // jni bindings + "io.pdal" %% "pdal" % "2.0.0", // core library + "io.pdal" % "pdal-native" % "2.0.0" // jni bindings ) ``` @@ -27,7 +27,7 @@ It's required to have native JNI binary in `java.library.path`: ```scala // Mac OS X example with manual JNI installation -// cp -f native/target/resource_managed/main/native/x86_64-darwin/libpdaljni.1.4.dylib /usr/local/lib/libpdaljni.1.4.dylib +// cp -f native/target/resource_managed/main/native/x86_64-darwin/libpdaljni.2.0.dylib /usr/local/lib/libpdaljni.2.0.dylib // place built binary into /usr/local/lib, and pass java.library.path to your JVM javaOptions += "-Djava.library.path=/usr/local/lib" ``` @@ -41,8 +41,8 @@ Scala API to build pipeline expressions instead of writing a raw JSON. ```scala libraryDependencies ++= Seq( - "io.pdal" %% "pdal-scala" % "1.9.0", // scala core library - "io.pdal" % "pdal-native" % "1.9.0" // jni bindings + "io.pdal" %% "pdal-scala" % "2.0.0", // scala core library + "io.pdal" % "pdal-native" % "2.0.0" // jni bindings ) ``` diff --git a/build.sbt b/build.sbt index f3cb0e9..a28d14e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,9 +1,9 @@ name := "pdal-jni" lazy val commonSettings = Seq( - version := "1.9.1" + Environment.versionSuffix, - scalaVersion := "2.11.12", - crossScalaVersions := Seq("2.12.8", "2.11.12"), + version := "2.0.0" + Environment.versionSuffix, + scalaVersion := "2.13.1", + crossScalaVersions := Seq("2.13.1", "2.12.10", "2.11.12"), organization := "io.pdal", description := "PDAL JNI bindings", licenses := Seq("BSD" -> url("https://github.com/PDAL/PDAL/blob/master/LICENSE.txt")), @@ -20,13 +20,13 @@ lazy val commonSettings = Seq( "-language:existentials", "-feature" ), - test in assembly := {}, + assembly / test := {}, shellPrompt := { s => Project.extract(s).currentProject.id + " > " }, commands ++= Seq( Commands.processJavastyleCommand("publish"), Commands.processJavastyleCommand("publishSigned") ), - publishArtifact in Test := false, + Test / publishArtifact := false, publishTo := { val nexus = "https://oss.sonatype.org/" if (isSnapshot.value) @@ -47,8 +47,8 @@ lazy val commonSettings = Seq( ), - PgpKeys.useGpg in Global := true, - PgpKeys.gpgCommand in Global := "gpg" + Global / PgpKeys.useGpg := true, + Global / PgpKeys.gpgCommand := "gpg" ) lazy val root = (project in file(".")) @@ -58,12 +58,12 @@ lazy val root = (project in file(".")) lazy val `core-scala` = project .settings(commonSettings: _*) .settings(name := "pdal-scala") - .settings(target in javah := (sourceDirectory in nativeCompile in native).value / "include") + .settings(javah / target := (native / nativeCompile / sourceDirectory).value / "include") .settings(libraryDependencies ++= Seq( - Dependencies.circeCore, - Dependencies.circeGeneric, - Dependencies.circeGenericExtras, - Dependencies.circeParser, + Dependencies.circe("core").value, + Dependencies.circe("generic").value, + Dependencies.circe("generic-extras").value, + Dependencies.circe("parser").value, Dependencies.jtsCore, Dependencies.scalaTest % Test )) @@ -74,7 +74,7 @@ lazy val `core-scala` = project lazy val core = project .settings(commonSettings: _*) .settings(name := "pdal") - .settings(target in javah := (sourceDirectory in nativeCompile in native).value / "include") + .settings(javah / target := (native / nativeCompile / sourceDirectory).value / "include") .settings(libraryDependencies += Dependencies.scalaTest % Test) .dependsOn(Environment.dependOnNative(native % Runtime): _*) @@ -82,7 +82,7 @@ lazy val native = project .settings(commonSettings: _*) .settings(crossPaths := false) .settings(name := "pdal-native") - .settings(sourceDirectory in nativeCompile := sourceDirectory.value) + .settings(nativeCompile / sourceDirectory := sourceDirectory.value) .settings(artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) => artifact.name + "-" + nativePlatform.value + "-" + module.revision + "." + artifact.extension }) diff --git a/core-scala/src/test/scala/io/pdal/pipeline/PipelineExpressionsSpec.scala b/core-scala/src/test/scala/io/pdal/pipeline/PipelineExpressionsSpec.scala index a1b364b..c66f9fa 100644 --- a/core-scala/src/test/scala/io/pdal/pipeline/PipelineExpressionsSpec.scala +++ b/core-scala/src/test/scala/io/pdal/pipeline/PipelineExpressionsSpec.scala @@ -20,9 +20,11 @@ import io.circe._ import io.circe.syntax._ import io.circe.parser._ -import org.scalatest._ +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers -class PipelineExpressionsSpec extends FunSpec with Matchers with BeforeAndAfterAll { +class PipelineExpressionsSpec extends AnyFunSpec with Matchers with BeforeAndAfterAll { describe("Pipeline Expressions spec") { it("should print a correct json, using DSL") { val expected = diff --git a/core/src/main/scala/io/pdal/Native.scala b/core/src/main/scala/io/pdal/Native.scala index 949b0dd..9cb2b5e 100644 --- a/core/src/main/scala/io/pdal/Native.scala +++ b/core/src/main/scala/io/pdal/Native.scala @@ -34,7 +34,7 @@ package io.pdal trait Native { - protected var nativeHandle = 0l // C++ pointer + protected var nativeHandle = 0L // C++ pointer def ptr(): Long = nativeHandle def dispose(): Unit } diff --git a/core/src/main/scala/io/pdal/Pipeline.scala b/core/src/main/scala/io/pdal/Pipeline.scala index ecd1f53..d7fce33 100644 --- a/core/src/main/scala/io/pdal/Pipeline.scala +++ b/core/src/main/scala/io/pdal/Pipeline.scala @@ -50,7 +50,7 @@ class Pipeline(val json: String) extends Native { @native def getLog(): String } -@nativeLoader("pdaljni.1.4") +@nativeLoader("pdaljni.2.0") object Pipeline { def apply(json: String): Pipeline = { val p = new Pipeline(json); p.initialize(); p } } diff --git a/core/src/main/scala/io/pdal/PointCloud.scala b/core/src/main/scala/io/pdal/PointCloud.scala index a5deb18..a3fb935 100644 --- a/core/src/main/scala/io/pdal/PointCloud.scala +++ b/core/src/main/scala/io/pdal/PointCloud.scala @@ -36,22 +36,22 @@ package io.pdal import java.nio.{ByteBuffer, ByteOrder} import java.util -import scala.collection.JavaConversions._ +import scala.collection.JavaConverters._ /** * PointCloud abstraction to work with packed point(s) in JVM memory. * SizedDimType contains size and offset for a particular packed point with the current set of dims. **/ case class PointCloud(bytes: Array[Byte], dimTypes: util.Map[String, SizedDimType]) { - val pointSize: Int = dimTypes.values.map(_.size).sum.toInt + val pointSize: Int = dimTypes.values.asScala.map(_.size).sum.toInt val length: Int = bytes.length / pointSize val isPoint: Boolean = length == 1 - def dimSize(dim: SizedDimType) = dimTypes(dim.dimType.id).size - def dimSize(dim: DimType) = dimTypes(dim.id).size - def dimSize(dim: String) = dimTypes(dim).size - def findDimType(dim: String) = dimTypes(dim).dimType - def findSizedDimType(dim: String) = dimTypes(dim) + def dimSize(dim: SizedDimType): Long = dimTypes.asScala(dim.dimType.id).size + def dimSize(dim: DimType): Long = dimTypes.asScala(dim.id).size + def dimSize(dim: String): Long = dimTypes.asScala(dim).size + def findDimType(dim: String): DimType = dimTypes.asScala(dim).dimType + def findSizedDimType(dim: String): SizedDimType = dimTypes.asScala(dim) /** * Reads a packed point by point id from a set of packed points. @@ -114,7 +114,7 @@ case class PointCloud(bytes: Array[Byte], dimTypes: util.Map[String, SizedDimTyp * Reads dim from a packed point. */ def get(packedPoint: Array[Byte], dim: String): Array[Byte] = { - val sdt = dimTypes(dim) + val sdt = dimTypes.asScala(dim) val from = sdt.offset.toInt val dimSize = sdt.size.toInt val result = new Array[Byte](dimSize) diff --git a/core/src/main/scala/io/pdal/PointLayout.scala b/core/src/main/scala/io/pdal/PointLayout.scala index 1585dec..f73dd38 100644 --- a/core/src/main/scala/io/pdal/PointLayout.scala +++ b/core/src/main/scala/io/pdal/PointLayout.scala @@ -34,7 +34,7 @@ package io.pdal import java.util -import scala.collection.JavaConversions._ +import scala.collection.JavaConverters._ class PointLayout extends Native { def dimSize(dimType: DimType): Long = dimSize(dimType.id) @@ -42,8 +42,8 @@ class PointLayout extends Native { def sizedDimTypes(): util.Map[String, SizedDimType] = toSizedDimTypes(dimTypes()) def toSizedDimTypes(dimTypes: Array[DimType]): util.Map[String, SizedDimType] = { - var (i, offset, length) = (0, 0l, dimTypes.length) - val result = new util.HashMap[String, SizedDimType]() + var (i, offset, length) = (0, 0L, dimTypes.length) + val result = new util.HashMap[String, SizedDimType]().asScala while(i < length) { val dt = dimTypes(i) val size = dimSize(dt) @@ -51,7 +51,7 @@ class PointLayout extends Native { offset += size i += 1 } - result + result.asJava } @native def dimTypes(): Array[DimType] diff --git a/core/src/test/scala/io/pdal/PipelineSpec.scala b/core/src/test/scala/io/pdal/PipelineSpec.scala index 9779491..f1be56d 100644 --- a/core/src/test/scala/io/pdal/PipelineSpec.scala +++ b/core/src/test/scala/io/pdal/PipelineSpec.scala @@ -169,7 +169,7 @@ class PipelineSpec extends TestEnvironmentSpec { it("should read crs correct") { val pvi = pipeline.getPointViews() val pv = pvi.next() - pv.getCrsProj4 should be (proj4String) + pv.getCrsProj4 should (be (proj4String) or be(proj4StringNew)) pv.dispose() pvi.dispose() } diff --git a/core/src/test/scala/io/pdal/PointCloudSpec.scala b/core/src/test/scala/io/pdal/PointCloudSpec.scala index 4d18083..2bc00a8 100644 --- a/core/src/test/scala/io/pdal/PointCloudSpec.scala +++ b/core/src/test/scala/io/pdal/PointCloudSpec.scala @@ -35,7 +35,7 @@ package io.pdal import java.nio.{ByteBuffer, ByteOrder} -import scala.collection.JavaConversions._ +import scala.collection.JavaConverters._ class PointCloudSpec extends TestEnvironmentSpec { var packedPoints: PointCloud = _ @@ -116,7 +116,7 @@ class PointCloudSpec extends TestEnvironmentSpec { it("should read correctly data as a packed point") { val pvi = pipeline.getPointViews() val pv = pvi.next() - packedPoints.dimTypes.foreach { case (_, sdt) => + packedPoints.dimTypes.asScala.foreach { case (_, sdt) => pv.get(0, sdt.dimType) should be (packedPoints.get(0, sdt)) } pv.dispose() @@ -162,7 +162,7 @@ class PointCloudSpec extends TestEnvironmentSpec { val pvi = pipeline.getPointViews() val pv = pvi.next() val length = pv.length - val dimTypes = packedPoints.dimTypes.values().map(_.dimType) + val dimTypes = packedPoints.dimTypes.values().asScala.map(_.dimType) for (i <- 0 until length) { packedPoints.get(i) should be (pv.getPackedPoint(i)) packedPoints.getX(i) should be (pv.getX(i)) diff --git a/core/src/test/scala/io/pdal/TestEnvironmentSpec.scala b/core/src/test/scala/io/pdal/TestEnvironmentSpec.scala index 23c51ea..9ecb489 100644 --- a/core/src/test/scala/io/pdal/TestEnvironmentSpec.scala +++ b/core/src/test/scala/io/pdal/TestEnvironmentSpec.scala @@ -33,9 +33,11 @@ package io.pdal -import org.scalatest._ +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers -trait TestEnvironmentSpec extends FunSpec with Matchers with BeforeAndAfterAll { +trait TestEnvironmentSpec extends AnyFunSpec with Matchers with BeforeAndAfterAll { def getJson(resource: String): String = { val stream = getClass.getResourceAsStream(resource) val lines = scala.io.Source.fromInputStream(stream).getLines @@ -58,7 +60,8 @@ trait TestEnvironmentSpec extends FunSpec with Matchers with BeforeAndAfterAll { |} """.stripMargin - val proj4String = "+proj=lcc +lat_1=43 +lat_2=45.5 +lat_0=41.75 +lon_0=-120.5 +x_0=400000 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs" + val proj4String = "+proj=lcc +lat_1=43 +lat_2=45.5 +lat_0=41.75 +lon_0=-120.5 +x_0=400000 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs" + val proj4StringNew = "+proj=lcc +lat_0=41.75 +lon_0=-120.5 +lat_1=43 +lat_2=45.5 +x_0=400000 +y_0=0 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs" val pipeline: Pipeline = Pipeline(json) diff --git a/examples/pdal-jni/build.sbt b/examples/pdal-jni/build.sbt index 1698085..aec0152 100644 --- a/examples/pdal-jni/build.sbt +++ b/examples/pdal-jni/build.sbt @@ -1,7 +1,7 @@ name := "pdal-jni" version := "0.1.0-SNAPSHOT" -scalaVersion := "2.12.8" -crossScalaVersions := Seq("2.12.8", "2.11.12") +scalaVersion := "2.13.1" +crossScalaVersions := Seq("2.13.1", "2.12.10", "2.11.12") organization := "com.azavea" scalacOptions ++= Seq( "-deprecation", @@ -21,11 +21,11 @@ resolvers ++= Seq( fork := true -val pdalVersion = "1.9.0" +val pdalVersion = "2.0.0" libraryDependencies ++= Seq( "io.pdal" %% "pdal" % pdalVersion, "io.pdal" %% "pdal-scala" % pdalVersion, "io.pdal" % "pdal-native" % pdalVersion, - "org.scalatest" %% "scalatest" % "3.0.7" % Test + "org.scalatest" %% "scalatest" % "3.1.0" % Test ) diff --git a/examples/pdal-jni/sbt b/examples/pdal-jni/sbt index a14731f..2fad435 100755 --- a/examples/pdal-jni/sbt +++ b/examples/pdal-jni/sbt @@ -6,11 +6,11 @@ set -o pipefail -declare -r sbt_release_version="1.2.8" -declare -r sbt_unreleased_version="1.2.8" +declare -r sbt_release_version="1.3.6" +declare -r sbt_unreleased_version="1.3.6" -declare -r latest_213="2.13.0-M5" -declare -r latest_212="2.12.8" +declare -r latest_213="2.13.1" +declare -r latest_212="2.12.10" declare -r latest_211="2.11.12" declare -r latest_210="2.10.7" declare -r latest_29="2.9.3" @@ -18,18 +18,17 @@ declare -r latest_28="2.8.2" declare -r buildProps="project/build.properties" -declare -r sbt_launch_ivy_release_repo="http://repo.typesafe.com/typesafe/ivy-releases" +declare -r sbt_launch_ivy_release_repo="https://repo.typesafe.com/typesafe/ivy-releases" declare -r sbt_launch_ivy_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots" -declare -r sbt_launch_mvn_release_repo="http://repo.scala-sbt.org/scalasbt/maven-releases" -declare -r sbt_launch_mvn_snapshot_repo="http://repo.scala-sbt.org/scalasbt/maven-snapshots" +declare -r sbt_launch_mvn_release_repo="https://repo.scala-sbt.org/scalasbt/maven-releases" +declare -r sbt_launch_mvn_snapshot_repo="https://repo.scala-sbt.org/scalasbt/maven-snapshots" -declare -r default_jvm_opts_common="-Xms512m -Xss2m" +declare -r default_jvm_opts_common="-Xms512m -Xss2m -XX:MaxInlineLevel=18" declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" declare sbt_jar sbt_dir sbt_create sbt_version sbt_script sbt_new declare sbt_explicit_version declare verbose noshare batch trace_level -declare debugUs declare java_cmd="java" declare sbt_launch_dir="$HOME/.sbt/launchers" @@ -41,11 +40,14 @@ declare -a java_args scalac_args sbt_commands residual_args # args to jvm/sbt via files or environment variables declare -a extra_jvm_opts extra_sbt_opts -echoerr () { echo >&2 "$@"; } -vlog () { [[ -n "$verbose" ]] && echoerr "$@"; } -die () { echo "Aborting: $*" ; exit 1; } +echoerr() { echo >&2 "$@"; } +vlog() { [[ -n "$verbose" ]] && echoerr "$@"; } +die() { + echo "Aborting: $*" + exit 1 +} -setTrapExit () { +setTrapExit() { # save stty and trap exit, to ensure echo is re-enabled if we are interrupted. SBT_STTY="$(stty -g 2>/dev/null)" export SBT_STTY @@ -64,9 +66,12 @@ setTrapExit () { # this seems to cover the bases on OSX, and someone will # have to tell me about the others. -get_script_path () { +get_script_path() { local path="$1" - [[ -L "$path" ]] || { echo "$path" ; return; } + [[ -L "$path" ]] || { + echo "$path" + return + } local -r target="$(readlink "$path")" if [[ "${target:0:1}" == "/" ]]; then @@ -81,7 +86,7 @@ declare -r script_path script_name="${script_path##*/}" declare -r script_name -init_default_option_file () { +init_default_option_file() { local overriding_var="${!1}" local default_file="$2" if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then @@ -96,64 +101,78 @@ init_default_option_file () { sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" -build_props_sbt () { - [[ -r "$buildProps" ]] && \ +build_props_sbt() { + [[ -r "$buildProps" ]] && grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }' } -set_sbt_version () { +set_sbt_version() { sbt_version="${sbt_explicit_version:-$(build_props_sbt)}" [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version export sbt_version } -url_base () { +url_base() { local version="$1" case "$version" in - 0.7.*) echo "http://simple-build-tool.googlecode.com" ;; - 0.10.* ) echo "$sbt_launch_ivy_release_repo" ;; + 0.7.*) echo "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/simple-build-tool" ;; + 0.10.*) echo "$sbt_launch_ivy_release_repo" ;; 0.11.[12]) echo "$sbt_launch_ivy_release_repo" ;; 0.*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" - echo "$sbt_launch_ivy_snapshot_repo" ;; - 0.*) echo "$sbt_launch_ivy_release_repo" ;; - *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" - echo "$sbt_launch_mvn_snapshot_repo" ;; - *) echo "$sbt_launch_mvn_release_repo" ;; + echo "$sbt_launch_ivy_snapshot_repo" ;; + 0.*) echo "$sbt_launch_ivy_release_repo" ;; + *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmddThhMMss" + echo "$sbt_launch_mvn_snapshot_repo" ;; + *) echo "$sbt_launch_mvn_release_repo" ;; esac } -make_url () { +make_url() { local version="$1" local base="${sbt_launch_repo:-$(url_base "$version")}" case "$version" in - 0.7.*) echo "$base/files/sbt-launch-0.7.7.jar" ;; - 0.10.* ) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; + 0.7.*) echo "$base/sbt-launch-0.7.7.jar" ;; + 0.10.*) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; - 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; - *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch-${version}.jar" ;; esac } -addJava () { vlog "[addJava] arg = '$1'" ; java_args+=("$1"); } -addSbt () { vlog "[addSbt] arg = '$1'" ; sbt_commands+=("$1"); } -addScalac () { vlog "[addScalac] arg = '$1'" ; scalac_args+=("$1"); } -addResidual () { vlog "[residual] arg = '$1'" ; residual_args+=("$1"); } +addJava() { + vlog "[addJava] arg = '$1'" + java_args+=("$1") +} +addSbt() { + vlog "[addSbt] arg = '$1'" + sbt_commands+=("$1") +} +addScalac() { + vlog "[addScalac] arg = '$1'" + scalac_args+=("$1") +} +addResidual() { + vlog "[residual] arg = '$1'" + residual_args+=("$1") +} + +addResolver() { addSbt "set resolvers += $1"; } -addResolver () { addSbt "set resolvers += $1"; } -addDebugger () { addJava "-Xdebug" ; addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } -setThisBuild () { +addDebugger() { addJava "-Xdebug" && addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } + +setThisBuild() { vlog "[addBuild] args = '$*'" local key="$1" && shift addSbt "set $key in ThisBuild := $*" } -setScalaVersion () { +setScalaVersion() { [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")' addSbt "++ $1" } -setJavaHome () { +setJavaHome() { java_cmd="$1/bin/java" setThisBuild javaHome "_root_.scala.Some(file(\"$1\"))" export JAVA_HOME="$1" @@ -166,9 +185,9 @@ getJavaVersion() { # java -version on java8 says 1.8.x # but on 9 and 10 it's 9.x.y and 10.x.y. - if [[ "$str" =~ ^1\.([0-9]+)\..*$ ]]; then + if [[ "$str" =~ ^1\.([0-9]+)(\..*)?$ ]]; then echo "${BASH_REMATCH[1]}" - elif [[ "$str" =~ ^([0-9]+)\..*$ ]]; then + elif [[ "$str" =~ ^([0-9]+)(\..*)?$ ]]; then echo "${BASH_REMATCH[1]}" elif [[ -n "$str" ]]; then echoerr "Can't parse java version from: $str" @@ -178,8 +197,8 @@ getJavaVersion() { checkJava() { # Warn if there is a Java version mismatch between PATH and JAVA_HOME/JDK_HOME - [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java" - [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java" + [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java" + [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java" if [[ -n "$java" ]]; then pathJavaVersion=$(getJavaVersion java) @@ -193,23 +212,25 @@ checkJava() { fi } -java_version () { +java_version() { local -r version=$(getJavaVersion "$java_cmd") vlog "Detected Java version: $version" echo "$version" } # MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+ -default_jvm_opts () { +default_jvm_opts() { local -r v="$(java_version)" - if [[ $v -ge 8 ]]; then + if [[ $v -ge 10 ]]; then + echo "$default_jvm_opts_common -XX:+UnlockExperimentalVMOptions -XX:+UseJVMCICompiler" + elif [[ $v -ge 8 ]]; then echo "$default_jvm_opts_common" else echo "-XX:MaxPermSize=384m $default_jvm_opts_common" fi } -build_props_scala () { +build_props_scala() { if [[ -r "$buildProps" ]]; then versionLine="$(grep '^build.scala.versions' "$buildProps")" versionString="${versionLine##build.scala.versions=}" @@ -217,7 +238,7 @@ build_props_scala () { fi } -execRunner () { +execRunner() { # print the arguments one to a line, quoting any containing spaces vlog "# Executing command line:" && { for arg; do @@ -235,40 +256,36 @@ execRunner () { setTrapExit if [[ -n "$batch" ]]; then - "$@" < /dev/null + "$@" /dev/null 2>&1; then + if command -v curl >/dev/null 2>&1; then curl --fail --silent --location "$url" --output "$jar" - elif command -v wget > /dev/null 2>&1; then + elif command -v wget >/dev/null 2>&1; then wget -q -O "$jar" "$url" fi } && [[ -r "$jar" ]] } -acquire_sbt_jar () { +acquire_sbt_jar() { { sbt_jar="$(jar_file "$sbt_version")" [[ -r "$sbt_jar" ]] @@ -277,11 +294,66 @@ acquire_sbt_jar () { [[ -r "$sbt_jar" ]] } || { sbt_jar="$(jar_file "$sbt_version")" - download_url "$(make_url "$sbt_version")" "$sbt_jar" + jar_url="$(make_url "$sbt_version")" + + echoerr "Downloading sbt launcher for ${sbt_version}:" + echoerr " From ${jar_url}" + echoerr " To ${sbt_jar}" + + download_url "${jar_url}" "${sbt_jar}" + + case "${sbt_version}" in + 0.*) + vlog "SBT versions < 1.0 do not have published MD5 checksums, skipping check" + echo "" + ;; + *) verify_sbt_jar "${sbt_jar}" ;; + esac } } -usage () { +verify_sbt_jar() { + local jar="${1}" + local md5="${jar}.md5" + md5url="$(make_url "${sbt_version}").md5" + + echoerr "Downloading sbt launcher ${sbt_version} md5 hash:" + echoerr " From ${md5url}" + echoerr " To ${md5}" + + download_url "${md5url}" "${md5}" >/dev/null 2>&1 + + if command -v md5sum >/dev/null 2>&1; then + if echo "$(cat "${md5}") ${jar}" | md5sum -c -; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v md5 >/dev/null 2>&1; then + if [ "$(md5 -q "${jar}")" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v openssl >/dev/null 2>&1; then + if [ "$(openssl md5 -r "${jar}" | awk '{print $1}')" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + else + echoerr "Could not find an MD5 command" + return 1 + fi +} + +usage() { set_sbt_version cat < Run the specified file as a scala script # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version) - -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version - -sbt-version use the specified version of sbt (default: $sbt_release_version) - -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version - -sbt-jar use the specified jar as the sbt launcher - -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) - -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) + -sbt-version use the specified version of sbt (default: $sbt_release_version) + -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version + -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version + -sbt-jar use the specified jar as the sbt launcher + -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) + -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) # scala version (default: as chosen by sbt) - -28 use $latest_28 - -29 use $latest_29 - -210 use $latest_210 - -211 use $latest_211 - -212 use $latest_212 - -213 use $latest_213 - -scala-home use the scala build at the specified directory - -scala-version use the specified version of scala - -binary-version use the specified scala version when searching for dependencies + -28 use $latest_28 + -29 use $latest_29 + -210 use $latest_210 + -211 use $latest_211 + -212 use $latest_212 + -213 use $latest_213 + -scala-home use the scala build at the specified directory + -scala-version use the specified version of scala + -binary-version use the specified scala version when searching for dependencies # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) - -java-home alternate JAVA_HOME + -java-home alternate JAVA_HOME # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found - $(default_jvm_opts) - JVM_OPTS environment variable holding either the jvm args directly, or - the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') - Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. - -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) - -Dkey=val pass -Dkey=val directly to the jvm - -J-X pass option -X directly to the jvm (-J is stripped) + $(default_jvm_opts) + JVM_OPTS environment variable holding either the jvm args directly, or + the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') + Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. + -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) + -Dkey=val pass -Dkey=val directly to the jvm + -J-X pass option -X directly to the jvm (-J is stripped) # passing options to sbt, OR to this runner - SBT_OPTS environment variable holding either the sbt args directly, or - the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') - Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. - -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) - -S-X add -X to sbt's scalacOptions (-S is stripped) + SBT_OPTS environment variable holding either the sbt args directly, or + the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') + Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. + -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) + -S-X add -X to sbt's scalacOptions (-S is stripped) EOM + exit 0 } -process_args () { - require_arg () { +process_args() { + require_arg() { local type="$1" local opt="$2" local arg="$3" @@ -368,50 +435,55 @@ process_args () { } while [[ $# -gt 0 ]]; do case "$1" in - -h|-help) usage; exit 0 ;; - -v) verbose=true && shift ;; - -d) addSbt "--debug" && shift ;; - -w) addSbt "--warn" && shift ;; - -q) addSbt "--error" && shift ;; - -x) debugUs=true && shift ;; - -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; - -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; - -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; - -no-share) noshare=true && shift ;; - -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; - -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; - -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; - -offline) addSbt "set offline in Global := true" && shift ;; - -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; - -batch) batch=true && shift ;; - -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; - -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;; - - -sbt-create) sbt_create=true && shift ;; - -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; + -h | -help) usage ;; + -v) verbose=true && shift ;; + -d) addSbt "--debug" && shift ;; + -w) addSbt "--warn" && shift ;; + -q) addSbt "--error" && shift ;; + -x) shift ;; # currently unused + -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; + -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; + + -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; + -sbt-create) sbt_create=true && shift ;; + -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; + -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; + -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; + -no-share) noshare=true && shift ;; + -offline) addSbt "set offline in Global := true" && shift ;; + -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; + -batch) batch=true && shift ;; + -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; + -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;; + -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;; - -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; - -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; - -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; - -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; - -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; - -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; - -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;; - -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; - -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; - -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; - - -D*) addJava "$1" && shift ;; - -J*) addJava "${1:2}" && shift ;; - -S*) addScalac "${1:2}" && shift ;; - -28) setScalaVersion "$latest_28" && shift ;; - -29) setScalaVersion "$latest_29" && shift ;; - -210) setScalaVersion "$latest_210" && shift ;; - -211) setScalaVersion "$latest_211" && shift ;; - -212) setScalaVersion "$latest_212" && shift ;; - -213) setScalaVersion "$latest_213" && shift ;; - new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;; - *) addResidual "$1" && shift ;; + -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; + -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; + -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; + -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; + -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; + + -28) setScalaVersion "$latest_28" && shift ;; + -29) setScalaVersion "$latest_29" && shift ;; + -210) setScalaVersion "$latest_210" && shift ;; + -211) setScalaVersion "$latest_211" && shift ;; + -212) setScalaVersion "$latest_212" && shift ;; + -213) setScalaVersion "$latest_213" && shift ;; + + -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; + -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; + -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;; + -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; + -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; + -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; + + -D*) addJava "$1" && shift ;; + -J*) addJava "${1:2}" && shift ;; + -S*) addScalac "${1:2}" && shift ;; + + new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;; + + *) addResidual "$1" && shift ;; esac done } @@ -425,7 +497,7 @@ readConfigFile() { until $end; do read -r || end=true [[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo "$REPLY" - done < "$1" + done <"$1" } # if there are file/environment sbt_opts, process again so we @@ -435,7 +507,7 @@ if [[ -r "$sbt_opts_file" ]]; then while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file") elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then vlog "Using sbt options defined in variable \$SBT_OPTS" - IFS=" " read -r -a extra_sbt_opts <<< "$SBT_OPTS" + IFS=" " read -r -a extra_sbt_opts <<<"$SBT_OPTS" else vlog "No extra sbt options have been defined" fi @@ -454,8 +526,8 @@ checkJava # only exists in 0.12+ setTraceLevel() { case "$sbt_version" in - "0.7."* | "0.10."* | "0.11."* ) echoerr "Cannot set trace level in sbt version $sbt_version" ;; - *) setThisBuild traceLevel "$trace_level" ;; + "0.7."* | "0.10."* | "0.11."*) echoerr "Cannot set trace level in sbt version $sbt_version" ;; + *) setThisBuild traceLevel "$trace_level" ;; esac } @@ -466,12 +538,12 @@ setTraceLevel() { vlog "Detected sbt version $sbt_version" if [[ -n "$sbt_script" ]]; then - residual_args=( "$sbt_script" "${residual_args[@]}" ) + residual_args=("$sbt_script" "${residual_args[@]}") else # no args - alert them there's stuff in here - (( argumentCount > 0 )) || { + ((argumentCount > 0)) || { vlog "Starting $script_name: invoke with -help for other options" - residual_args=( shell ) + residual_args=(shell) } fi @@ -497,7 +569,7 @@ EOM # no jar? download it. [[ -r "$sbt_jar" ]] || acquire_sbt_jar || { # still no jar? uh-oh. - echo "Download failed. Obtain the jar manually and place it at $sbt_jar" + echo "Could not download and verify the launcher. Obtain the jar manually and place it at $sbt_jar" exit 1 } @@ -507,12 +579,12 @@ if [[ -n "$noshare" ]]; then done else case "$sbt_version" in - "0.7."* | "0.10."* | "0.11."* | "0.12."* ) + "0.7."* | "0.10."* | "0.11."* | "0.12."*) [[ -n "$sbt_dir" ]] || { sbt_dir="$HOME/.sbt/$sbt_version" vlog "Using $sbt_dir as sbt dir, -sbt-dir to override." } - ;; + ;; esac if [[ -n "$sbt_dir" ]]; then @@ -525,54 +597,18 @@ if [[ -r "$jvm_opts_file" ]]; then while read -r opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file") elif [[ -n "$JVM_OPTS" && ! ("$JVM_OPTS" =~ ^@.*) ]]; then vlog "Using jvm options defined in \$JVM_OPTS variable" - IFS=" " read -r -a extra_jvm_opts <<< "$JVM_OPTS" + IFS=" " read -r -a extra_jvm_opts <<<"$JVM_OPTS" else vlog "Using default jvm options" - IFS=" " read -r -a extra_jvm_opts <<< "$(default_jvm_opts)" + IFS=" " read -r -a extra_jvm_opts <<<"$( default_jvm_opts)" fi # traceLevel is 0.12+ [[ -n "$trace_level" ]] && setTraceLevel -main () { - execRunner "$java_cmd" \ - "${extra_jvm_opts[@]}" \ - "${java_args[@]}" \ - -jar "$sbt_jar" \ - "${sbt_commands[@]}" \ - "${residual_args[@]}" -} - -# sbt inserts this string on certain lines when formatting is enabled: -# val OverwriteLine = "\r\u001BM\u001B[2K" -# ...in order not to spam the console with a million "Resolving" lines. -# Unfortunately that makes it that much harder to work with when -# we're not going to print those lines anyway. We strip that bit of -# line noise, but leave the other codes to preserve color. -mainFiltered () { - local -r excludeRegex=$(grep -E -v '^#|^$' ~/.sbtignore | paste -sd'|' -) - - echoLine () { - local -r line="$1" - local -r line1="${line//\r\x1BM\x1B\[2K//g}" # This strips the OverwriteLine code. - local -r line2="${line1//\x1B\[[0-9;]*[JKmsu]//g}" # This strips all codes - we test regexes against this. - - if [[ $line2 =~ $excludeRegex ]]; then - [[ -n $debugUs ]] && echo "[X] $line1" - else - [[ -n $debugUs ]] && echo " $line1" || echo "$line1" - fi - } - - echoLine "Starting sbt with output filtering enabled." - main | while read -r line; do echoLine "$line"; done -} - -# Only filter if there's a filter file and we don't see a known interactive command. -# Obviously this is super ad hoc but I don't know how to improve on it. Testing whether -# stdin is a terminal is useless because most of my use cases for this filtering are -# exactly when I'm at a terminal, running sbt non-interactively. -shouldFilter () { [[ -f ~/.sbtignore ]] && ! grep -E -q '\b(shell|console|consoleProject)\b' <<<"${residual_args[@]}"; } - -# run sbt -if shouldFilter; then mainFiltered; else main; fi +execRunner "$java_cmd" \ + "${extra_jvm_opts[@]}" \ + "${java_args[@]}" \ + -jar "$sbt_jar" \ + "${sbt_commands[@]}" \ + "${residual_args[@]}" diff --git a/native/src/CMakeLists.txt b/native/src/CMakeLists.txt index e216478..c8fc26f 100644 --- a/native/src/CMakeLists.txt +++ b/native/src/CMakeLists.txt @@ -8,8 +8,8 @@ set(MAKE_COLOR_MAKEFILE ON) # project (pdaljni) -set(PROJECT_VERSION_MAJOR 1) -set(PROJECT_VERSION_MINOR 4) +set(PROJECT_VERSION_MAJOR 2) +set(PROJECT_VERSION_MINOR 0) set(PROJECT_VERSION_PATCH 0) set(PDAL_LIB_NAME pdalcpp) @@ -32,7 +32,7 @@ if (JNI_FOUND) endif() if (NOT PDAL_BUILD) - find_package(PDAL 1.0.0 REQUIRED CONFIG) + find_package(PDAL 2.0.0 REQUIRED CONFIG) endif() # Include directories diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2d7c082..6397747 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,11 +1,19 @@ import sbt._ +import sbt.Keys._ object Dependencies { - val circeCore = "io.circe" %% "circe-core" % Version.circe - val circeGeneric = "io.circe" %% "circe-generic" % Version.circe - val circeGenericExtras = "io.circe" %% "circe-generic-extras" % Version.circe - val circeParser = "io.circe" %% "circe-parser" % Version.circe + private def ver(for211: String, for213: String) = Def.setting { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, 11)) => for211 + case Some((2, 12)) | Some((2, 13)) => for213 + case _ => sys.error("not good") + } + } - val jtsCore = "org.locationtech.jts" % "jts-core" % Version.jtsCore - val scalaTest = "org.scalatest" %% "scalatest" % Version.scalaTest + def circe(module: String) = Def.setting { + "io.circe" %% s"circe-$module" % ver("0.11.2", "0.12.2").value + } + + val jtsCore = "org.locationtech.jts" % "jts-core" % "1.16.1" + val scalaTest = "org.scalatest" %% "scalatest" % "3.1.0" } diff --git a/project/build.properties b/project/build.properties index c0bab04..00b48d9 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.8 +sbt.version=1.3.6 diff --git a/project/plugins.sbt b/project/plugins.sbt index 5c010f7..6d4ca2b 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,5 +1,5 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9") -addSbtPlugin("ch.jodersky" % "sbt-jni" % "1.3.2") -addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.2") -addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.4.0") -addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.2.0") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") +addSbtPlugin("ch.jodersky" % "sbt-jni" % "1.3.4") +addSbtPlugin("com.jsuereth" % "sbt-pgp" % "2.0.1") +addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.5.0") +addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.4.0") diff --git a/sbt b/sbt index a14731f..2fad435 100755 --- a/sbt +++ b/sbt @@ -6,11 +6,11 @@ set -o pipefail -declare -r sbt_release_version="1.2.8" -declare -r sbt_unreleased_version="1.2.8" +declare -r sbt_release_version="1.3.6" +declare -r sbt_unreleased_version="1.3.6" -declare -r latest_213="2.13.0-M5" -declare -r latest_212="2.12.8" +declare -r latest_213="2.13.1" +declare -r latest_212="2.12.10" declare -r latest_211="2.11.12" declare -r latest_210="2.10.7" declare -r latest_29="2.9.3" @@ -18,18 +18,17 @@ declare -r latest_28="2.8.2" declare -r buildProps="project/build.properties" -declare -r sbt_launch_ivy_release_repo="http://repo.typesafe.com/typesafe/ivy-releases" +declare -r sbt_launch_ivy_release_repo="https://repo.typesafe.com/typesafe/ivy-releases" declare -r sbt_launch_ivy_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots" -declare -r sbt_launch_mvn_release_repo="http://repo.scala-sbt.org/scalasbt/maven-releases" -declare -r sbt_launch_mvn_snapshot_repo="http://repo.scala-sbt.org/scalasbt/maven-snapshots" +declare -r sbt_launch_mvn_release_repo="https://repo.scala-sbt.org/scalasbt/maven-releases" +declare -r sbt_launch_mvn_snapshot_repo="https://repo.scala-sbt.org/scalasbt/maven-snapshots" -declare -r default_jvm_opts_common="-Xms512m -Xss2m" +declare -r default_jvm_opts_common="-Xms512m -Xss2m -XX:MaxInlineLevel=18" declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" declare sbt_jar sbt_dir sbt_create sbt_version sbt_script sbt_new declare sbt_explicit_version declare verbose noshare batch trace_level -declare debugUs declare java_cmd="java" declare sbt_launch_dir="$HOME/.sbt/launchers" @@ -41,11 +40,14 @@ declare -a java_args scalac_args sbt_commands residual_args # args to jvm/sbt via files or environment variables declare -a extra_jvm_opts extra_sbt_opts -echoerr () { echo >&2 "$@"; } -vlog () { [[ -n "$verbose" ]] && echoerr "$@"; } -die () { echo "Aborting: $*" ; exit 1; } +echoerr() { echo >&2 "$@"; } +vlog() { [[ -n "$verbose" ]] && echoerr "$@"; } +die() { + echo "Aborting: $*" + exit 1 +} -setTrapExit () { +setTrapExit() { # save stty and trap exit, to ensure echo is re-enabled if we are interrupted. SBT_STTY="$(stty -g 2>/dev/null)" export SBT_STTY @@ -64,9 +66,12 @@ setTrapExit () { # this seems to cover the bases on OSX, and someone will # have to tell me about the others. -get_script_path () { +get_script_path() { local path="$1" - [[ -L "$path" ]] || { echo "$path" ; return; } + [[ -L "$path" ]] || { + echo "$path" + return + } local -r target="$(readlink "$path")" if [[ "${target:0:1}" == "/" ]]; then @@ -81,7 +86,7 @@ declare -r script_path script_name="${script_path##*/}" declare -r script_name -init_default_option_file () { +init_default_option_file() { local overriding_var="${!1}" local default_file="$2" if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then @@ -96,64 +101,78 @@ init_default_option_file () { sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" -build_props_sbt () { - [[ -r "$buildProps" ]] && \ +build_props_sbt() { + [[ -r "$buildProps" ]] && grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }' } -set_sbt_version () { +set_sbt_version() { sbt_version="${sbt_explicit_version:-$(build_props_sbt)}" [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version export sbt_version } -url_base () { +url_base() { local version="$1" case "$version" in - 0.7.*) echo "http://simple-build-tool.googlecode.com" ;; - 0.10.* ) echo "$sbt_launch_ivy_release_repo" ;; + 0.7.*) echo "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/simple-build-tool" ;; + 0.10.*) echo "$sbt_launch_ivy_release_repo" ;; 0.11.[12]) echo "$sbt_launch_ivy_release_repo" ;; 0.*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" - echo "$sbt_launch_ivy_snapshot_repo" ;; - 0.*) echo "$sbt_launch_ivy_release_repo" ;; - *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" - echo "$sbt_launch_mvn_snapshot_repo" ;; - *) echo "$sbt_launch_mvn_release_repo" ;; + echo "$sbt_launch_ivy_snapshot_repo" ;; + 0.*) echo "$sbt_launch_ivy_release_repo" ;; + *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmddThhMMss" + echo "$sbt_launch_mvn_snapshot_repo" ;; + *) echo "$sbt_launch_mvn_release_repo" ;; esac } -make_url () { +make_url() { local version="$1" local base="${sbt_launch_repo:-$(url_base "$version")}" case "$version" in - 0.7.*) echo "$base/files/sbt-launch-0.7.7.jar" ;; - 0.10.* ) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; + 0.7.*) echo "$base/sbt-launch-0.7.7.jar" ;; + 0.10.*) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; - 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; - *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch-${version}.jar" ;; esac } -addJava () { vlog "[addJava] arg = '$1'" ; java_args+=("$1"); } -addSbt () { vlog "[addSbt] arg = '$1'" ; sbt_commands+=("$1"); } -addScalac () { vlog "[addScalac] arg = '$1'" ; scalac_args+=("$1"); } -addResidual () { vlog "[residual] arg = '$1'" ; residual_args+=("$1"); } +addJava() { + vlog "[addJava] arg = '$1'" + java_args+=("$1") +} +addSbt() { + vlog "[addSbt] arg = '$1'" + sbt_commands+=("$1") +} +addScalac() { + vlog "[addScalac] arg = '$1'" + scalac_args+=("$1") +} +addResidual() { + vlog "[residual] arg = '$1'" + residual_args+=("$1") +} + +addResolver() { addSbt "set resolvers += $1"; } -addResolver () { addSbt "set resolvers += $1"; } -addDebugger () { addJava "-Xdebug" ; addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } -setThisBuild () { +addDebugger() { addJava "-Xdebug" && addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } + +setThisBuild() { vlog "[addBuild] args = '$*'" local key="$1" && shift addSbt "set $key in ThisBuild := $*" } -setScalaVersion () { +setScalaVersion() { [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")' addSbt "++ $1" } -setJavaHome () { +setJavaHome() { java_cmd="$1/bin/java" setThisBuild javaHome "_root_.scala.Some(file(\"$1\"))" export JAVA_HOME="$1" @@ -166,9 +185,9 @@ getJavaVersion() { # java -version on java8 says 1.8.x # but on 9 and 10 it's 9.x.y and 10.x.y. - if [[ "$str" =~ ^1\.([0-9]+)\..*$ ]]; then + if [[ "$str" =~ ^1\.([0-9]+)(\..*)?$ ]]; then echo "${BASH_REMATCH[1]}" - elif [[ "$str" =~ ^([0-9]+)\..*$ ]]; then + elif [[ "$str" =~ ^([0-9]+)(\..*)?$ ]]; then echo "${BASH_REMATCH[1]}" elif [[ -n "$str" ]]; then echoerr "Can't parse java version from: $str" @@ -178,8 +197,8 @@ getJavaVersion() { checkJava() { # Warn if there is a Java version mismatch between PATH and JAVA_HOME/JDK_HOME - [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java" - [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java" + [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java" + [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java" if [[ -n "$java" ]]; then pathJavaVersion=$(getJavaVersion java) @@ -193,23 +212,25 @@ checkJava() { fi } -java_version () { +java_version() { local -r version=$(getJavaVersion "$java_cmd") vlog "Detected Java version: $version" echo "$version" } # MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+ -default_jvm_opts () { +default_jvm_opts() { local -r v="$(java_version)" - if [[ $v -ge 8 ]]; then + if [[ $v -ge 10 ]]; then + echo "$default_jvm_opts_common -XX:+UnlockExperimentalVMOptions -XX:+UseJVMCICompiler" + elif [[ $v -ge 8 ]]; then echo "$default_jvm_opts_common" else echo "-XX:MaxPermSize=384m $default_jvm_opts_common" fi } -build_props_scala () { +build_props_scala() { if [[ -r "$buildProps" ]]; then versionLine="$(grep '^build.scala.versions' "$buildProps")" versionString="${versionLine##build.scala.versions=}" @@ -217,7 +238,7 @@ build_props_scala () { fi } -execRunner () { +execRunner() { # print the arguments one to a line, quoting any containing spaces vlog "# Executing command line:" && { for arg; do @@ -235,40 +256,36 @@ execRunner () { setTrapExit if [[ -n "$batch" ]]; then - "$@" < /dev/null + "$@" /dev/null 2>&1; then + if command -v curl >/dev/null 2>&1; then curl --fail --silent --location "$url" --output "$jar" - elif command -v wget > /dev/null 2>&1; then + elif command -v wget >/dev/null 2>&1; then wget -q -O "$jar" "$url" fi } && [[ -r "$jar" ]] } -acquire_sbt_jar () { +acquire_sbt_jar() { { sbt_jar="$(jar_file "$sbt_version")" [[ -r "$sbt_jar" ]] @@ -277,11 +294,66 @@ acquire_sbt_jar () { [[ -r "$sbt_jar" ]] } || { sbt_jar="$(jar_file "$sbt_version")" - download_url "$(make_url "$sbt_version")" "$sbt_jar" + jar_url="$(make_url "$sbt_version")" + + echoerr "Downloading sbt launcher for ${sbt_version}:" + echoerr " From ${jar_url}" + echoerr " To ${sbt_jar}" + + download_url "${jar_url}" "${sbt_jar}" + + case "${sbt_version}" in + 0.*) + vlog "SBT versions < 1.0 do not have published MD5 checksums, skipping check" + echo "" + ;; + *) verify_sbt_jar "${sbt_jar}" ;; + esac } } -usage () { +verify_sbt_jar() { + local jar="${1}" + local md5="${jar}.md5" + md5url="$(make_url "${sbt_version}").md5" + + echoerr "Downloading sbt launcher ${sbt_version} md5 hash:" + echoerr " From ${md5url}" + echoerr " To ${md5}" + + download_url "${md5url}" "${md5}" >/dev/null 2>&1 + + if command -v md5sum >/dev/null 2>&1; then + if echo "$(cat "${md5}") ${jar}" | md5sum -c -; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v md5 >/dev/null 2>&1; then + if [ "$(md5 -q "${jar}")" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v openssl >/dev/null 2>&1; then + if [ "$(openssl md5 -r "${jar}" | awk '{print $1}')" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + else + echoerr "Could not find an MD5 command" + return 1 + fi +} + +usage() { set_sbt_version cat < Run the specified file as a scala script # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version) - -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version - -sbt-version use the specified version of sbt (default: $sbt_release_version) - -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version - -sbt-jar use the specified jar as the sbt launcher - -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) - -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) + -sbt-version use the specified version of sbt (default: $sbt_release_version) + -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version + -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version + -sbt-jar use the specified jar as the sbt launcher + -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) + -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) # scala version (default: as chosen by sbt) - -28 use $latest_28 - -29 use $latest_29 - -210 use $latest_210 - -211 use $latest_211 - -212 use $latest_212 - -213 use $latest_213 - -scala-home use the scala build at the specified directory - -scala-version use the specified version of scala - -binary-version use the specified scala version when searching for dependencies + -28 use $latest_28 + -29 use $latest_29 + -210 use $latest_210 + -211 use $latest_211 + -212 use $latest_212 + -213 use $latest_213 + -scala-home use the scala build at the specified directory + -scala-version use the specified version of scala + -binary-version use the specified scala version when searching for dependencies # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) - -java-home alternate JAVA_HOME + -java-home alternate JAVA_HOME # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found - $(default_jvm_opts) - JVM_OPTS environment variable holding either the jvm args directly, or - the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') - Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. - -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) - -Dkey=val pass -Dkey=val directly to the jvm - -J-X pass option -X directly to the jvm (-J is stripped) + $(default_jvm_opts) + JVM_OPTS environment variable holding either the jvm args directly, or + the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') + Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. + -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) + -Dkey=val pass -Dkey=val directly to the jvm + -J-X pass option -X directly to the jvm (-J is stripped) # passing options to sbt, OR to this runner - SBT_OPTS environment variable holding either the sbt args directly, or - the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') - Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. - -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) - -S-X add -X to sbt's scalacOptions (-S is stripped) + SBT_OPTS environment variable holding either the sbt args directly, or + the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') + Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. + -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) + -S-X add -X to sbt's scalacOptions (-S is stripped) EOM + exit 0 } -process_args () { - require_arg () { +process_args() { + require_arg() { local type="$1" local opt="$2" local arg="$3" @@ -368,50 +435,55 @@ process_args () { } while [[ $# -gt 0 ]]; do case "$1" in - -h|-help) usage; exit 0 ;; - -v) verbose=true && shift ;; - -d) addSbt "--debug" && shift ;; - -w) addSbt "--warn" && shift ;; - -q) addSbt "--error" && shift ;; - -x) debugUs=true && shift ;; - -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; - -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; - -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; - -no-share) noshare=true && shift ;; - -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; - -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; - -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; - -offline) addSbt "set offline in Global := true" && shift ;; - -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; - -batch) batch=true && shift ;; - -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; - -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;; - - -sbt-create) sbt_create=true && shift ;; - -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; + -h | -help) usage ;; + -v) verbose=true && shift ;; + -d) addSbt "--debug" && shift ;; + -w) addSbt "--warn" && shift ;; + -q) addSbt "--error" && shift ;; + -x) shift ;; # currently unused + -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; + -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; + + -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; + -sbt-create) sbt_create=true && shift ;; + -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; + -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; + -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; + -no-share) noshare=true && shift ;; + -offline) addSbt "set offline in Global := true" && shift ;; + -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; + -batch) batch=true && shift ;; + -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; + -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;; + -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;; - -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; - -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; - -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; - -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; - -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; - -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; - -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;; - -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; - -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; - -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; - - -D*) addJava "$1" && shift ;; - -J*) addJava "${1:2}" && shift ;; - -S*) addScalac "${1:2}" && shift ;; - -28) setScalaVersion "$latest_28" && shift ;; - -29) setScalaVersion "$latest_29" && shift ;; - -210) setScalaVersion "$latest_210" && shift ;; - -211) setScalaVersion "$latest_211" && shift ;; - -212) setScalaVersion "$latest_212" && shift ;; - -213) setScalaVersion "$latest_213" && shift ;; - new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;; - *) addResidual "$1" && shift ;; + -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; + -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; + -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; + -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; + -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; + + -28) setScalaVersion "$latest_28" && shift ;; + -29) setScalaVersion "$latest_29" && shift ;; + -210) setScalaVersion "$latest_210" && shift ;; + -211) setScalaVersion "$latest_211" && shift ;; + -212) setScalaVersion "$latest_212" && shift ;; + -213) setScalaVersion "$latest_213" && shift ;; + + -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; + -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; + -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;; + -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; + -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; + -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; + + -D*) addJava "$1" && shift ;; + -J*) addJava "${1:2}" && shift ;; + -S*) addScalac "${1:2}" && shift ;; + + new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;; + + *) addResidual "$1" && shift ;; esac done } @@ -425,7 +497,7 @@ readConfigFile() { until $end; do read -r || end=true [[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo "$REPLY" - done < "$1" + done <"$1" } # if there are file/environment sbt_opts, process again so we @@ -435,7 +507,7 @@ if [[ -r "$sbt_opts_file" ]]; then while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file") elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then vlog "Using sbt options defined in variable \$SBT_OPTS" - IFS=" " read -r -a extra_sbt_opts <<< "$SBT_OPTS" + IFS=" " read -r -a extra_sbt_opts <<<"$SBT_OPTS" else vlog "No extra sbt options have been defined" fi @@ -454,8 +526,8 @@ checkJava # only exists in 0.12+ setTraceLevel() { case "$sbt_version" in - "0.7."* | "0.10."* | "0.11."* ) echoerr "Cannot set trace level in sbt version $sbt_version" ;; - *) setThisBuild traceLevel "$trace_level" ;; + "0.7."* | "0.10."* | "0.11."*) echoerr "Cannot set trace level in sbt version $sbt_version" ;; + *) setThisBuild traceLevel "$trace_level" ;; esac } @@ -466,12 +538,12 @@ setTraceLevel() { vlog "Detected sbt version $sbt_version" if [[ -n "$sbt_script" ]]; then - residual_args=( "$sbt_script" "${residual_args[@]}" ) + residual_args=("$sbt_script" "${residual_args[@]}") else # no args - alert them there's stuff in here - (( argumentCount > 0 )) || { + ((argumentCount > 0)) || { vlog "Starting $script_name: invoke with -help for other options" - residual_args=( shell ) + residual_args=(shell) } fi @@ -497,7 +569,7 @@ EOM # no jar? download it. [[ -r "$sbt_jar" ]] || acquire_sbt_jar || { # still no jar? uh-oh. - echo "Download failed. Obtain the jar manually and place it at $sbt_jar" + echo "Could not download and verify the launcher. Obtain the jar manually and place it at $sbt_jar" exit 1 } @@ -507,12 +579,12 @@ if [[ -n "$noshare" ]]; then done else case "$sbt_version" in - "0.7."* | "0.10."* | "0.11."* | "0.12."* ) + "0.7."* | "0.10."* | "0.11."* | "0.12."*) [[ -n "$sbt_dir" ]] || { sbt_dir="$HOME/.sbt/$sbt_version" vlog "Using $sbt_dir as sbt dir, -sbt-dir to override." } - ;; + ;; esac if [[ -n "$sbt_dir" ]]; then @@ -525,54 +597,18 @@ if [[ -r "$jvm_opts_file" ]]; then while read -r opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file") elif [[ -n "$JVM_OPTS" && ! ("$JVM_OPTS" =~ ^@.*) ]]; then vlog "Using jvm options defined in \$JVM_OPTS variable" - IFS=" " read -r -a extra_jvm_opts <<< "$JVM_OPTS" + IFS=" " read -r -a extra_jvm_opts <<<"$JVM_OPTS" else vlog "Using default jvm options" - IFS=" " read -r -a extra_jvm_opts <<< "$(default_jvm_opts)" + IFS=" " read -r -a extra_jvm_opts <<<"$( default_jvm_opts)" fi # traceLevel is 0.12+ [[ -n "$trace_level" ]] && setTraceLevel -main () { - execRunner "$java_cmd" \ - "${extra_jvm_opts[@]}" \ - "${java_args[@]}" \ - -jar "$sbt_jar" \ - "${sbt_commands[@]}" \ - "${residual_args[@]}" -} - -# sbt inserts this string on certain lines when formatting is enabled: -# val OverwriteLine = "\r\u001BM\u001B[2K" -# ...in order not to spam the console with a million "Resolving" lines. -# Unfortunately that makes it that much harder to work with when -# we're not going to print those lines anyway. We strip that bit of -# line noise, but leave the other codes to preserve color. -mainFiltered () { - local -r excludeRegex=$(grep -E -v '^#|^$' ~/.sbtignore | paste -sd'|' -) - - echoLine () { - local -r line="$1" - local -r line1="${line//\r\x1BM\x1B\[2K//g}" # This strips the OverwriteLine code. - local -r line2="${line1//\x1B\[[0-9;]*[JKmsu]//g}" # This strips all codes - we test regexes against this. - - if [[ $line2 =~ $excludeRegex ]]; then - [[ -n $debugUs ]] && echo "[X] $line1" - else - [[ -n $debugUs ]] && echo " $line1" || echo "$line1" - fi - } - - echoLine "Starting sbt with output filtering enabled." - main | while read -r line; do echoLine "$line"; done -} - -# Only filter if there's a filter file and we don't see a known interactive command. -# Obviously this is super ad hoc but I don't know how to improve on it. Testing whether -# stdin is a terminal is useless because most of my use cases for this filtering are -# exactly when I'm at a terminal, running sbt non-interactively. -shouldFilter () { [[ -f ~/.sbtignore ]] && ! grep -E -q '\b(shell|console|consoleProject)\b' <<<"${residual_args[@]}"; } - -# run sbt -if shouldFilter; then mainFiltered; else main; fi +execRunner "$java_cmd" \ + "${extra_jvm_opts[@]}" \ + "${java_args[@]}" \ + -jar "$sbt_jar" \ + "${sbt_commands[@]}" \ + "${residual_args[@]}" diff --git a/scripts/crosscompile.sh b/scripts/crosscompile.sh index 7c7db55..629b58c 100755 --- a/scripts/crosscompile.sh +++ b/scripts/crosscompile.sh @@ -21,7 +21,7 @@ docker run -it --rm \ -v $PWD:/pdal-java \ -v $HOME/.ivy2:/root/.ivy2 \ -v $HOME/.sbt:/root/.sbt \ - daunnc/pdal-debian:1.8.0 bash -c "cd ./pdal-java; ./scripts/pack-native.sh --suffix=${PDAL_VERSION_SUFFIX}" + daunnc/pdal-debian:2.0.0 bash -c "cd ./pdal-java; ./scripts/pack-native.sh --suffix=${PDAL_VERSION_SUFFIX}" # docker run -it --rm \ # -v $PWD:/workdir \ diff --git a/scripts/docker/2.0.0/alpine/Dockerfile b/scripts/docker/2.0.0/alpine/Dockerfile new file mode 100644 index 0000000..10e1e70 --- /dev/null +++ b/scripts/docker/2.0.0/alpine/Dockerfile @@ -0,0 +1,72 @@ +FROM pdal/pdal:2.0 +MAINTAINER Grigory Pomadchin + +# A few reasons for installing distribution-provided OpenJDK: +# +# 1. Oracle. Licensing prevents us from redistributing the official JDK. +# +# 2. Compiling OpenJDK also requires the JDK to be installed, and it gets +# really hairy. +# +# For some sample build times, see Debian's buildd logs: +# https://buildd.debian.org/status/logs.php?pkg=openjdk-8 + +# Default to UTF-8 file.encoding +ENV LANG C.UTF-8 + +# add a simple script that can auto-detect the appropriate JAVA_HOME value +# based on whether the JDK or only the JRE is installed +RUN { \ + echo '#!/bin/sh'; \ + echo 'set -e'; \ + echo; \ + echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \ + } > /usr/local/bin/docker-java-home \ + && chmod +x /usr/local/bin/docker-java-home +ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk +ENV PATH $PATH:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin + +ENV JAVA_VERSION 8u151 +ENV JAVA_ALPINE_VERSION 8.151.12-r0 + +RUN set -x \ + && apk add --no-cache \ + openjdk8="$JAVA_ALPINE_VERSION" \ + && [ "$JAVA_HOME" = "$(docker-java-home)" ] + +# If you're reading this and have any feedback on how this image could be +# improved, please open an issue or a pull request so we can discuss it! +# +# https://github.com/docker-library/openjdk/issues + +# Install deleted deps +RUN apk add --no-cache \ + bash \ + alpine-sdk \ + unzip \ + cmake \ + hexer-dev \ + nitro-dev \ + gdal-dev \ + geos-dev \ + laz-perf-dev \ + libexecinfo-dev \ + libgeotiff-dev \ + libxml2-dev \ + python-dev \ + py-numpy-dev \ + hdf5-dev \ + cpd-dev \ + fgt-dev \ + sqlite-dev \ + postgresql-dev \ + curl-dev \ + linux-headers \ + libspatialite-dev \ + cython-dev \ + py-packaging \ + clang \ + libharu-dev \ + poppler-dev \ + boost-dev \ + zlib-dev diff --git a/scripts/docker/2.0.0/crossbuild/Dockerfile b/scripts/docker/2.0.0/crossbuild/Dockerfile new file mode 100644 index 0000000..89ac788 --- /dev/null +++ b/scripts/docker/2.0.0/crossbuild/Dockerfile @@ -0,0 +1,19 @@ +FROM multiarch/crossbuild:latest +MAINTAINER Grigory Pomadchin + +RUN set -ex && \ + echo 'deb http://deb.debian.org/debian jessie-backports main' \ + > /etc/apt/sources.list.d/jessie-backports.list && \ + + apt update -y && \ + apt install -t \ + jessie-backports \ + openjdk-8-jdk \ + ca-certificates-java -y + +ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 + +RUN echo 'deb http://ftp.us.debian.org/debian/ unstable main contrib non-free' > /etc/apt/sources.list.d/debian-unstable.list + +# https://tracker.debian.org/pkg/pdal +RUN apt-get update && apt-get -y install pdal libpdal-dev bash diff --git a/scripts/docker/2.0.0/debian/Dockerfile b/scripts/docker/2.0.0/debian/Dockerfile new file mode 100644 index 0000000..aceae9c --- /dev/null +++ b/scripts/docker/2.0.0/debian/Dockerfile @@ -0,0 +1,14 @@ +FROM debian:sid +MAINTAINER Grigory Pomadchin + +RUN set -ex && \ + apt update -y && \ + apt install \ + openjdk-8-jdk \ + ca-certificates-java -y + +ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 + +# https://tracker.debian.org/pkg/pdal +RUN apt-get -y install pdal libpdal-dev bash gcc g++ cmake +RUN apt-get -y install wget diff --git a/scripts/docker/2.0.0/debian/mbio/Dockerfile b/scripts/docker/2.0.0/debian/mbio/Dockerfile new file mode 100644 index 0000000..4b04e26 --- /dev/null +++ b/scripts/docker/2.0.0/debian/mbio/Dockerfile @@ -0,0 +1,328 @@ +FROM ubuntu:bionic +MAINTAINER Grigory Pomadchin + +ENV CC gcc +ENV CXX g++ + +RUN dpkg --add-architecture i386 +RUN apt-get update -y +RUN apt-get install -y wget +RUN apt-get install -y xorg-dev +RUN apt-get install -y libxm4:i386 libuil4:i386 libmrm4:i386 libxm4 libuil4 libmrm4 +RUN apt-get install -y libmotif-common libmotif-dev libgl1-mesa-dev libglu1-mesa-dev +RUN apt-get install -y mesa-common-dev libfftw3-3 gfortran libfftw3-dev + +RUN apt-get update -qq; \ + apt-get -qq remove postgis; \ + apt-get install -y --fix-missing --no-install-recommends \ + software-properties-common + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing --no-install-recommends \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + gfortran \ + git \ + libgeotiff-dev \ + libarpack2-dev \ + libflann-dev \ + libhdf5-serial-dev \ + liblapack-dev \ + libtiff5-dev \ + openssh-client \ + python3-dev \ + python3-numpy \ + wget \ + automake \ + libtool \ + libspatialite-dev \ + libsqlite3-mod-spatialite \ + libhdf5-dev \ + libjsoncpp-dev \ + libboost-filesystem1.65-dev \ + libboost-iostreams1.65-dev \ + libboost-program-options1.65-dev \ + libboost-system1.65-dev \ + libboost-thread1.65-dev \ + clang \ + libproj-dev \ + libc6-dev \ + libnetcdf-dev \ + libpng-dev \ + libjpeg-dev \ + libgif-dev \ + libwebp-dev \ + libhdf4-alt-dev \ + libhdf5-dev \ + unixodbc-dev \ + libsqlite3-dev \ + libltdl-dev \ + libcurl4-openssl-dev \ + libspatialite-dev \ + cython \ + python3-pip \ + time \ + ninja-build \ + python3-setuptools \ + unzip \ + libeigen3-dev \ + libxml2-dev \ + libssl-dev \ + liblzma-dev \ + libzstd1-dev \ + pkg-config \ + libgdal-dev \ + bash-completion + +RUN apt-get install -y libnetcdf-dev netcdf-bin libproj12 libproj-dev gdal-bin libgdal-dev + +# Follow http://gmt.soest.hawaii.edu/projects/gmt/wiki/BuildingGMT +RUN apt-get install -y subversion cmake +RUN mkdir /gmt && cd /gmt && wget ftp://ftp.soest.hawaii.edu/gshhg/legacy/gshhg-gmt-2.3.6.tar.gz && tar xvzf gshhg-gmt-2.3.6.tar.gz && rm gshhg-gmt-2.3.6.tar.gz +RUN cd /gmt && wget ftp://ftp.soest.hawaii.edu/dcw/dcw-gmt-1.1.2.tar.gz && tar xvzf dcw-gmt-1.1.2.tar.gz && rm dcw-gmt-1.1.2.tar.gz +RUN cd /gmt && svn checkout svn://gmtserver.soest.hawaii.edu/gmt5/trunk gmt5-dev && cd /gmt/gmt5-dev && export CMAKE_INSTALL_PREFIX=/opt/gmt && export GSHHG_ROOT=/gmt/gshhg-gmt-2.3.6 && export DCW_ROOT=/gmt/dcw-gmt-1.1.2 && mkdir build && cd build && cmake .. && make && make install + +# RUN wget ftp://ftp.ldeo.columbia.edu/pub/MB-System/MB-System.tar.gz && tar xvzf MB-System.tar.gz && rm MB-System.tar.gz +RUN apt-get install -y git +RUN git clone https://github.com/dwcaress/MB-System.git && cd /MB-System && chmod 777 ./configure && ./configure && make && make install + +ARG PDAL_VERSION=2.0-maintenance +ARG DESTDIR="/build" +ARG tiledb_version=1.4.1 + +RUN git clone https://github.com/LASzip/LASzip.git laszip; \ + cd laszip; \ + git checkout 3.1.1; \ + cmake \ + -G Ninja \ + -DCMAKE_INSTALL_PREFIX=/usr/ \ + -DCMAKE_BUILD_TYPE="Release" \ + . ; \ + ninja ; \ + ninja install; \ + DESTDIR=/ ninja install; \ + rm -rf laszip + +RUN git clone https://github.com/hobu/laz-perf.git; \ + cd laz-perf; \ + mkdir build; \ + cd build; \ + cmake .. \ + -G Ninja \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE="Release" \ + ; \ + ninja; \ + ninja install; \ + DESTDIR=/ ninja install; \ + rm -rf /laz-perf + + +RUN mkdir /nitro; cd /nitro; \ + git clone https://github.com/hobu/nitro; \ + cd nitro; \ + mkdir build; \ + cd build; \ + cmake ..\ + -G Ninja -DCMAKE_INSTALL_PREFIX=/usr/ \ + ; \ + ninja; \ + ninja install; \ + DESTDIR=/ ninja install; \ + rm -rf /nitro + +RUN git clone --depth 1 --branch v0.4.6 https://github.com/gadomski/fgt.git \ + && cd fgt \ + && cmake . \ + -DWITH_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DEIGEN3_INCLUDE_DIR=/usr/include/eigen3 \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE=Release \ + -G "Ninja" \ + && ninja \ + && ninja install \ + && DESTDIR=/ ninja install \ + && rm -rf /fgt + +RUN git clone --depth 1 --branch v0.5.1 https://github.com/gadomski/cpd.git \ + && cd cpd \ + && cmake . \ + -DWITH_TESTS=OFF \ + -DWITH_JSONCPP=OFF \ + -DWITH_FGT=ON \ + -DCMAKE_SYSTEM_PREFIX_PATH="${DESTDIR}/usr" \ + -DCMAKE_CXX_FLAGS="-isystem ${DESTDIR}/usr/include" \ + -DWITH_STRICT_WARNINGS=OFF \ + -DWITH_DOCS=OFF \ + -DEIGEN3_INCLUDE_DIR=/usr/include/eigen3 \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE=Release \ + -G "Ninja" \ + && ninja \ + && ninja install \ + && DESTDIR=/ ninja install \ + && rm -rf /cpd + +#RUN \ +# wget https://github.com/TileDB-Inc/TileDB/archive/${tiledb_version}.tar.gz; \ +# tar xzf ${tiledb_version}.tar.gz; \ +# rm ${tiledb_version}.tar.gz; \ +# cd TileDB-${tiledb_version}; \ +# mkdir build; \ +# cd build; \ +# ../bootstrap --prefix=/usr --enable-s3; \ +# make -j2; \ +# make install-tiledb; \ +# DESTDIR=/ make install-tiledb; \ +# cd /; \ +# rm -rf TileDB-${tiledb_version}; + +RUN git clone --depth 1 --branch ${PDAL_VERSION} https://github.com/PDAL/PDAL.git pdal-git; \ + cd pdal-git; \ + mkdir build;\ + cd build; \ + cmake .. \ + -G Ninja \ + -DCMAKE_INSTALL_PREFIX=/usr/ \ + -DBUILD_PLUGIN_CPD=ON \ + -DBUILD_PLUGIN_MBIO=ON \ + -DBUILD_PLUGIN_GREYHOUND=ON \ + -DBUILD_PLUGIN_I3S=ON \ + -DBUILD_PLUGIN_HEXBIN=ON \ + -DBUILD_PLUGIN_ICEBRIDGE=ON \ + -DBUILD_PLUGIN_MRSID=OFF \ + -DBUILD_PLUGIN_NITF=ON \ + -DBUILD_PLUGIN_OCI=OFF \ + -DBUILD_PLUGIN_PCL=OFF \ + -DBUILD_PLUGIN_PGPOINTCLOUD=ON \ + -DPYTHON_EXECUTABLE=/usr/bin/python3 \ + -DBUILD_PLUGIN_SQLITE=ON \ + -DBUILD_PLUGIN_RIVLIB=OFF \ + -DBUILD_PLUGIN_PYTHON=ON \ + -DBUILD_PLUGIN_TILEDB=OFF \ + -DENABLE_CTEST=OFF \ + -DWITH_LAZPERF=ON \ + -DWITH_LASZIP=ON \ + -DWITH_ZSTD=ON \ + -DWITH_ZLIB=ON \ + -DWITH_TESTS=OFF \ + -DWITH_PDAL_JNI=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + ; \ + ninja; \ + ninja install; \ + DESTDIR=/ ninja install; \ + rm -rf /pdal-git + +# +# Haru hasn't been updated for years. This tag is the head. +# +RUN git clone https://github.com/libharu/libharu \ + && cd libharu \ + && git checkout d84867ebf9f3de6afd661d2cdaff102457fbc371 \ + && mkdir build \ + && cd build \ + && cmake \ + -G Ninja \ + -DCMAKE_INSTALL_PREFIX=/usr/ \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH:FILEPATH="${DESTDIR}" \ + -DCMAKE_SYSTEM_PREFIX_PATH="${DESTDIR}/usr" \ + -DCMAKE_CXX_FLAGS="-isystem ${DESTDIR}/usr/include" \ + ..; \ + ninja; \ + ninja install; \ + DESTDIR=/ ninja install; \ + rm -rf /libharu + +RUN git clone https://github.com/PDAL/PRC \ + && cd PRC \ + && mkdir build \ + && cd build \ + && CXXFLAGS="-fno-stack-protector" CFLAGS="-fno-stack-protector" cmake \ + -DCMAKE_INSTALL_PREFIX=/usr/ \ + -DPDAL_DIR="/usr/lib/pdal/cmake" \ + -DCMAKE_BUILD_TYPE=Release .. \ + -G "Ninja" \ + && ninja \ + && ninja install \ + && DESTDIR=/ ninja install \ + && rm -rf /PRC + + + +RUN \ + git clone https://github.com/pubgeo/pubgeo.git \ + && cd pubgeo \ + && mkdir build \ + && cd build \ + && CXXFLAGS="-fno-stack-protector" CFLAGS="-fno-stack-protector" cmake \ + -DCMAKE_INSTALL_PREFIX=/usr/ \ + -DPDAL_DIR="/usr/lib/pdal/cmake" \ + -DCMAKE_BUILD_TYPE=Release .. \ + -G "Ninja" \ + && ninja \ + && ninja install \ + && DESTDIR=/ ninja install \ + && rm -rf /pubgeo + +# FROM ubuntu:bionic as runner + +RUN apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + sudo curl \ + vim unzip \ + unixodbc \ + libgeotiff2 \ + libgeotiff-epsg \ + libexpat1 \ + libxml2 \ + libjsoncpp1 \ + libwebp6 \ + netcdf-bin \ + libhdf4-0-alt \ + libgif7 \ + libdapclient6v5 \ + libspatialite7 \ + libsqlite3-mod-spatialite \ + spatialite-bin \ + libtiff5 \ + libflann1.9 \ + libssl1.1 \ + libpython2.7 \ + libproj-dev \ + proj-bin \ + gdal-bin \ + proj-data \ + zlib1g \ + libpng16-16 \ + python3-setuptools \ + liblzma5 \ + libzstd1 \ + libhdf5-cpp-100 + +RUN date + +# COPY --from=builder /build/usr/bin/ /usr/bin/ +# COPY --from=builder /build/usr/lib/ /usr/lib/ +# COPY --from=builder /build/usr/include/ /usr/include/ + +RUN \ + curl -LOs http://download.osgeo.org/proj/proj-datumgrid-1.8.zip && unzip -j -u -o proj-datumgrid-1.8.zip -d /usr/share/proj; \ + curl -LOs http://download.osgeo.org/proj/proj-datumgrid-europe-1.2.zip && unzip -j -u -o proj-datumgrid-europe-1.2.zip -d /usr/share/proj; \ + curl -LOs http://download.osgeo.org/proj/proj-datumgrid-oceania-1.0.zip && unzip -j -u -o proj-datumgrid-oceania-1.0.zip -d /usr/share/proj; \ + curl -LOs http://download.osgeo.org/proj/proj-datumgrid-world-1.0.zip && unzip -j -u -o proj-datumgrid-world-1.0.zip -d /usr/share/proj; \ + curl -LOs http://download.osgeo.org/proj/proj-datumgrid-north-america-1.2.zip && unzip -j -u -o proj-datumgrid-north-america-1.2.zip -d /usr/share/proj; + +RUN set -ex && \ + apt update -y && \ + apt install \ + openjdk-8-jdk \ + ca-certificates-java -y + +ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 + + diff --git a/scripts/merge-native.sh b/scripts/merge-native.sh index 009b7c8..e02e4fa 100755 --- a/scripts/merge-native.sh +++ b/scripts/merge-native.sh @@ -17,13 +17,13 @@ done export PDAL_VERSION_SUFFIX=${PDAL_VERSION_SUFFIX-"-SNAPSHOT"} cd ./native/target -rm -f ./pdal-native-1.9.1${PDAL_VERSION_SUFFIX}.jar +rm -f ./pdal-native-2.0.0${PDAL_VERSION_SUFFIX}.jar rm -rf ./tmp; mkdir -p ./tmp -cd tmp; jar -xf ../pdal-native-x86_64-darwin-1.9.1${PDAL_VERSION_SUFFIX}.jar; cd ~- -cd tmp; jar -xf ../pdal-native-x86_64-linux-1.9.1${PDAL_VERSION_SUFFIX}.jar; cd ~- +cd tmp; jar -xf ../pdal-native-x86_64-darwin-2.0.0${PDAL_VERSION_SUFFIX}.jar; cd ~- +cd tmp; jar -xf ../pdal-native-x86_64-linux-2.0.0${PDAL_VERSION_SUFFIX}.jar; cd ~- -jar -cvf pdal-native-1.9.1${PDAL_VERSION_SUFFIX}.jar -C tmp . +jar -cvf pdal-native-2.0.0${PDAL_VERSION_SUFFIX}.jar -C tmp . cd ./tmp diff --git a/scripts/publish-211.sh b/scripts/publish-211.sh new file mode 100755 index 0000000..faf811e --- /dev/null +++ b/scripts/publish-211.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# --suffix: sets the suffix you want to publish lib with +# --signed: makes a PGP signed publish of the library, for maven central this sign is required + +for i in "$@" +do + case $i in + --suffix=*) + PDAL_VERSION_SUFFIX="${i#*=}" + shift + ;; + --signed) + SIGNED=true + shift + ;; + *) + ;; + esac +done + +export PDAL_VERSION_SUFFIX=${PDAL_VERSION_SUFFIX-"-SNAPSHOT"} +SIGNED=${SIGNED:-false} + +COMMAND=publish + +if ${SIGNED}; then + COMMAND=publishSigned +fi + +PDAL_DEPEND_ON_NATIVE=false ./sbt "-211" "project core" ${COMMAND} +PDAL_DEPEND_ON_NATIVE=false ./sbt "-211" "project core-scala" ${COMMAND} diff --git a/scripts/publish-all.sh b/scripts/publish-all.sh index 3fc8411..2fb63bf 100755 --- a/scripts/publish-all.sh +++ b/scripts/publish-all.sh @@ -2,5 +2,6 @@ ./scripts/publish.sh "$@" ./scripts/publish-212.sh "$@" +./scripts/publish-211.sh "$@" ./scripts/publish-javastyle.sh "$@" ./scripts/publish-native.sh "$@" diff --git a/scripts/publish-javastyle.sh b/scripts/publish-javastyle.sh index c8f385c..a897eef 100755 --- a/scripts/publish-javastyle.sh +++ b/scripts/publish-javastyle.sh @@ -28,4 +28,4 @@ if ${SIGNED}; then COMMAND=publishSigned fi -PDAL_DEPEND_ON_NATIVE=false ./sbt "-212" "project core" ${COMMAND}Javastyle +PDAL_DEPEND_ON_NATIVE=false ./sbt "project core" ${COMMAND}Javastyle diff --git a/scripts/publish-local-211.sh b/scripts/publish-local-211.sh new file mode 100755 index 0000000..1287d55 --- /dev/null +++ b/scripts/publish-local-211.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# --suffix: sets the suffix you want to publish lib with + +for i in "$@" +do + case $i in + --suffix=*) + PDAL_VERSION_SUFFIX="${i#*=}" + shift + ;; + *) + ;; + esac +done + +export PDAL_VERSION_SUFFIX=${PDAL_VERSION_SUFFIX-"-SNAPSHOT"} + +PDAL_DEPEND_ON_NATIVE=false ./sbt "-211" "project core" publishLocal +PDAL_DEPEND_ON_NATIVE=false ./sbt "-211" "project core-scala" publishLocal +./sbt "-211" "project native" publishLocal From 4afbba8c254f5d6a8e90b0ffb49a943342680f62 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Tue, 14 Jan 2020 15:00:21 -0500 Subject: [PATCH 2/2] Fix test-all scripts --- .travis.yml | 2 ++ .travis/test-all.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f553d67..9f40ef6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ matrix: - docker - os: osx + osx_image: xcode9.3 scala: - 2.13.1 compiler: @@ -33,6 +34,7 @@ matrix: - brew install pdal || true # to make travis happy - os: osx + osx_image: xcode9.3 scala: - 2.12.10 compiler: diff --git a/.travis/test-all.sh b/.travis/test-all.sh index f2b21ee..ec3979c 100755 --- a/.travis/test-all.sh +++ b/.travis/test-all.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -if [ `echo $TRAVIS_SCALA_VERSION | cut -f1-2 -d "."` = "2.13" ]; then +if [[ `echo $TRAVIS_SCALA_VERSION | cut -f1-2 -d "."` = "2.13" ]]; then .travis/test.sh; else .travis/test-212.sh;