move S3Mock to s3 testkit

build.sbt

@@ -139,7 +139,15 @@ lazy val sparkTestkit: Project = Project("spark-testkit", file("spark-testkit"))
   settings(commonSettings: _*)
 
 lazy val s3 = Project("s3", file("s3")).
-  dependsOn(sparkTestkit % "test->test", spark % "provided;test->test").
+  dependsOn(spark % "provided;test->test").
+  settings(commonSettings: _*)
+
+lazy val s3Test = Project("s3-test", file("s3-test")).
+  dependsOn(s3 % "provided", s3Testkit, sparkTestkit, spark % "provided;test->test").
+  settings(commonSettings: _*)
+
+lazy val s3Testkit = Project("s3-testkit", file("s3-testkit")).
+  dependsOn(s3 % "provided", spark % "provided;test->test").
   settings(commonSettings: _*)
 
 lazy val accumulo = Project("accumulo", file("accumulo")).

s3-test/build.sbt

@@ -0,0 +1,11 @@
+import Dependencies._
+
+name := "geotrellis-s3-test"
+libraryDependencies ++= Seq(
+  sparkCore % "provided",
+  awsSdkS3,
+  spire,
+  scalatest % "test")
+
+fork in Test := false
+parallelExecution in Test := false

s3/src/test/scala/geotrellis/spark/io/s3/GeoTiffS3InputFormatSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/GeoTiffS3InputFormatSpec.scala

@@ -1,15 +1,17 @@
 package geotrellis.spark.io.s3
 
-import java.nio.file.{ Paths, Files }
-
-import com.amazonaws.auth.AWSCredentials
 import geotrellis.raster._
 import geotrellis.vector._
 import geotrellis.spark._
 import geotrellis.spark.io.hadoop._
+import geotrellis.spark.io.s3.testkit._
+
+import com.amazonaws.auth.AWSCredentials
 import org.apache.hadoop.mapreduce.{ TaskAttemptContext, InputSplit }
 import org.scalatest._
 
+import java.nio.file.{ Paths, Files }
+
 class MockGeoTiffS3InputFormat extends GeoTiffS3InputFormat {
   override def getS3Client(credentials: AWSCredentials): S3Client = new MockS3Client
   override def createRecordReader(split: InputSplit, context: TaskAttemptContext) =

s3/src/test/scala/geotrellis/spark/io/s3/S3AttributeStoreSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/S3AttributeStoreSpec.scala

@@ -2,6 +2,7 @@ package geotrellis.spark.io.s3
 
 import geotrellis.spark._
 import geotrellis.spark.io._
+import geotrellis.spark.io.s3.testkit._
 
 class S3AttributeStoreSpec extends AttributeStoreSpec {
   val bucket = "attribute-store-test-mock-bucket"

s3/src/test/scala/geotrellis/spark/io/s3/S3ByteReaderSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/S3ByteReaderSpec.scala

@@ -2,6 +2,7 @@ package geotrellis.spark.io.s3.util
 
 import geotrellis.util._
 import geotrellis.spark.io.s3._
+import geotrellis.spark.io.s3.testkit._
 
 import java.nio.{ByteBuffer, ByteOrder}
 import scala.collection.mutable._

s3/src/test/scala/geotrellis/spark/io/s3/S3GeoTiffReadingSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/S3GeoTiffReadingSpec.scala

@@ -5,6 +5,7 @@ import java.nio.file.{Files, Paths}
 import geotrellis.util._
 import geotrellis.vector.Extent
 import geotrellis.spark.io.s3._
+import geotrellis.spark.io.s3.testkit._
 import geotrellis.raster.testkit._
 import geotrellis.raster.io.geotiff._
 import geotrellis.raster.io.geotiff.reader._

s3/src/test/scala/geotrellis/spark/io/s3/S3InputFormatSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/S3InputFormatSpec.scala

@@ -1,10 +1,12 @@
 package geotrellis.spark.io.s3
 
-import com.amazonaws.auth.AWSCredentials
+import geotrellis.spark.io.s3.testkit._
 import geotrellis.proj4.LatLng
 import geotrellis.raster.Tile
 import geotrellis.spark.TestEnvironment
 import geotrellis.vector.{ Extent, ProjectedExtent }
+
+import com.amazonaws.auth.AWSCredentials
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.mapreduce.{ Job, RecordReader, TaskAttemptContext, InputSplit }
 import org.scalatest._

s3/src/test/scala/geotrellis/spark/io/s3/S3SaveImagesSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/S3SaveImagesSpec.scala

@@ -6,6 +6,7 @@ import geotrellis.spark.TestEnvironment
 import geotrellis.spark.render._
 import geotrellis.spark.testfiles.TestFiles
 import geotrellis.spark.io.s3._
+import geotrellis.spark.io.s3.testkit._
 import geotrellis.spark.io.s3.SaveToS3
 
 import org.scalatest._

s3/src/test/scala/geotrellis/spark/io/s3/S3SpaceTimeSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/S3SpaceTimeSpec.scala

@@ -2,6 +2,7 @@ package geotrellis.spark.io.s3
 
 import geotrellis.raster.Tile
 import geotrellis.spark.io._
+import geotrellis.spark.io.s3.testkit._
 import geotrellis.spark.io.index._
 import geotrellis.spark.testfiles.TestFiles
 import geotrellis.spark._

s3/src/test/scala/geotrellis/spark/io/s3/S3SpatialSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/S3SpatialSpec.scala

@@ -2,6 +2,7 @@ package geotrellis.spark.io.s3
 
 import geotrellis.raster.Tile
 import geotrellis.spark.io._
+import geotrellis.spark.io.s3.testkit._
 import geotrellis.spark.io.index._
 import geotrellis.spark.testfiles.TestFiles
 import geotrellis.spark._

s3/src/test/scala/geotrellis/spark/io/s3/S3StreamBytesSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/S3StreamBytesSpec.scala

@@ -4,6 +4,7 @@ import java.nio.file.{ Paths, Files }
 import java.nio.ByteBuffer
 import geotrellis.util._
 import geotrellis.spark.io.s3._
+import geotrellis.spark.io.s3.testkit._
 import spire.syntax.cfor._
 
 import com.amazonaws.services.s3.model._

s3/src/test/scala/geotrellis/spark/io/s3/S3TileFeatureSpaceTimeSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/S3TileFeatureSpaceTimeSpec.scala

@@ -2,6 +2,7 @@ package geotrellis.spark.io.s3
 
 import geotrellis.raster.{Tile, TileFeature}
 import geotrellis.spark.io._
+import geotrellis.spark.io.s3.testkit._
 import geotrellis.spark.io.index._
 import geotrellis.spark.testfiles.TestTileFeatureFiles
 import geotrellis.spark._

s3/src/test/scala/geotrellis/spark/io/s3/S3TileFeatureSpatialSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/S3TileFeatureSpatialSpec.scala

@@ -3,6 +3,7 @@ package geotrellis.spark.io.s3
 import geotrellis.raster.{Tile, TileFeature}
 import geotrellis.spark._
 import geotrellis.spark.io._
+import geotrellis.spark.io.s3.testkit._
 import geotrellis.spark.io.index._
 import geotrellis.spark.testfiles.TestTileFeatureFiles
 

s3/src/test/scala/geotrellis/spark/io/s3/TemporalGeoTiffS3InputFormatSpec.scala → s3-test/src/test/scala/geotrellis/spark/io/s3/TemporalGeoTiffS3InputFormatSpec.scala

@@ -1,17 +1,20 @@
 package geotrellis.spark.io.s3
 
-import com.amazonaws.auth.AWSCredentials
 import geotrellis.proj4.LatLng
 import geotrellis.raster._
 import geotrellis.spark._
 import geotrellis.spark.tiling._
 import geotrellis.spark.io.hadoop._
 import geotrellis.spark.ingest._
 import geotrellis.util.Filesystem
+import geotrellis.spark.io.json._
+import geotrellis.spark.io.s3.testkit._
+
+import com.amazonaws.auth.AWSCredentials
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.mapreduce._
 import org.apache.hadoop.mapreduce.task._
-import geotrellis.spark.io.json._
+
 import java.time.{ZoneOffset, ZonedDateTime}
 import java.nio.file.{Files, Path, Paths}
 import java.time.format.DateTimeFormatter

s3-testkit/build.sbt

@@ -0,0 +1,10 @@
+import Dependencies._
+
+name := "geotrellis-s3-testkit"
+libraryDependencies ++= Seq(
+  sparkCore % "provided",
+  awsSdkS3,
+  spire,
+  logging,
+  scalatest
+)

s3/src/test/scala/geotrellis/spark/io/s3/MockS3ByteReader.scala → s3-testkit/src/main/scala/geotrellis/spark/io/s3/testkit/MockS3ByteReader.scala

@@ -1,4 +1,4 @@
-package geotrellis.spark.io.s3.util
+package geotrellis.spark.io.s3.testkit
 
 import geotrellis.util._
 import geotrellis.spark.io.s3._

s3/src/test/scala/geotrellis/spark/io/s3/MockS3Client.scala → s3-testkit/src/main/scala/geotrellis/spark/io/s3/testkit/MockS3Client.scala

@@ -1,5 +1,6 @@
-package geotrellis.spark.io.s3
+package geotrellis.spark.io.s3.testkit
 
+import geotrellis.spark.io.s3._
 import java.io.ByteArrayInputStream
 import com.amazonaws.services.s3.model._
 import java.util.concurrent.ConcurrentHashMap

s3/src/test/scala/geotrellis/spark/io/s3/MockS3LayerCollectionReader.scala → s3-testkit/src/main/scala/geotrellis/spark/io/s3/testkit/MockS3LayerCollectionReader.scala

@@ -1,6 +1,7 @@
-package geotrellis.spark.io.s3
+package geotrellis.spark.io.s3.testkit
 
 import geotrellis.spark.io._
+import geotrellis.spark.io.s3._
 import org.apache.spark._
 
 class MockS3LayerCollectionReader(

s3/src/test/scala/geotrellis/spark/io/s3/MockS3LayerReader.scala → s3-testkit/src/main/scala/geotrellis/spark/io/s3/testkit/MockS3LayerReader.scala

@@ -1,7 +1,8 @@
-package geotrellis.spark.io.s3
+package geotrellis.spark.io.s3.testkit
 
 import geotrellis.spark._
 import geotrellis.spark.io._
+import geotrellis.spark.io.s3._
 import geotrellis.spark.io.json._
 
 import org.apache.spark._

s3/src/test/scala/geotrellis/spark/io/s3/MockS3LayerWriter.scala → s3-testkit/src/main/scala/geotrellis/spark/io/s3/testkit/MockS3LayerWriter.scala

@@ -1,6 +1,7 @@
-package geotrellis.spark.io.s3
+package geotrellis.spark.io.s3.testkit
 
 import geotrellis.spark.io._
+import geotrellis.spark.io.s3._
 import geotrellis.spark.io.json._
 
 import spray.json._

s3/src/test/scala/geotrellis/spark/io/s3/MockS3StreamBytes.scala → s3-testkit/src/main/scala/geotrellis/spark/io/s3/testkit/MockS3StreamBytes.scala

@@ -1,4 +1,4 @@
-package geotrellis.spark.io.s3.util
+package geotrellis.spark.io.s3.testkit
 
 import geotrellis.util._
 import geotrellis.spark.io.s3._

s3/src/test/scala/geotrellis/spark/io/s3/MockS3ClientSpec.scala → s3-testkit/src/test/scala/geotrellis/spark/io/s3/testkit/MockS3ClientSpec.scala

@@ -1,4 +1,6 @@
-package geotrellis.spark.io.s3
+package geotrellis.spark.io.s3.testkit
+
+import geotrellis.spark.io.s3._
 
 import org.scalatest._
 import scala.io.Source

scripts/buildall.sh

@@ -10,7 +10,9 @@ HOSTALIASES=/tmp/hostaliases ./sbt -J-Xmx2G "project geowave" test || { exit 1;
 ./sbt -J-Xmx2G "project proj4" test || { exit 1; }
 ./sbt -J-Xmx2G "project raster-test" test || { exit 1; }
 ./sbt -J-Xmx2G "project raster-testkit" compile || { exit 1; }
-./sbt -J-Xmx2G "project s3" test  || { exit 1; }
+./sbt -J-Xmx2G "project s3" compile  || { exit 1; }
+./sbt -J-Xmx2G "project s3-test" test  || { exit 1; }
+./sbt -J-Xmx2G "project s3-testkit" test  || { exit 1; }
 ./sbt -J-Xmx2G "project shapefile" compile || { exit 1; }
 ./sbt -J-Xmx2G "project slick" test:compile || { exit 1; }
 ./sbt -J-Xmx2G "project spark" test  || { exit 1; }

scripts/cleanall.sh

@@ -7,6 +7,8 @@
 ./sbt -J-Xmx2G "project geowave" clean || { exit 1; }
 ./sbt -J-Xmx2G "project hbase" clean || { exit 1; }
 ./sbt -J-Xmx2G "project proj4" clean || { exit 1; }
+./sbt -J-Xmx2G "project s3" clean || { exit 1; }
+./sbt -J-Xmx2G "project s3-test" clean || { exit 1; }
 ./sbt -J-Xmx2G "project raster-test" clean || { exit 1; }
 ./sbt -J-Xmx2G "project shapefile" clean || { exit 1; }
 ./sbt -J-Xmx2G "project slick" clean || { exit 1; }
@@ -27,6 +29,8 @@ rm -r raster-test/target
 rm -r raster-testkit/target
 rm -r raster/target
 rm -r s3/target
+rm -r s3-test/target
+rm -r s3-testkit/target
 rm -r shapefile/target
 rm -r slick/target
 rm -r spark-testkit/target

scripts/publish-local.sh

@@ -12,6 +12,7 @@
 ./sbt "project raster" publish-local && \
 ./sbt "project raster-testkit" publish-local && \
 ./sbt "project s3" publish-local && \
+./sbt "project s3-testkit" publish-local && \
 ./sbt "project geowave" publish-local && \
 ./sbt "project shapefile" publish-local && \
 ./sbt "project slick" publish-local && \

scripts/publish-m2.sh

@@ -11,6 +11,7 @@
       "project raster" +publish-m2 \
       "project raster-testkit" +publish-m2 \
       "project s3" +publish-m2 \
+      "project s3-testkit" +publish-m2 \
       "project shapefile" +publish-m2 \
       "project slick" +publish-m2 \
       "project spark" +publish-m2 \

scripts/publish-snapshot.sh

@@ -11,6 +11,7 @@
       "project raster" publish \
       "project raster-testkit" publish \
       "project s3" publish \
+      "project s3-testkit" publish \
       "project geowave" publish \
       "project accumulo" publish \
       "project cassandra" publish \