Merge pull request #664 from locationtech-labs/feature/gt-2.0

GeoTrellis 2.0 and tag for 0.4.2

.gitignore

@@ -111,4 +111,6 @@ prof/
 .#*
 
 # Visual Studio Code
-.vscode
+.vscode
+
+.pytest_cache

.travis.yml

@@ -65,7 +65,7 @@ script:
 
 before_deploy:
   - export GEOPYSPARK_VERSION_SUFFIX="-${TRAVIS_COMMIT:0:7}"
-  - aws s3 rm s3://geopyspark-dependency-jars/geotrellis-backend-assembly-0.4.1.jar
+  - aws s3 rm s3://geopyspark-dependency-jars/geotrellis-backend-assembly-0.4.2.jar
 
 deploy:
   - provider: script

Makefile

@@ -6,11 +6,11 @@ rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst
 
 JAR-PATH := geopyspark/jars
 
-ASSEMBLYNAME := geotrellis-backend-assembly-0.4.1.jar
+ASSEMBLYNAME := geotrellis-backend-assembly-0.4.2.jar
 BUILD-ASSEMBLY := geopyspark-backend/geotrellis/target/scala-2.11/${ASSEMBLYNAME}
 DIST-ASSEMBLY := ${JAR-PATH}/${ASSEMBLYNAME}
 
-WHEELNAME := geopyspark-0.4.1-py3-none-any.whl
+WHEELNAME := geopyspark-0.4.2-py3-none-any.whl
 WHEEL := dist/${WHEELNAME}
 
 SCALA_SRC := $(call rwildcard, geopyspark-backend/geotrellis/src/, *.scala)

docs/CHANGELOG.rst

@@ -1,6 +1,31 @@
 Changelog
 ==========
 
+0.4.2
+------
+
+Experimental New Features
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Creating a RasterLayer From URIs Using rasterio
+************************************************
+
+While the ability to create a ``RasterLayer``
+from ``URI``\s already exists with the ``geopyspark.geotrellis.geotiff.get``
+function, it is limited to just working with GeoTiffs. However, with the
+new ``geopyspark.geotrellis.rasterio`` module, it is now possible to
+create ``RasterLayer``\s from different file types.
+
+.. code:: python3
+
+  uris = ["file://images/image_1.jp2", "file://images/image_2.jp2"]
+
+  raster_layer = gps.rasterio.get(uris)
+
+
+**Note:** This feature is experimental, and will most likely be improved
+and/or changed in the future releases of GeoPySpark.
+
 
 0.4.1
 ------

docs/conf.py

@@ -29,10 +29,10 @@
     os.environ['SPARK_HOME'] = './spark-2.1.1-bin-hadoop2.7/'
 
 os.chdir('..')
-jar = 'geotrellis-backend-assembly-0.2.2.jar'
+jar = 'geotrellis-backend-assembly-0.4.2.jar'
 
 if not path.isfile(path.join('geopyspark/jars', jar)):
-    url = 'https://github.com/locationtech-labs/geopyspark/releases/download/v0.4.1/'
+    url = 'https://github.com/locationtech-labs/geopyspark/releases/download/v0.4.2/'
     subprocess.call(['curl', '-L', url+jar, '-o', path.join('geopyspark/jars', jar)])
 
 sys.path.insert(0, path.abspath(os.curdir))
@@ -73,9 +73,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '0.4.1'
+version = '0.4.2'
 # The full version, including alpha/beta/rc tags.
-release = '0.4.1'
+release = '0.4.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

geopyspark-backend/build.sbt

@@ -1,3 +1,4 @@
+
 lazy val commonSettings = Seq(
   version := Version.geopyspark + scala.util.Properties.envOrElse("GEOPYSPARK_VERSION_SUFFIX", ""),
   scalaVersion := Version.scala,
@@ -24,7 +25,8 @@ lazy val commonSettings = Seq(
     "LocationTech Snapshots" at "https://repo.locationtech.org/content/groups/snapshots",
     "LocationTech Releases" at "https://repo.locationtech.org/content/groups/releases",
     Resolver.bintrayRepo("azavea", "maven"),
-    DefaultMavenRepository
+    DefaultMavenRepository,
+    Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
   ),
 
   addCompilerPlugin("org.spire-math" % "kind-projector" % "0.9.3" cross CrossVersion.binary),

geopyspark-backend/geotools/build.sbt

@@ -1,7 +1,7 @@
 name := "geotools-backend"
 
 libraryDependencies ++= Seq(
-  "org.apache.spark"            %% "spark-core"            % "2.2.0" % "provided",
+  "org.apache.spark"            %% "spark-core"            % "2.3.0" % "provided",
   "org.locationtech.geotrellis" %% "geotrellis-s3"         % Version.geotrellis,
   "org.locationtech.geotrellis" %% "geotrellis-s3-testkit" % Version.geotrellis,
   "org.locationtech.geotrellis" %% "geotrellis-spark"      % Version.geotrellis,

geopyspark-backend/geotrellis/build.sbt

@@ -4,8 +4,8 @@ libraryDependencies ++= Seq(
   "org.typelevel"               %% "cats"                  % "0.9.0",
   "com.typesafe.akka"           %% "akka-http"             % "10.0.10",
   "com.typesafe.akka"           %% "akka-http-spray-json"  % "10.0.10",
-  "net.sf.py4j"                 %  "py4j"                  % "0.10.5",
-  "org.apache.spark"            %% "spark-core"            % "2.2.0" % "provided",
+  "net.sf.py4j"                 %  "py4j"                  % "0.10.6",
+  "org.apache.spark"            %% "spark-core"            % "2.3.0" % "provided",
   "org.apache.commons"          % "commons-math3"          % "3.6.1",
   "org.locationtech.geotrellis" %% "geotrellis-s3"         % Version.geotrellis,
   "org.locationtech.geotrellis" %% "geotrellis-s3-testkit" % Version.geotrellis,
@@ -24,13 +24,12 @@ assemblyShadeRules in assembly := {
 }
 
 assemblyMergeStrategy in assembly := {
-  case "reference.conf" => MergeStrategy.concat
-  case "application.conf" => MergeStrategy.concat
-  case "META-INF/MANIFEST.MF" => MergeStrategy.discard
-  case "META-INF\\MANIFEST.MF" => MergeStrategy.discard
-  case "META-INF/ECLIPSEF.RSA" => MergeStrategy.discard
-  case "META-INF/ECLIPSEF.SF" => MergeStrategy.discard
-  case x if x.startsWith("META-INF/services") => MergeStrategy.concat
+  case s if s.startsWith("META-INF/services") => MergeStrategy.concat
+  case "reference.conf" | "application.conf"  => MergeStrategy.concat
+  case "META-INF/ECLIPSE.RSA" | "META-INF/ECLIPSE.SF" => MergeStrategy.discard
+  case "META-INF/ECLIPSEF.RSA" | "META-INF/ECLIPSEF.SF" => MergeStrategy.discard
+  case "META-INF/ECLIPSE_.RSA" | "META-INF/ECLIPSE_.SF" => MergeStrategy.discard
+  case "META-INF/MANIFEST.MF" | "META-INF\\MANIFEST.MF" => MergeStrategy.discard
   case _ => MergeStrategy.first
 }
 

geopyspark-backend/project/Version.scala

@@ -1,6 +1,6 @@
 object Version {
-  val geopyspark = "0.4.1"
-  val geotrellis = "2.0.0-SNAPSHOT"
+  val geopyspark = "0.4.2"
+  val geotrellis = "2.0.0-RC1"
   val scala       = "2.11.11"
   val scalaTest   = "2.2.0"
 }

geopyspark-backend/vectorpipe/build.sbt

@@ -1,7 +1,7 @@
 name := "vectorpipe"
 
 libraryDependencies ++= Seq(
-  "org.apache.spark"            %% "spark-hive"            % "2.2.0" % "provided",
+  "org.apache.spark"            %% "spark-hive"            % "2.3.0" % "provided",
   "org.locationtech.geotrellis" %% "geotrellis-s3"         % Version.geotrellis,
   "org.locationtech.geotrellis" %% "geotrellis-s3-testkit" % Version.geotrellis,
   "org.locationtech.geotrellis" %% "geotrellis-spark"      % Version.geotrellis,

geopyspark/__init__.py

@@ -114,9 +114,8 @@ def geopyspark_conf(master=None, appName=None, additional_jar_dirs=[]):
     if not jars:
         raise IOError("Failed to find any jars. Looked at these paths {}".format(possible_jars))
 
-    jar_string = ",".join(jars)
+    jar_string = ",".join(set(jars))
     conf.set(key='spark.jars', value=jar_string)
-
     conf.set(key='spark.driver.memory', value='8G')
     conf.set(key='spark.executor.memory', value='8G')
 

geopyspark/command/configuration.py

@@ -10,7 +10,7 @@
 from geopyspark.geopyspark_constants import JAR, CWD
 
 
-JAR_URL = 'https://github.com/locationtech-labs/geopyspark/releases/download/v0.4.1/' + JAR
+JAR_URL = 'https://github.com/locationtech-labs/geopyspark/releases/download/v0.4.2/' + JAR
 DEFAULT_JAR_PATH = path.join(CWD, 'jars')
 CONF = path.join(CWD, 'command', 'geopyspark.conf')
 

geopyspark/geopyspark_constants.py

@@ -2,7 +2,7 @@
 from os import path
 
 """GeoPySpark version."""
-VERSION = '0.4.1'
+VERSION = '0.4.2'
 
 """Backend jar name."""
 JAR = 'geotrellis-backend-assembly-' + VERSION + '.jar'

setup.py

@@ -7,7 +7,7 @@
 
 setup_args = dict(
     name='geopyspark',
-    version='0.4.1',
+    version='0.4.2',
     author='Jacob Bouffard, James McClain',
     author_email='jbouffard@azavea.com, jmcclain@azavea.com',
     download_url='http://github.com/locationtech-labs/geopyspark',