From 3e72f2f05154bb1e6c96a8209a104d644fec0bdf Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Thu, 13 Jul 2017 19:31:10 -0400 Subject: [PATCH] Remove caching from AttributeStoreProviders Signed-off-by: Eugene Cheipesh --- accumulo/build.sbt | 1 - .../spark/io/accumulo/AccumuloLayerProvider.scala | 9 +-------- cassandra/build.sbt | 1 - .../spark/io/cassandra/CassandraLayerProvider.scala | 9 +-------- hbase/build.sbt | 1 - .../geotrellis/spark/io/hbase/HBaseLayerProvider.scala | 9 +-------- project/Dependencies.scala | 1 - s3/build.sbt | 1 - .../scala/geotrellis/spark/io/s3/S3LayerProvider.scala | 8 +------- spark/build.sbt | 1 - .../main/scala/geotrellis/spark/io/AttributeStore.scala | 1 - .../geotrellis/spark/io/file/FileLayerProvider.scala | 8 +------- .../geotrellis/spark/io/hadoop/HadoopLayerProvider.scala | 7 +------ 13 files changed, 6 insertions(+), 51 deletions(-) diff --git a/accumulo/build.sbt b/accumulo/build.sbt index 5571ff48d1..02340b1870 100644 --- a/accumulo/build.sbt +++ b/accumulo/build.sbt @@ -7,7 +7,6 @@ libraryDependencies ++= Seq( exclude("org.apache.hadoop", "hadoop-client"), sparkCore % "provided", spire, - scaffeine, scalatest % "test") fork in Test := false diff --git a/accumulo/src/main/scala/geotrellis/spark/io/accumulo/AccumuloLayerProvider.scala b/accumulo/src/main/scala/geotrellis/spark/io/accumulo/AccumuloLayerProvider.scala index f09c5968e4..9cdbfa4558 100644 --- a/accumulo/src/main/scala/geotrellis/spark/io/accumulo/AccumuloLayerProvider.scala +++ b/accumulo/src/main/scala/geotrellis/spark/io/accumulo/AccumuloLayerProvider.scala @@ -19,16 +19,11 @@ package geotrellis.spark.io.accumulo import geotrellis.spark._ import geotrellis.spark.io._ import geotrellis.util.UriUtils -import com.github.blemale.scaffeine.{Scaffeine, Cache} import org.apache.spark.SparkContext import org.apache.accumulo.core.client.security.tokens.PasswordToken import com.typesafe.config.ConfigFactory import java.net.URI -object AccumuloLayerProvider { - private val cache: Cache[(String, String), AttributeStore] = Scaffeine().softValues().build() -} - /** * Provides [[AccumuloAttributeStore]] instance for URI with `accumulo` scheme. * ex: `accumulo://[user[:password]@]zookeeper/instance-name[?attributes=table1[&layers=table2]]` @@ -44,9 +39,7 @@ class AccumuloLayerProvider extends AttributeStoreProvider with LayerReaderProvi val params = UriUtils.getParams(uri) val attributeTable = params.getOrElse("attributes", ConfigFactory.load().getString("geotrellis.accumulo.catalog")) - - AccumuloLayerProvider.cache.get(uri.getSchemeSpecificPart -> attributeTable, - _ => AccumuloAttributeStore(instance, attributeTable)) + AccumuloAttributeStore(instance, attributeTable) } def layerReader(uri: URI, store: AttributeStore, sc: SparkContext): FilteringLayerReader[LayerId] = { diff --git a/cassandra/build.sbt b/cassandra/build.sbt index cea8d00bf0..a7742eee4c 100644 --- a/cassandra/build.sbt +++ b/cassandra/build.sbt @@ -8,7 +8,6 @@ libraryDependencies ++= Seq( ExclusionRule("org.slf4j"), ExclusionRule("io.spray"), ExclusionRule("com.typesafe.akka") ) exclude("org.apache.hadoop", "hadoop-client"), sparkCore % "provided", - scaffeine, spire, scalatest % "test") diff --git a/cassandra/src/main/scala/geotrellis/spark/io/cassandra/CassandraLayerProvider.scala b/cassandra/src/main/scala/geotrellis/spark/io/cassandra/CassandraLayerProvider.scala index cdf71cf98b..de8f9faa19 100644 --- a/cassandra/src/main/scala/geotrellis/spark/io/cassandra/CassandraLayerProvider.scala +++ b/cassandra/src/main/scala/geotrellis/spark/io/cassandra/CassandraLayerProvider.scala @@ -19,14 +19,9 @@ package geotrellis.spark.io.cassandra import geotrellis.spark._ import geotrellis.spark.io._ import geotrellis.util.UriUtils -import com.github.blemale.scaffeine.{Scaffeine, Cache} import org.apache.spark.SparkContext import java.net.URI -object CassandraLayerProvider { - private val cache: Cache[(String, String), AttributeStore] = Scaffeine().softValues().build() -} - /** * Provides [[CassandraAttributeStore]] instance for URI with `cassandra` scheme. * ex: `cassandra://[user:password@]zookeeper[:port][/keyspace][?attributes=table1[&layers=table2]]` @@ -44,9 +39,7 @@ class CassandraLayerProvider extends AttributeStoreProvider with LayerReaderProv Cassandra.cfg.getString("catalog")) val keyspace = Option(uri.getPath.drop(1)).getOrElse( Cassandra.cfg.getString("keyspace")) - - CassandraLayerProvider.cache.get(uri.getSchemeSpecificPart -> attributeTable, - _ => CassandraAttributeStore(instance, keyspace, attributeTable)) + CassandraAttributeStore(instance, keyspace, attributeTable) } def layerReader(uri: URI, store: AttributeStore, sc: SparkContext): FilteringLayerReader[LayerId] = { diff --git a/hbase/build.sbt b/hbase/build.sbt index a2862b7331..5bcd1276c9 100644 --- a/hbase/build.sbt +++ b/hbase/build.sbt @@ -9,7 +9,6 @@ libraryDependencies ++= Seq( "org.codehaus.jackson" % "jackson-core-asl" % "1.9.13", sparkCore % "provided", spire, - scaffeine, scalatest % "test") fork in Test := false diff --git a/hbase/src/main/scala/geotrellis/spark/io/hbase/HBaseLayerProvider.scala b/hbase/src/main/scala/geotrellis/spark/io/hbase/HBaseLayerProvider.scala index 3fc3b36207..d668dba65f 100644 --- a/hbase/src/main/scala/geotrellis/spark/io/hbase/HBaseLayerProvider.scala +++ b/hbase/src/main/scala/geotrellis/spark/io/hbase/HBaseLayerProvider.scala @@ -19,15 +19,10 @@ package geotrellis.spark.io.hbase import geotrellis.spark._ import geotrellis.spark.io._ import geotrellis.util.UriUtils -import com.github.blemale.scaffeine.{Scaffeine, Cache} import com.typesafe.config.ConfigFactory import org.apache.spark.SparkContext import java.net.URI -object HBaseLayerProvider { - private val cache: Cache[(String, String), AttributeStore] = Scaffeine().softValues().build() -} - /** * Provides [[HBaseAttributeStore]] instance for URI with `hbase` scheme. * ex: `hbase://zookeeper[:port][?master=host][?attributes=table1[&layers=table2]]` @@ -43,9 +38,7 @@ class HBaseLayerProvider extends AttributeStoreProvider with LayerReaderProvider val params = UriUtils.getParams(uri) val attributeTable = params.getOrElse("attributes", ConfigFactory.load().getString("geotrellis.hbase.catalog")) - - HBaseLayerProvider.cache.get(uri.getSchemeSpecificPart -> attributeTable, - _ => HBaseAttributeStore(instance, attributeTable)) + HBaseAttributeStore(instance, attributeTable) } def layerReader(uri: URI, store: AttributeStore, sc: SparkContext): FilteringLayerReader[LayerId] = { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b705022455..352cd9417f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -17,7 +17,6 @@ import sbt._ object Dependencies { - val scaffeine = "com.github.blemale" %% "scaffeine" % "2.2.0" val typesafeConfig = "com.typesafe" % "config" % "1.3.1" val logging = "com.typesafe.scala-logging" %% "scala-logging" % "3.5.0" val scalatest = "org.scalatest" %% "scalatest" % "3.0.1" diff --git a/s3/build.sbt b/s3/build.sbt index c07ceb65cd..85a343a5f3 100644 --- a/s3/build.sbt +++ b/s3/build.sbt @@ -5,7 +5,6 @@ libraryDependencies ++= Seq( sparkCore % "provided", awsSdkS3, spire, - scaffeine, scalatest % "test") fork in Test := false diff --git a/s3/src/main/scala/geotrellis/spark/io/s3/S3LayerProvider.scala b/s3/src/main/scala/geotrellis/spark/io/s3/S3LayerProvider.scala index ab898e2b44..0155acc37f 100644 --- a/s3/src/main/scala/geotrellis/spark/io/s3/S3LayerProvider.scala +++ b/s3/src/main/scala/geotrellis/spark/io/s3/S3LayerProvider.scala @@ -20,13 +20,8 @@ import geotrellis.spark._ import geotrellis.spark.io._ import org.apache.spark._ import com.amazonaws.services.s3.AmazonS3URI -import com.github.blemale.scaffeine.{Scaffeine, Cache} import java.net.URI -object S3LayerProvider { - private val cache: Cache[String, AttributeStore] = Scaffeine().softValues().build() -} - /** * Provides [[S3LayerReader]] instance for URI with `s3` scheme. * The uri represents S3 bucket an prefix of catalog root. @@ -38,8 +33,7 @@ class S3LayerProvider extends AttributeStoreProvider def attributeStore(uri: URI): AttributeStore = { val s3Uri = new AmazonS3URI(uri) - S3LayerProvider.cache.get(uri.getSchemeSpecificPart, - _ => new S3AttributeStore(bucket = s3Uri.getBucket, prefix = s3Uri.getKey)) + new S3AttributeStore(bucket = s3Uri.getBucket, prefix = s3Uri.getKey) } def layerReader(uri: URI, store: AttributeStore, sc: SparkContext): FilteringLayerReader[LayerId] = { diff --git a/spark/build.sbt b/spark/build.sbt index ebbc71160d..e4dcd841b2 100644 --- a/spark/build.sbt +++ b/spark/build.sbt @@ -11,7 +11,6 @@ libraryDependencies ++= Seq( monocleCore, monocleMacro, chronoscala, scalazStream, - scaffeine, scalatest % "test" ) diff --git a/spark/src/main/scala/geotrellis/spark/io/AttributeStore.scala b/spark/src/main/scala/geotrellis/spark/io/AttributeStore.scala index 0279e2fc38..2fdc542f54 100644 --- a/spark/src/main/scala/geotrellis/spark/io/AttributeStore.scala +++ b/spark/src/main/scala/geotrellis/spark/io/AttributeStore.scala @@ -60,7 +60,6 @@ object AttributeStore { /** * Produce AttributeStore instance based on URI description. * This method uses instances of [[AttributeServiceProvider]] loaded through Java SPI. - * Repeated calls to this function will return previosly instantiated instances. */ def apply(uri: URI): AttributeStore = { import scala.collection.JavaConversions._ diff --git a/spark/src/main/scala/geotrellis/spark/io/file/FileLayerProvider.scala b/spark/src/main/scala/geotrellis/spark/io/file/FileLayerProvider.scala index 441ed1cf33..baa05aae27 100644 --- a/spark/src/main/scala/geotrellis/spark/io/file/FileLayerProvider.scala +++ b/spark/src/main/scala/geotrellis/spark/io/file/FileLayerProvider.scala @@ -18,15 +18,10 @@ package geotrellis.spark.io.file import geotrellis.spark._ import geotrellis.spark.io._ -import com.github.blemale.scaffeine.{Scaffeine, Cache} import org.apache.spark.SparkContext import java.net.URI import java.io.File -object FileLayerProvider { - private val cache: Cache[String, AttributeStore] = Scaffeine().softValues().build() -} - /** * Provides [[FileLayerReader]] instance for URI with `file` scheme. * The uri represents local path to catalog root. @@ -39,8 +34,7 @@ class FileLayerProvider extends AttributeStoreProvider def attributeStore(uri: URI): AttributeStore = { val file = new File(uri) - FileLayerProvider.cache.get(file.getCanonicalPath, - canonicalPath => new FileAttributeStore(canonicalPath)) + new FileAttributeStore(file.getCanonicalPath) } def layerReader(uri: URI, store: AttributeStore, sc: SparkContext): FilteringLayerReader[LayerId] = { diff --git a/spark/src/main/scala/geotrellis/spark/io/hadoop/HadoopLayerProvider.scala b/spark/src/main/scala/geotrellis/spark/io/hadoop/HadoopLayerProvider.scala index f2620d994d..4bc830b3ea 100644 --- a/spark/src/main/scala/geotrellis/spark/io/hadoop/HadoopLayerProvider.scala +++ b/spark/src/main/scala/geotrellis/spark/io/hadoop/HadoopLayerProvider.scala @@ -19,16 +19,11 @@ package geotrellis.spark.io.hadoop import geotrellis.spark._ import geotrellis.spark.io._ import geotrellis.util.UriUtils -import com.github.blemale.scaffeine.{Scaffeine, Cache} import org.apache.hadoop.fs.Path import org.apache.hadoop.conf.Configuration import org.apache.spark.SparkContext import java.net.URI -object HadoopLayerProvider { - private val cache: Cache[Path, AttributeStore] = Scaffeine().softValues().build() -} - /** * Provides [[HadoopAttributeStore]] instance for URI with `hdfs`, `hdfs+file`, `s3n`, and `s3a` schemes. * The uri represents Hadoop [[Path]] of catalog root. @@ -49,7 +44,7 @@ class HadoopLayerProvider extends AttributeStoreProvider def attributeStore(uri: URI): AttributeStore = { val path = new Path(trim(uri)) val conf = new Configuration() - HadoopLayerProvider.cache.get(path, new HadoopAttributeStore(_, conf)) + new HadoopAttributeStore(path, conf) } def layerReader(uri: URI, store: AttributeStore, sc: SparkContext): FilteringLayerReader[LayerId] = {