Skip to content

Commit

Permalink
Relax safeguards on delete methods
Browse files Browse the repository at this point in the history
Prior to this change, delete methods threw in any case where the
reference to a layer happened to be faulty. This is based on metadata
which, though always present in case of tile data (and necessary for
its deletion), might not be present. Lacking this metadata, we should
log information about the failure and delete any _attribute data we can.
  • Loading branch information
moradology committed Mar 2, 2017
1 parent 3517ac2 commit 02a84a5
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ class S3AttributeStore(val bucket: String, val prefix: String) extends BlobLayer
.exists(_.getKey.endsWith(s"${AttributeStore.Fields.metadata}${SEP}${layerId.name}${SEP}${layerId.zoom}.json"))

def delete(layerId: LayerId, attributeName: String): Unit = {
if(!layerExists(layerId)) throw new LayerNotFoundError(layerId)
s3Client.deleteObject(bucket, attributePath(layerId, attributeName))
clearCache(layerId, attributeName)
}
Expand All @@ -112,7 +111,6 @@ class S3AttributeStore(val bucket: String, val prefix: String) extends BlobLayer
}

def delete(layerId: LayerId): Unit = {
if(!layerExists(layerId)) throw new LayerNotFoundError(layerId)
layerKeys(layerId).foreach(s3Client.deleteObject(bucket, _))
clearCache(layerId)
}
Expand Down
25 changes: 14 additions & 11 deletions s3/src/main/scala/geotrellis/spark/io/s3/S3LayerDeleter.scala
Original file line number Diff line number Diff line change
Expand Up @@ -18,26 +18,29 @@ package geotrellis.spark.io.s3

import geotrellis.spark.LayerId
import geotrellis.spark.io._
import geotrellis.util.LazyLogging

import spray.json.JsonFormat
import spray.json.DefaultJsonProtocol._

class S3LayerDeleter(val attributeStore: AttributeStore) extends LayerDeleter[LayerId] {
class S3LayerDeleter(val attributeStore: AttributeStore) extends LazyLogging with LayerDeleter[LayerId] {

def getS3Client: () => S3Client = () => S3Client.DEFAULT

def delete(id: LayerId): Unit = {
if (!attributeStore.layerExists(id)) throw new LayerNotFoundError(id)
val header = try {
attributeStore.readHeader[S3LayerHeader](id)
try {
val header = attributeStore.readHeader[S3LayerHeader](id)
val bucket = header.bucket
val prefix = header.key
val s3Client = getS3Client()

s3Client.deleteListing(bucket, s3Client.listObjects(bucket, prefix))
} catch {
case e: AttributeNotFoundError => throw new LayerDeleteError(id).initCause(e)
case e: AttributeNotFoundError =>
logger.info(s"Metadata for $id was not found. Any associated layer data (if any) will require manual deletion")
case e: Exception =>
throw e
}

val bucket = header.bucket
val prefix = header.key
val s3Client = getS3Client()

s3Client.deleteListing(bucket, s3Client.listObjects(bucket, prefix))
attributeStore.delete(id)
}
}
Expand Down

0 comments on commit 02a84a5

Please sign in to comment.