Skip to content

Commit

Permalink
update code to support cos scheme + support multiple configs
Browse files Browse the repository at this point in the history
  • Loading branch information
bassel-zeidan committed Sep 22, 2017
1 parent f857cca commit 74463f2
Showing 1 changed file with 12 additions and 5 deletions.
17 changes: 12 additions & 5 deletions scala/src/main/scala/Osconfig.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package com.ibm.ibmos2spark
import scala.collection.mutable.HashMap
import org.apache.spark.SparkContext

val DEFAULT_SERVICE_NAME = "service"

object urlbuilder{
def swifturl2d(name: String, container_name: String, object_name: String): String = {
Expand Down Expand Up @@ -155,7 +156,7 @@ class bluemix(sc: SparkContext, name: String, creds: HashMap[String, String],
Warning: creating a new instance of this class would overwrite the existing
spark hadoop configs if set before if used with the same spark context instance.
*/
class CloudObjectStorage(sc: SparkContext, credentials: HashMap[String, String], cosId: String = "") {
class CloudObjectStorage(sc: SparkContext, credentials: HashMap[String, String], configurationName: String = "") {

// check if all credentials are available
val requiredValues = Array("endPoint", "accessKey", "secretKey")
Expand All @@ -167,16 +168,22 @@ class CloudObjectStorage(sc: SparkContext, credentials: HashMap[String, String],

// set config
val hadoopConf = sc.hadoopConfiguration
val prefix = "fs.s3d.service"
val prefix = "fs.cos." + getConfigurationName()

hadoopConf.set(prefix + ".endpoint", credentials("endPoint"))
hadoopConf.set(prefix + ".access.key", credentials("accessKey"))
hadoopConf.set(prefix + ".secret.key", credentials("secretKey"))

def getCosId() : String = {
return cosId
private def getConfigurationName() : String = {
if (configurationName != "") {
return configurationName
} else {
return DEFAULT_SERVICE_NAME
}
}

def url(bucketName: String, objectName: String) : String = {
return "s3d://" + bucketName + ".service/" + objectName
var serviceName = getConfigurationName()
return "cos://" + bucketName + "." + serviceName + "/" + objectName
}
}

0 comments on commit 74463f2

Please sign in to comment.