Skip to content

Commit

Permalink
extend params for COS class and add validation function
Browse files Browse the repository at this point in the history
  • Loading branch information
bassel-zeidan committed Oct 2, 2017
1 parent 3f7b519 commit 0792d63
Showing 1 changed file with 13 additions and 5 deletions.
18 changes: 13 additions & 5 deletions scala/src/main/scala/Osconfig.scala
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,13 @@ class bluemix(sc: SparkContext, name: String, creds: HashMap[String, String],
multiple configurations to different Object Storage accounts.
if a configuration name is not passed the default one will be used "service".
*/
class CloudObjectStorage(sc: SparkContext, credentials: HashMap[String, String], configurationName: String = "") {
class CloudObjectStorage(sc: SparkContext, credentials: HashMap[String, String],
configurationName: String = "", cos_type="softlayer_cos",
auth_method="api_key") {

// check for valid credentials
_validate_credentials(credentials, cos_type, auth_method)

// check if all credentials are available
val requiredValues = Array("endPoint", "accessKey", "secretKey")
for ( key <- requiredValues ) {
if (!credentials.contains(key)) {
Expand All @@ -161,22 +165,26 @@ class CloudObjectStorage(sc: SparkContext, credentials: HashMap[String, String],

// set config
val hadoopConf = sc.hadoopConfiguration
val prefix = "fs.cos." + getConfigurationName()
val prefix = "fs.cos." + _getConfigurationName()

hadoopConf.set(prefix + ".endpoint", credentials("endPoint"))
hadoopConf.set(prefix + ".access.key", credentials("accessKey"))
hadoopConf.set(prefix + ".secret.key", credentials("secretKey"))

private def getConfigurationName() : String = {
private def _getConfigurationName() : String = {
if (configurationName != "") {
return configurationName
} else {
return globalVariables.DEFAULT_SERVICE_NAME
}
}

private def _validate_credentials() {

}

def url(bucketName: String, objectName: String) : String = {
var serviceName = getConfigurationName()
var serviceName = _getConfigurationName()
return "cos://" + bucketName + "." + serviceName + "/" + objectName
}
}

0 comments on commit 0792d63

Please sign in to comment.