Skip to content

Commit

Permalink
Merge pull request #58 from iantmoore/clean-config-rendering
Browse files Browse the repository at this point in the history 
tidied up config rendering and masking of secrets
  • Loading branch information
@iantmoore
iantmoore committed Aug 17, 2017
2 parents f217ef1 + b1aaf46 commit 9aced60
Show file tree
Hide file tree
Showing 9 changed files with 134 additions and 155 deletions.
47 changes: 39 additions & 8 deletions api/src/main/scala/org/substeps/config/SubstepsConfigLoader.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,42 @@ object SubstepsConfigLoader {
val log = LoggerFactory.getLogger("org.substeps.config.SubstepsConfigLoader")
val options: ConfigRenderOptions = ConfigRenderOptions.defaults.setComments(false).setFormatted(true).setJson(false).setOriginComments(false)

def render(cfg: Config): String = cfg.withOnlyPath("org.substeps").root().render(options)
def render(c: Config): String = {

val cfg = c.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))

val cleaned =
if (cfg.hasPath("org.substeps.webdriver.remote.credentials.username")){

val remoteUserName = cfg.getString("org.substeps.webdriver.remote.credentials.username")
val remoteToken = cfg.getString("org.substeps.webdriver.remote.credentials.token")

(cfg.withoutPath("org.substeps.webdriver.remote.credentials")
.withValue("org.substeps.webdriver.remote.credentials", ConfigValueFactory.fromMap(Map("username" -> "*****", "token" -> "*****").asJava)),
Some(remoteUserName),
Some(remoteToken))

}
else {
(cfg, None, None)
}

val rendered =
cleaned match {
case (cleanedConfig, Some(userName), Some(token)) => {

cleanedConfig.withOnlyPath("org.substeps")
.root().render(options).replaceAll(userName, "******").replaceAll(token, "******")

}
case (cleanedConfig, _, _) => {
cleanedConfig.withOnlyPath("org.substeps")
.root().render(options)

}
}
rendered
}

def loadResolvedConfig(): Config = {

Expand All @@ -38,7 +73,7 @@ object SubstepsConfigLoader {

val envConfig = ConfigFactory.parseFile(environmentConfigFile, ConfigParseOptions.defaults().setAllowMissing(true))

log.debug("Env config from file (" + environmentConfigFile.getAbsolutePath() + "):\n" + NewSubstepsExecutionConfig.render(envConfig))
log.debug("Env config from file (" + environmentConfigFile.getAbsolutePath() + "):\n" + SubstepsConfigLoader.render(envConfig))


loadResolvedConfig(mavenConfigSettings, envConfig)
Expand All @@ -56,11 +91,7 @@ object SubstepsConfigLoader {
val masterCfg =
ConfigFactory.load(ConfigParseOptions.defaults(), ConfigResolveOptions.noSystem().setAllowUnresolved(true))

log.debug("master config:\n" + NewSubstepsExecutionConfig.render(masterCfg))


resolveConfig(masterCfg, mavenConfigSettings, envConfig)
// loadResolvedConfig(mavenConfigSettings)
}

def environmentConfigFile() = {
Expand All @@ -84,7 +115,7 @@ object SubstepsConfigLoader {

val envConfig = ConfigFactory.parseResources(environmentConfigFile(), ConfigParseOptions.defaults().setAllowMissing(true))

log.debug("Env config:\n" + NewSubstepsExecutionConfig.render(envConfig))
log.debug("Env config:\n" + SubstepsConfigLoader.render(envConfig))
envConfig
}

Expand Down Expand Up @@ -113,7 +144,7 @@ object SubstepsConfigLoader {

val masterConfig = envConfig.withFallback(initialMasterConfig).resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true))

log.debug("masterConfig:\n" + NewSubstepsExecutionConfig.render(masterConfig))
log.debug("masterConfig:\n" + SubstepsConfigLoader.render(masterConfig))

val baseExecutionConfig = masterConfig.getConfig("org.substeps.baseExecutionConfig")

Expand Down
135 changes: 2 additions & 133 deletions api/src/main/scala/org/substeps/runner/NewSubstepsExecutionConfig.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,11 @@ package org.substeps.runner

import java.io.File

import com.google.common.collect.Lists
import com.technophobia.substeps.model.{Configuration, SubSteps}
import com.technophobia.substeps.model.exception.SubstepsConfigurationException
import com.technophobia.substeps.runner.{IExecutionListener, SubstepsExecutionConfig}
import com.typesafe.config._
import org.slf4j.{Logger, LoggerFactory}
import org.substeps.config.SubstepsConfigLoader
import org.substeps.report.{IExecutionResultsCollector, IReportBuilder}

import scala.collection.JavaConverters._
Expand Down Expand Up @@ -154,9 +153,6 @@ object NewSubstepsExecutionConfig extends SubstepsConfigKeys {

val dir = new File(cfg.getString(`rootDataDirKey`), cfg.getString(`executionConfigDataOutputDir`))

// if (!dir.exists()){
// throw new SubstepsConfigurationException(s"data dir ${dir.getAbsolutePath()} for execution config doesn't exist")
// }
dir
}

Expand All @@ -165,8 +161,6 @@ object NewSubstepsExecutionConfig extends SubstepsConfigKeys {
// transitional method
def toConfig(substepsConfig : SubstepsExecutionConfig) : Config = {

import com.google.common.collect.Lists

val execConfig1 = new java.util.HashMap[String, AnyRef]
execConfig1.put("description", Option(substepsConfig.getDescription).getOrElse("Substeps test execution description"))
execConfig1.put("featureFile", substepsConfig.getFeatureFile)
Expand Down Expand Up @@ -199,103 +193,18 @@ object NewSubstepsExecutionConfig extends SubstepsConfigKeys {

if (!substepsConfig.isFastFailParseErrors) execConfig1.put("fastFailParseErrors", Boolean.box(substepsConfig.isFastFailParseErrors))





execConfig1.put("dataOutputDir", "1")


val cfg = ConfigFactory.empty
.withValue("org.substeps.executionConfigs", ConfigValueFactory.fromIterable(List(ConfigValueFactory.fromMap(execConfig1)).asJava) )
// .withValue("org.substeps.config.jmxPort", ConfigValueFactory.fromAnyRef(this.jmxPort))
// .withValue("org.substeps.config.vmArgs", ConfigValueFactory.fromAnyRef(this.vmArgs))

// these are provided in ref.conf
//.withValue("org.substeps.config.executionResultsCollector", ConfigValueFactory.fromAnyRef("org.substeps.report.ExecutionResultsCollector"))
//.withValue("org.substeps.config.reportBuilder", ConfigValueFactory.fromAnyRef("org.substeps.report.ReportBuilder"))


// this is to pick up the reference.conf
cfg.withFallback(ConfigFactory.load(ConfigParseOptions.defaults(), ConfigResolveOptions.noSystem().setAllowUnresolved(true)))
}



val options: ConfigRenderOptions = ConfigRenderOptions.defaults.setComments(false).setFormatted(true).setJson(false).setOriginComments(false)

def render(cfg : Config) : String = {
// TODO - trim out bits of the config we're not interested in
cfg.root()
.withoutKey("awt")
.withoutKey("java")
.withoutKey("line")
.withoutKey("os")
.withoutKey("sun")
.withoutKey("user")
.withValue("remote.token", ConfigValueFactory.fromAnyRef("******"))
.withValue("remote.username", ConfigValueFactory.fromAnyRef("******"))
// TODO - correct the webdriver paths here or work out how to mask certain fields - surely some config ?

.render(options)
}




// def toConfigList(cfgFileList : java.util.List[String], mavenConfig : Config) = {
//
// cfgFileList.asScala.flatMap(cfgFile =>{
//
// loadConfig(cfgFile, Some(mavenConfig))
// }).asJava
// }

// def loadConfig( cfgFile: String): List[Config] = {
// loadConfig(cfgFile, None)
// }

// def loadConfig( cfgFile: String, mvnConfigOption : Option[Config]): List[Config] = {
//
// val base = ConfigFactory.load(cfgFile, ConfigParseOptions.defaults(), ConfigResolveOptions.defaults().setAllowUnresolved(true))
//
// loadConfig(base, mvnConfigOption)
// }
//
//
// def loadConfig( base: Config, mvnConfigOption : Option[Config]): List[Config] = {
//
// val masterConfig = loadMasterConfig(base, mvnConfigOption)
// // there might be multilple execonfigs in there - return multiple configs for each one
// splitConfig(masterConfig)
// }



// def loadMasterConfig( base: Config): Config = {
// loadMasterConfig(base, None)
// }
//
// def loadMasterConfig( base: Config, mvnConfigOption : Option[Config]): Config = {
// val environment = System.getProperty("ENVIRONMENT", "localhost") + ".conf"
//
// val envConfig = ConfigFactory.load(environment)
//
//
// val masterConfig =
// mvnConfigOption match {
// case Some(mvnConfig) => {
// log.debug("MAVEN CONFIG:\n" + render(mvnConfig))
//
// envConfig.withFallback(base).withFallback(mvnConfig).resolve()
// }
// case None => envConfig.withFallback(base).resolve()
//
// }
//
// log.debug("LOADED MASTER CONFIG:\n" + render(masterConfig))
// masterConfig
// }

val legacyConfigKeys = List("step.depth.description",
"log.unused.uncalled",
Expand Down Expand Up @@ -324,7 +233,7 @@ object NewSubstepsExecutionConfig extends SubstepsConfigKeys {
"* YOUR CONFIG CONTAINS LEGACY OVERRIDE KEYS !! *\n" +
"****************************************************************************\n" +
"\nAll Substeps keys have now moved under org.substeps and can be overriden like this:\n"
+ NewSubstepsExecutionConfig.render(masterConfig.getConfig("org.substeps")))
+ SubstepsConfigLoader.render(masterConfig.getConfig("org.substeps")))
log.warn("Overrides will currently still work but support for them will be removed in a subsequent release")
}
case _ =>
Expand All @@ -333,33 +242,6 @@ object NewSubstepsExecutionConfig extends SubstepsConfigKeys {
}




// def splitConfigAsJava(masterConfig : Config) = {
// splitConfig(masterConfig).asJava
// }
//
// def splitConfig(masterConfig : Config): List[Config] = {
// val exeConfigList = masterConfig.getConfigList("org.substeps.config.executionConfigs").asScala
//
// val baseConfig = masterConfig.withoutPath("org.substeps.config.executionConfigs")
//
// exeConfigList.map(ec => {
// baseConfig.withValue("org.substeps.config.executionConfig", ec.root())
//
// }).toList
// }
//
// def splitConfigAsOne(masterConfig : Config): Config = {
// splitConfig(masterConfig).head
// }







def getInitialisationClasses(cfg : Config) : Array[Class[_]]= {

if (cfg.hasPath(`initialisationClassesKey`)) {
Expand Down Expand Up @@ -509,17 +391,6 @@ object NewSubstepsExecutionConfig extends SubstepsConfigKeys {
new File(cfg.getString(`substepsReportDir`))
}

// def buildInitialisationClassList(cfg: Config) : Array[Class[_]] = {
//
// val stepImplementationClasses = NewSubstepsExecutionConfig.getStepImplementationClasses(cfg)
// val initialisationClasses = NewSubstepsExecutionConfig.getInitialisationClasses(cfg)
// var initClassList = null
// if (initialisationClasses != null) initClassList = Lists.newArrayList(initialisationClasses)
//
// ExecutionConfigWrapper.buildInitialisationClassList(stepImplementationClasses, initClassList);
// }


def substepsConfig: Config = threadLocalConfig()

def stepDepthForDescription = substepsConfig.getInt(`stepDepthDescriptionKey`)
Expand All @@ -528,6 +399,4 @@ object NewSubstepsExecutionConfig extends SubstepsConfigKeys {

def prettyPrintReportData = substepsConfig.getBoolean(`prettyPrintReportDataKey`)

// def reportDataBaseDir = substepsConfig.getString(`reportDataBaseDirKey`)

}
22 changes: 22 additions & 0 deletions api/src/test/resources/test-conf-with-defaults.conf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# a properties file more suited to local development and test - using a visual browser rather than htmlunit
# and keeping the window open on failure

wait.seconds=10
#base.url="src/web"
base.url="http://substeps.github.io/substeps-webdriver/"
#driver.type=HTMLUNIT
#driver.type=FIREFOX
driver.type=CHROME
# FIREFOX or HTMLUNIT, or CHROME, sorry not tested with IE yet


default.webdriver.timeout.secs=10

# this flag prevents webdriver from shutting down after the tests have finished, useful if using a visual webdriver and debugging
#webdriver.shutdown=false
visual.webdriver.close.on.fail=false

step.depth.description=5



31 changes: 31 additions & 0 deletions api/src/test/resources/travis.conf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@

external.content="https://raw.githubusercontent.com/Substeps/substeps-webdriver/master/README.md"

test.filename="one.file"
test.filename2="two.file"

iframe.test.page="/iframe-test.html"

org {
substeps {
baseExecutionConfig {
webdriver {
driver.type=REMOTE
remote.driver.url="https://"${org.substeps.webdriver.remote.credentials.username}":"${org.substeps.webdriver.remote.credentials.token}"@ondemand.saucelabs.com:443/wd/hub"
remote.driver.platform=Linux
}

}
webdriver {
remote {
credentials {
username = "saucelabsuser"
username=${?SAUCE_USERNAME}

token = "acess-key"
token=${?SAUCE_ACCESS_KEY}
}
}
}
}
}
32 changes: 28 additions & 4 deletions api/src/test/scala/org.substeps.config/ConfigTest.scala
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
package org.substeps.config

import com.typesafe.config.ConfigFactory
import org.scalatest.{FlatSpec,Matchers}
import java.util
import java.util.List

import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions}
import org.hamcrest.core.IsNot.not
import org.junit.Assert.assertThat
import org.scalatest.{FlatSpec, Matchers}
import org.substeps.runner.NewSubstepsExecutionConfig

/**
Expand All @@ -13,14 +18,33 @@ class ConfigTest extends FlatSpec with Matchers{

val cfg = ConfigFactory.load("test-conf-with-defaults.conf")

println("cfg root:\n" + NewSubstepsExecutionConfig.render(cfg))
// println("cfg root:\n" + NewSubstepsExecutionConfig.render(cfg))

val substepsCfg = cfg.getConfig("org.substeps")

NewSubstepsExecutionConfig.checkMasterConfigForLegacyDefaults(cfg)

println("#8888888888888888888\n\nsubstepsCfg root:\n" + NewSubstepsExecutionConfig.render(substepsCfg))
println("#8888888888888888888\n\nsubstepsCfg root:\n" + SubstepsConfigLoader.render(substepsCfg))


}

"rendering of config" must "not include remote credentials" in {


System.setProperty("environment", "travis")
System.setProperty("SAUCE_USERNAME", "saucelabsuser")
System.setProperty("SAUCE_ACCESS_KEY", "access-token")
val cfg: Config = SubstepsConfigLoader.loadResolvedConfig
val options: ConfigRenderOptions = ConfigRenderOptions.defaults.setComments(false).setFormatted(true).setJson(false).setOriginComments(false)
val cfgString: String = cfg.root.withoutKey("awt").withoutKey("java").withoutKey("line").withoutKey("os").withoutKey("sun").withoutKey("user").render(options)
val configs: util.List[Config] = SubstepsConfigLoader.splitMasterConfig(cfg)
val renderedConfig: String = SubstepsConfigLoader.render(cfg)

System.out.println("MASTER CFG:\n" + renderedConfig + "\n\n\n\n\n")

renderedConfig should not contain ("saucelabsuser")
renderedConfig should not contain ("access-token")

}
}
Loading

0 comments on commit 9aced60

Please sign in to comment.