New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run includeCurliesInChains=true #12

Open
wants to merge 1 commit into
base: 0.4.6
from
The table of contents is too big for display.
+19,058 −14,310
Diff settings

Always

Just for now

View
@@ -1 +1,2 @@
project.git = true
includeCurliesInChains = true
@@ -772,8 +772,8 @@ object Engine {
qas.map { case (qx, (q, a)) => (qx, (serving.supplementBase(q), a)) }
}
val algoPredictsMap: Map[EX, RDD[(QX, Seq[P])]] = (0 until evalCount).map {
ex =>
val algoPredictsMap: Map[EX, RDD[(QX, Seq[P])]] = (0 until evalCount)
.map { ex =>
{
val modelMap: Map[AX, Any] = algoModelsMap(ex)
@@ -807,7 +807,8 @@ object Engine {
(ex, unionAlgoPredicts)
}
}.toMap
}
.toMap
val servingQPAMap: Map[EX, RDD[(Q, P, A)]] = algoPredictsMap.map {
case (ex, psMap) => {
@@ -825,11 +826,13 @@ object Engine {
}
}
(0 until evalCount).map { ex =>
{
(evalInfoMap(ex), servingQPAMap(ex))
(0 until evalCount)
.map { ex =>
{
(evalInfoMap(ex), servingQPAMap(ex))
}
}
}.toSeq
.toSeq
}
}
@@ -130,30 +130,34 @@ object FastEvalEngineWorkflow {
prefix: AlgorithmsPrefix): Map[EX, RDD[(QX, Seq[P])]] = {
val algoMap: Map[AX, BaseAlgorithm[PD, _, Q, P]] =
prefix.algorithmParamsList.map {
case (algoName, algoParams) => {
try {
Doer(workflow.engine.algorithmClassMap(algoName), algoParams)
} catch {
case e: NoSuchElementException => {
val algorithmClassMap = workflow.engine.algorithmClassMap
if (algoName == "") {
logger.error(
"Empty algorithm name supplied but it could not " +
"match with any algorithm in the engine's definition. " +
"Existing algorithm name(s) are: " +
s"${algorithmClassMap.keys.mkString(", ")}. Aborting.")
} else {
logger.error(
s"${algoName} cannot be found in the engine's " +
"definition. Existing algorithm name(s) are: " +
s"${algorithmClassMap.keys.mkString(", ")}. Aborting.")
prefix.algorithmParamsList
.map {
case (algoName, algoParams) => {
try {
Doer(workflow.engine.algorithmClassMap(algoName), algoParams)
} catch {
case e: NoSuchElementException => {
val algorithmClassMap = workflow.engine.algorithmClassMap
if (algoName == "") {
logger.error(
"Empty algorithm name supplied but it could not " +
"match with any algorithm in the engine's definition. " +
"Existing algorithm name(s) are: " +
s"${algorithmClassMap.keys.mkString(", ")}. Aborting.")
} else {
logger.error(
s"${algoName} cannot be found in the engine's " +
"definition. Existing algorithm name(s) are: " +
s"${algorithmClassMap.keys.mkString(", ")}. Aborting.")
}
sys.exit(1)
}
sys.exit(1)
}
}
}
}.zipWithIndex.map(_.swap).toMap
.zipWithIndex
.map(_.swap)
.toMap
val algoCount = algoMap.size
@@ -169,38 +173,41 @@ object FastEvalEngineWorkflow {
workflow = workflow,
prefix = new DataSourcePrefix(prefix))
val algoResult: Map[EX, RDD[(QX, Seq[P])]] = dataSourceResult.par.map {
case (ex, (td, ei, iqaRDD)) => {
val modelsMap: Map[AX, Any] = algoModelsMap(ex)
val qs: RDD[(QX, Q)] = iqaRDD.mapValues(_._1)
val algoPredicts: Seq[RDD[(QX, (AX, P))]] = (0 until algoCount).map {
ax =>
{
val algo = algoMap(ax)
val model = modelsMap(ax)
val rawPredicts: RDD[(QX, P)] =
algo.batchPredictBase(workflow.sc, model, qs)
val predicts: RDD[(QX, (AX, P))] = rawPredicts.map {
case (qx, p) => (qx, (ax, p))
val algoResult: Map[EX, RDD[(QX, Seq[P])]] = dataSourceResult.par
.map {
case (ex, (td, ei, iqaRDD)) => {
val modelsMap: Map[AX, Any] = algoModelsMap(ex)
val qs: RDD[(QX, Q)] = iqaRDD.mapValues(_._1)
val algoPredicts: Seq[RDD[(QX, (AX, P))]] = (0 until algoCount).map {
ax =>
{
val algo = algoMap(ax)
val model = modelsMap(ax)
val rawPredicts: RDD[(QX, P)] =
algo.batchPredictBase(workflow.sc, model, qs)
val predicts: RDD[(QX, (AX, P))] = rawPredicts.map {
case (qx, p) => (qx, (ax, p))
}
predicts
}
predicts
}
}
}
val unionAlgoPredicts: RDD[(QX, Seq[P])] =
workflow.sc.union(algoPredicts).groupByKey.mapValues { ps =>
{
assert(ps.size == algoCount,
"Must have same length as algoCount")
// TODO. Check size == algoCount
ps.toSeq.sortBy(_._1).map(_._2)
val unionAlgoPredicts: RDD[(QX, Seq[P])] =
workflow.sc.union(algoPredicts).groupByKey.mapValues { ps =>
{
assert(ps.size == algoCount,
"Must have same length as algoCount")
// TODO. Check size == algoCount
ps.toSeq.sortBy(_._1).map(_._2)
}
}
}
(ex, unionAlgoPredicts)
(ex, unionAlgoPredicts)
}
}
}.seq.toMap
.seq
.toMap
algoResult
}
@@ -250,11 +257,13 @@ object FastEvalEngineWorkflow {
}
}
val servingResult = (0 until evalQAsMap.size).map { ex =>
{
(evalInfoMap(ex), servingQPAMap(ex))
val servingResult = (0 until evalQAsMap.size)
.map { ex =>
{
(evalInfoMap(ex), servingQPAMap(ex))
}
}
}.toSeq
.toSeq
cache += Tuple2(prefix, servingResult)
}
@@ -210,13 +210,15 @@ class MetricEvaluator[EI, Q, P, A, R](
params: WorkflowParams): MetricEvaluatorResult[R] = {
val evalResultList: Seq[(EngineParams, MetricScores[R])] =
engineEvalDataSet.zipWithIndex.par.map {
case ((engineParams, evalDataSet), idx) =>
val metricScores =
MetricScores[R](metric.calculate(sc, evalDataSet),
otherMetrics.map(_.calculate(sc, evalDataSet)))
(engineParams, metricScores)
}.seq
engineEvalDataSet.zipWithIndex.par
.map {
case ((engineParams, evalDataSet), idx) =>
val metricScores =
MetricScores[R](metric.calculate(sc, evalDataSet),
otherMetrics.map(_.calculate(sc, evalDataSet)))
(engineParams, metricScores)
}
.seq
implicit lazy val formats =
Utils.json4sDefaultFormats + new NameParamsSerializer
@@ -184,25 +184,26 @@ object WorkflowUtils extends Logging {
error(s"Unable to extract $field name and params $jv")
throw e
}
val extractedParams = np.params.map { p =>
try {
if (!classMap.contains(np.name)) {
error(
s"Unable to find $field class with name '${np.name}'" +
val extractedParams = np.params
.map { p =>
try {
if (!classMap.contains(np.name)) {
error(s"Unable to find $field class with name '${np.name}'" +
" defined in Engine.")
sys.exit(1)
sys.exit(1)
}
WorkflowUtils.extractParams(engineLanguage,
compact(render(p)),
classMap(np.name),
jsonExtractor,
formats)
} catch {
case e: Exception =>
error(s"Unable to extract $field params $p")
throw e
}
WorkflowUtils.extractParams(engineLanguage,
compact(render(p)),
classMap(np.name),
jsonExtractor,
formats)
} catch {
case e: Exception =>
error(s"Unable to extract $field params $p")
throw e
}
}.getOrElse(EmptyParams())
.getOrElse(EmptyParams())
(np.name, extractedParams)
} getOrElse ("", EmptyParams())
@@ -254,12 +255,14 @@ object WorkflowUtils extends Logging {
"HADOOP_CONF_DIR" -> "core-site.xml",
"HBASE_CONF_DIR" -> "hbase-site.xml")
thirdPartyFiles.keys.toSeq.map { k: String =>
sys.env.get(k) map { x =>
val p = Seq(x, thirdPartyFiles(k)).mkString(File.separator)
if (new File(p).exists) Seq(p) else Seq[String]()
} getOrElse Seq[String]()
}.flatten
thirdPartyFiles.keys.toSeq
.map { k: String =>
sys.env.get(k) map { x =>
val p = Seq(x, thirdPartyFiles(k)).mkString(File.separator)
if (new File(p).exists) Seq(p) else Seq[String]()
} getOrElse Seq[String]()
}
.flatten
}
def thirdPartyClasspaths: Seq[String] = {
Oops, something went wrong.