Skip to content

Commit

Permalink
Work in progress
Browse files Browse the repository at this point in the history
  • Loading branch information
@fanf
fanf committed Jun 14, 2018
1 parent da5261a commit b8e07ae
Show file tree
Hide file tree
Showing 2 changed files with 190 additions and 292 deletions.
279 changes: 69 additions & 210 deletions rudder-core/src/main/scala/com/normation/rudder/services/reports/ExecutionBatch.scala
View file
Expand Up @@ -1026,13 +1026,14 @@ object ExecutionBatch extends Loggable {

// an utility class that store an expected value and the list of mathing reports for it
final case class Value(
value : String
, unexpanded : String
, cardinality : Int // number of expected reports, most of the time '1'
, isVar : Boolean
, pattern : Pattern
, specificity : Int // how specific the pattern is. ".*" is 0 (not specific at all), "foobarbaz" is 9.
, matchingReports: List[Reports]
value : String
, unexpanded : String
, cardinality : Int // number of expected reports, most of the time '1'
, numberDuplicates: Int // the number of dropped duplicated message for that pairing, so that we can know how bad syslog is. Should be 0.
, isVar : Boolean
, pattern : Pattern
, specificity : Int // how specific the pattern is. ".*" is 0 (not specific at all), "foobarbaz" is 9.
, matchingReports : List[Reports]
)

/*
Expand All @@ -1045,18 +1046,46 @@ object ExecutionBatch extends Loggable {
*/
def recPairReport(reports: List[ResultReports], freeValues: List[Value], pairedValues: List[Value], unexpected: List[ResultReports], mode: UnexpectedReportInterpretation): (List[Value], List[ResultReports]) = {
// utility function: given a list of Values and one report, try to find a value whose pattern matches reports component value.
// If the predicate for incrementing expected reports cardinality is met, also increment cardinality
// the first matching value is used (they must be sorted by specificity).
// Returns (list of unmatched values, Option[matchedValue])
def findMachingValue(report: ResultReports, values: List[Value], incrementCardinality: (Value, ResultReports) => Boolean): (List[Value], Option[Value]) = {
// We also have two special case to deal with:
// - in some case, we want to ignore a report totally (ie: it must not change compliance). This is typically for
// what we want for duplicated reports. We need to log the report erasure as it won't be displayed in UI (by
// design). The log level should be "info" and not more because it was chosen by configuration to ignore them.
// - in some case, we want to accept more reports than originally expected. Then, we must update cardinality to
// trace that decision. It's typically what we want to do for
def findMachingValue(
report : ResultReports
, values : List[Value]
, dropDuplicated : (Value, ResultReports) => Boolean
, incrementCardinality: (Value, ResultReports) => Boolean
): (List[Value], Option[Value]) = {
values.foldLeft((Nil:List[Value]), Option.empty[Value]) { case ((stack, found), value) =>
found match {
case Some(x) => (value :: stack, Some(x))
case None =>
if(value.pattern.matcher(report.keyValue).matches()) {
val card = value.cardinality + (if(incrementCardinality(value, report)) 1 else 0)
val (r, nbDup) = if(dropDuplicated(value, report)) {
val msg = s"Following report is duplicated and will be ignored because of Rudder setting choice: ${report.toString}"
if(value.numberDuplicates <= 0) { //first time is an info
logger.info(msg)
(value.matchingReports, 1)
} else if(value.numberDuplicates == 1) { //second time is a warning
logger.warn(msg)
(value.matchingReports, 2)
} else { // more than two times: log error and let the report leads to an unexpected
val n = value.numberDuplicates + 1
logger.error(s"Following report is duplicated ${n} times. This is spurious and should be investigated. The message is kept as unexpected despite Rudder setting.")
(report :: value.matchingReports, n)
}
} else {
(report :: value.matchingReports, value.numberDuplicates)
}
(stack, Some(value.copy(
cardinality = value.cardinality + (if(incrementCardinality(value, report)) 1 else 0)
, matchingReports = report :: value.matchingReports
cardinality = card
, numberDuplicates = nbDup
, matchingReports = r
)))
} else {
(value :: stack, None)
Expand All @@ -1071,9 +1100,9 @@ object ExecutionBatch extends Loggable {
// we look in the free values for one matching the report. If found, we return (remaining freevalues, Some[paired value]
// else (all free values, None).
// never increment the cardinality for free value.
val (newFreeValues, tryPair) = findMachingValue(report, freeValues, (value, report) => false )
val (newFreeValues, tryPair) = findMachingValue(report, freeValues, (value, report) => false, (value, report) => false)

println(s"found a value for '${report.keyValue}'? " + tryPair)
logger.trace(s"found unpaired value for '${report.keyValue}'? " + tryPair)

tryPair match {
case Some(v) =>
Expand All @@ -1090,8 +1119,8 @@ object ExecutionBatch extends Loggable {
// more than 3, it's OK to raise attention of people on that, it may be an other problem than syslog
// being made, or syslog being so made than something must be done.
val dup = value.matchingReports.collect { case r if(r == report) => r}
//predicate OK on dup size of 1 or 2
(dup.size == 1 || dup.size == 2)
//predicate OK if we found at least one identical report
dup.size >= 1
}
}
val unboundedVar = (value: Value, report: ResultReports) => {
Expand All @@ -1100,14 +1129,14 @@ object ExecutionBatch extends Loggable {
value.isVar
}
}
// compose duplicate and unboundVar with OR if they exists.
// This give us the predicate to know if we are allowed to increment cardinality.
val predicate = (value: Value, report: ResultReports) => duplicate(value, report) || unboundedVar(value, report)

val (newPairedValues, pairedAgain) = findMachingValue(report, pairedValues, predicate)
val (newPairedValues, pairedAgain) = findMachingValue(report, pairedValues, duplicate, unboundedVar)

logger.trace(s"Found paired again value for ${report.keyValue}? " + pairedAgain)

pairedAgain match {
case None => //really unexpected after all
recPairReport(tail, newFreeValues, pairedValues, report :: unexpected, mode)
recPairReport(tail, newFreeValues, newPairedValues, report :: unexpected, mode)
case Some(v) => //found a new pair!
recPairReport(tail, newFreeValues, v :: newPairedValues, unexpected, mode)
}
Expand All @@ -1128,13 +1157,21 @@ object ExecutionBatch extends Loggable {
val pattern = replaceCFEngineVars(v)
val specificity = pattern.toString.replaceAll("""\\Q""", "").replaceAll("""\\E""", "").replaceAll("""\.\*""", "").size
// default cardinality for a value is 1
Value(v, u, 1, isVar, pattern, specificity, Nil)
// default duplicate is 0 (and hopefully will remain so)
Value(v, u, 1, 0, isVar, pattern, specificity, Nil)
}.sortWith( _.specificity > _.specificity)

val (pairedValue, unexpected) = recPairReport(filteredReports.toList, values, Nil, Nil, unexpectedInterpretation)
logger.trace("values order: \n - " + values.mkString("\n - "))

// we also need to sort reports to have a chance to not use a specific pattern for not the most specific report
val sortedReports = filteredReports.sortWith( _.keyValue.size > _.keyValue.size )

logger.trace("sorted reports: \n - "+ sortedReports.map( _.keyValue).mkString("\n - "))

println("paires: " + pairedValue)
println("unexpected: " + unexpected)
val (pairedValue, unexpected) = recPairReport(sortedReports.toList, values, Nil, Nil, unexpectedInterpretation)

logger.trace("paires: \n + " + pairedValue.mkString("\n + "))
logger.trace("unexpected: " + unexpected)

// now, we need to transform pairedValue into ComponentStatus reports
val unexpectedReportStatus = unexpected.map(r =>
Expand All @@ -1157,194 +1194,16 @@ object ExecutionBatch extends Loggable {
*/
ComponentStatusReport(
expectedComponent.componentName
, ComponentValueStatusReport.merge(unexpectedReportStatus ::: pairedReportStatus)
)
}


/*
* Recursively look into remaining reports for CFEngine variables,
* values are accumulated and we remove reports while we use them.
*
* The first matching pattern is used as the correct one, so
* they must be sorted from the most specific to the less one
* if you don't want to see things like in #7758 happen
*
* The returned reports are the one that are not matches by
* any cfengine variable value
*
* Visibility is for test
*
*/
private[reports] def extractCFVarsFromReports (
cfengineVars: List[(String, Pattern)]
, allReports : List[Reports]
, noAnswerType: ReportType
, policyMode : PolicyMode
) : (Seq[ComponentValueStatusReport], Seq[Reports]) = {

/*
* So, we don't have any simple, robust way to sort
* reports or patterns to be sure that we are not
* introducing loads of error case.
*
* So, we are going to test all patterns on all regex,
* and find the combination with the most report used.
*
* One optimisation thought: when a pattern match exactly
* one report, we can prune that couple for remaining
* pattern tests.
*
* It is known that that way of testing is quadratic and
* will takes A LOT OF TIME for big input.
* The rationnal to do it none the less is:
* - the quadratic nature is only value with CFEngine params,
* - the time remains ok below 20 or so entries,
* - the pruning helps
* - there is a long terme solution with the unique
* identification of a report for a component (and so no more
* guessing)
*
* Given all that, if an user fall in the quadratic nature, we
* can workaround the problem by splitting his long list of
* problematic patterns into two directives, with a little
* warn log message.
*/


/*
* From a list of pattern to match, find:
* - the pattern with exactly 0 or 1 matching reports and transform
* them into ComponentValueStatusReport,
* - the list of possible reports for patterns matching several reports
* when they are tested (but as pruning go, at the end they can have
* far less choice)
* - the list of reports not matched by exactly 1 patterns (i.e 0 or more than 2)
*/
def recExtract(
cfVars : List[(String, Pattern)]
, values : List[ComponentValueStatusReport]
, multiMatches: List[((String, Pattern), Seq[Reports])]
, reports : List[Reports]
): (List[ComponentValueStatusReport], List[((String, Pattern), Seq[Reports])], List[Reports]) = {
//utility: given a list of (pattern, option[report]), recursively construct the list of component value by
//looking for used reports: if the option(report) exists and is not used, we have a value, else a missing
def recPruneSimple(
choices: List[((String, Pattern), Option[Reports])]
, usedReports: Set[Reports]
, builtValues: List[ComponentValueStatusReport]
): (Set[Reports], List[ComponentValueStatusReport]) = {
choices match {
case Nil => (usedReports, builtValues)
case ((unexpanded, _), optReport) :: tail =>
optReport match {
case Some(report) if(!usedReports.contains(report)) => //youhou, a new value
val v = ComponentValueStatusReport(unexpanded, unexpanded, List(report.toMessageStatusReport(policyMode)))
recPruneSimple(tail, usedReports + report, v :: builtValues)
case _ =>
//here, either we don't have any reports matching the pattern, or the only possible reports was previously used => missing value for pattern
val v = ComponentValueStatusReport(unexpanded, unexpanded, List(MessageStatusReport(noAnswerType, None)))
recPruneSimple(tail, usedReports, v :: builtValues)
}
}
}


//prune the multiMatches with one reports, getting a new multi, a new list of matched reports, a new list of final values
def recPruneMulti(
pruneReports: List[Reports]
, multi: List[((String, Pattern), Seq[Reports])]
, values: List[ComponentValueStatusReport]
, used: List[Reports]
): (List[Reports], List[((String, Pattern), Seq[Reports])], List[ComponentValueStatusReport]) = {

pruneReports match {
case Nil => (used, multi, values)
case seq =>
val m = multi.map { case (x, r) => (x, r.diff(seq)) }
val newBuildableValues = m.collect { case (x, r) if(r.size <= 1) => (x, r.headOption) }
val (newUsedReports, newValues) = recPruneSimple(newBuildableValues, Set(), List())

val newMulti = m.collect { case (x, r) if(r.size > 1) => (x, r) }

recPruneMulti(newUsedReports.toList, newMulti, values ++ newValues, used ++ newUsedReports)
}
}

/*
* ********* actually call the logic *********
*/

cfVars match {
case Nil =>
// Nothing to do, time to return results
(values, multiMatches, reports)
case (unexpanded, pattern) :: remainingPatterns =>
//collect all the reports being matched by that pattern
val matchingReports = reports.collect { case(r) if(pattern.matcher(r.keyValue).matches) => r }

matchingReports match {
case Nil =>
// The pattern is not found in the reports, Create a NoAnswer
val v = ComponentValueStatusReport(unexpanded, unexpanded, List(MessageStatusReport(noAnswerType, None)))
recExtract(remainingPatterns, v :: values, multiMatches, reports)
case report :: Nil =>
// we have exactly one report for the pattern, best matches possible => we are sure to take that one,
// so remove report from both available reports and multiMatches possible case
val v = ComponentValueStatusReport(unexpanded, unexpanded, List(report.toMessageStatusReport(policyMode)))
val (newUsedReports, newMulti, newValues) = if(multiMatches.size > 0) {
recPruneMulti(List(report), multiMatches, List(), List())
} else {
(Nil, Nil, Nil)
}
recExtract(remainingPatterns, v :: newValues ::: values, newMulti, reports.diff( report :: newUsedReports))

case multi => // the pattern matches several reports, keep them for latter processing
recExtract(remainingPatterns, values, ((unexpanded, pattern), multi) :: multiMatches, reports)
, ComponentValueStatusReport.merge(unexpectedReportStatus ::: pairedReportStatus).mapValues { status =>
// here we want to ensure that if a message is unexpected, all other are
if(status.messages.exists( _.reportType == ReportType.Unexpected)) {
val msgs = status.messages.map(m => m.copy(reportType = ReportType.Unexpected))
status.copy(messages = msgs)
} else {
status
}
}
}

/*
* Now, we can still have some choices where several patterns matches the same sets of reports, typically:
* P1 => A, B
* P2 => B, C
* P3 => A, C
* We would need to find P1 => A, P2 => B, P3 => C (and not: P1 => A, P2 => C, P3 => ???)
* But the case is suffiently rare to ignore it, and just take one report at random for each
*
* Return the list of chosen values with the matching used reports.
*/
def recProcessMulti(
choices: List[((String, Pattern), Seq[Reports])]
, usedReports: List[Reports]
, values: List[ComponentValueStatusReport]
): (List[ComponentValueStatusReport], List[Reports]) = {
choices match {
case Nil => (values, usedReports)
case ((unexpanded, _), allReports) :: tail =>
(allReports.diff(usedReports)) match {
case Nil => //too bad, perhaps we could have chosen better
val v = ComponentValueStatusReport(unexpanded, unexpanded, List(MessageStatusReport(noAnswerType, None)))
recProcessMulti(tail, usedReports, v :: values)
case report :: _ =>
val v = ComponentValueStatusReport(unexpanded, unexpanded, List(report.toMessageStatusReport(policyMode)))
recProcessMulti(tail, report :: usedReports, v :: values)
}
}
}

if(cfengineVars.size > 0) {
//actually do the process
val (values, multiChoice, remainingReports) = recExtract(cfengineVars, List(), List(), allReports)
//chose for multi
val (newValues, newUsedReports) = recProcessMulti(multiChoice, Nil, Nil)

//return the final list of build values and remaining reports
(values ::: newValues, remainingReports.diff(newUsedReports))
} else {
(List(), allReports)
}
)
}


Expand Down

0 comments on commit b8e07ae

Please sign in to comment.