Permalink
Browse files

Merge pull request #250 from lshift/escalation-rules

Escalation rules
  • Loading branch information...
2 parents d6eee7b + f6ca4a6 commit 523947be5c2c446176e500a174c2c2e9675e6b24 @0x6e6562 0x6e6562 committed Aug 9, 2012
Showing with 311 additions and 107 deletions.
  1. +4 −4 agent/src/main/resources/net/lshift/diffa/agent/rest/castorMapping.xml
  2. +1 −12 agent/src/main/webapp/js/templates/settings/escalation.jst
  3. +1 −1 agent/src/main/webapp/js/templates/settings/paireditor.jst
  4. +2 −2 agent/src/test/scala/net/lshift/diffa/agent/client/ConfigurationRestClient.scala
  5. +1 −1 agent/src/test/scala/net/lshift/diffa/agent/itest/support/TestEnvironment.scala
  6. +6 −6 agent/src/test/scala/net/lshift/diffa/agent/rest/DiffaConfigReaderWriterTest.scala
  7. +8 −0 kernel/pom.xml
  8. +0 −3 kernel/src/main/scala/net/lshift/diffa/kernel/config/DomainConfigStore.scala
  9. +1 −2 kernel/src/main/scala/net/lshift/diffa/kernel/config/JooqConfigStoreCompanion.scala
  10. +1 −1 kernel/src/main/scala/net/lshift/diffa/kernel/config/JooqDomainConfigStore.scala
  11. +3 −1 kernel/src/main/scala/net/lshift/diffa/kernel/differencing/DifferencesManager.scala
  12. +46 −10 kernel/src/main/scala/net/lshift/diffa/kernel/escalation/EscalationManager.scala
  13. +8 −21 kernel/src/main/scala/net/lshift/diffa/kernel/frontend/DiffaConfig.scala
  14. +1 −1 kernel/src/main/scala/net/lshift/diffa/kernel/util/DocExamplesFactory.scala
  15. +3 −4 kernel/src/test/scala/net/lshift/diffa/kernel/config/JooqDomainConfigStoreTest.scala
  16. +1 −1 kernel/src/test/scala/net/lshift/diffa/kernel/differencing/JooqDomainDifferenceStoreTest.scala
  17. +35 −32 kernel/src/test/scala/net/lshift/diffa/kernel/escalation/EscalationManagerTest.scala
  18. +2 −2 kernel/src/test/scala/net/lshift/diffa/kernel/frontend/ConfigurationTest.scala
  19. +79 −2 kernel/src/test/scala/net/lshift/diffa/kernel/frontend/EscalationDefValidationTest.scala
  20. +10 −0 pom.xml
  21. +1 −0 release-notes/next.md
  22. +2 −1 schema/src/main/scala/net/lshift/diffa/schema/migrations/HibernateConfigStorePreparationStep.scala
  23. +95 −0 schema/src/main/scala/net/lshift/diffa/schema/migrations/steps/Step0045.scala
@@ -165,11 +165,11 @@
<field name="actionType" type="string">
<bind-xml name="type" node="attribute" />
</field>
- <field name="event" type="string">
- <bind-xml name="event" node="attribute" />
+ <field name="rule" type="string">
+ <bind-xml name="rule" node="attribute" />
</field>
- <field name="origin" type="string">
- <bind-xml name="origin" node="attribute" />
+ <field name="delay" type="int">
+ <bind-xml name="delay" node="attribute" />
</field>
</class>
@@ -11,18 +11,7 @@
<input data-bind-value="escalation.action" type="text">
</td>
<td>
- <select data-bind-value="escalation.event">
- <option value="upstream-missing">Upstream Missing</option>
- <option value="downstream-missing">Downstream Missing</option>
- <option value="mismatch">Mismatch</option>
- <option value="scan-failed">Scan Failed</option>
- <option value="scan-completed">Scan Completed</option>
- </select>
- </td>
- <td>
- <select data-bind-value="escalation.origin">
- <option value="scan">Scan</option>
- </select>
+ <input data-bind-value="escalation.rule" type="text">
</td>
<td>
<input data-bind-value="escalation.delay" type="number">
@@ -77,7 +77,7 @@
<div class="escalations">
<table class="editable-table" data-template="escalation">
- <thead><tr><td>Name</td><td>Type</td><td>Action</td><td>Event</td><td>Origin</td><td>Delay</td><td>Actions</td></tr></thead>
+ <thead><tr><td>Name</td><td>Type</td><td>Action</td><td>Rule</td><td>Delay</td><td>Actions</td></tr></thead>
</table>
<a class="add-link" href="">Add New</a>
</div>
@@ -46,8 +46,8 @@ class ConfigurationRestClient(serverRootUrl:String, domain:String, params: RestC
delete("/pairs/"+pairKey+"/repair-actions/"+name)
}
- def declareEscalation(name: String, pairKey: String, action: String, actionType: String, event: String, origin: String) = {
- val escalation = new EscalationDef(name, action, actionType, event, origin)
+ def declareEscalation(name: String, pairKey: String, action: String, actionType: String, rule: String, delay: Int) = {
+ val escalation = new EscalationDef(name, action, actionType, rule, delay)
create("/pairs/"+pairKey+"/escalations", escalation)
escalation
}
@@ -155,7 +155,7 @@ class TestEnvironment(val pairKey: String,
createPair
configurationClient.declareRepairAction(entityScopedActionName, entityScopedActionUrl, RepairAction.ENTITY_SCOPE, pairKey)
- configurationClient.declareEscalation(escalationName, pairKey, entityScopedActionName, EscalationActionType.REPAIR, EscalationEvent.DOWNSTREAM_MISSING, EscalationOrigin.SCAN)
+ configurationClient.declareEscalation(escalationName, pairKey, entityScopedActionName, EscalationActionType.REPAIR, "downstreamMissing", 0)
def createPair = configurationClient.declarePair(PairDef(key = pairKey,
@@ -71,9 +71,9 @@ class DiffaConfigReaderWriterTest {
PairReportDef(name="Bulk Fix Differences", reportType="differences", target="http://example.com/bulk_diff_handler")
),
escalations = Set(
- EscalationDef(name="Delete From Upstream", action="Delete Result", actionType="repair", event="upstream-missing", origin="scan"),
- EscalationDef(name="Resend Missing Downstream", action="Resend Sauce", actionType="repair", event="downstream-missing", origin="scan"),
- EscalationDef(name="Resend On Mismatch", action="Resend Sauce", actionType="repair", event="mismatch", origin="scan")
+ EscalationDef(name="Delete From Upstream", action="Delete Result", actionType="repair", rule="upstreamVsn is null", delay=20),
+ EscalationDef(name="Resend Missing Downstream", action="Resend Sauce", actionType="repair", rule="downstreamVsn is null", delay=30),
+ EscalationDef(name="Resend On Mismatch", action="Resend Sauce", actionType="repair", rule="upstreamVsn is not null and downstreamVsn is not null and upstreamVsn != downstreamVsn", delay=0)
)
),
PairDef("ac", "same", 5, "upstream1", "downstream1", allowManualScans = false))
@@ -119,9 +119,9 @@ class DiffaConfigReaderWriterTest {
scan-schedule="0 0 0 * 0 0" scan-schedule-enabled="false">
<repair-action url="http://example.com/resend/{id}" name="Resend Sauce" scope="entity" />
<repair-action url="http://example.com/delete/{id}" name="Delete Result" scope="entity" />
- <escalation name="Delete From Upstream" action="Delete Result" type="repair" event="upstream-missing" origin="scan" />
- <escalation name="Resend Missing Downstream" action="Resend Sauce" type="repair" event="downstream-missing" origin="scan" />
- <escalation name="Resend On Mismatch" action="Resend Sauce" type="repair" event="mismatch" origin="scan" />
+ <escalation name="Delete From Upstream" action="Delete Result" type="repair" rule="upstreamVsn is null" delay="20" />
+ <escalation name="Resend Missing Downstream" action="Resend Sauce" type="repair" rule="downstreamVsn is null" delay="30" />
+ <escalation name="Resend On Mismatch" action="Resend Sauce" type="repair" rule="upstreamVsn is not null and downstreamVsn is not null and upstreamVsn != downstreamVsn" delay="0" />
<report name="Bulk Fix Differences" report-type="differences" target="http://example.com/bulk_diff_handler" />
<view name="little-view" scan-schedule="0 0 0 * * 0" />
</pair>
View
@@ -163,6 +163,14 @@
<artifactId>icu4j</artifactId>
</dependency>
<dependency>
+ <groupId>com.gentlyweb</groupId>
+ <artifactId>gentlyweb-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.josql</groupId>
+ <artifactId>josql</artifactId>
+ </dependency>
+ <dependency>
<groupId>postgresql</groupId>
<artifactId>postgresql</artifactId>
<scope>test</scope>
@@ -328,9 +328,6 @@ object RepairAction {
* Enumeration of valid types that an escalating difference should trigger.
*/
object EscalationEvent {
- val UPSTREAM_MISSING = "upstream-missing"
- val DOWNSTREAM_MISSING = "downstream-missing"
- val MISMATCH = "mismatch"
val SCAN_FAILED = "scan-failed"
val SCAN_COMPLETED = "scan-completed"
}
@@ -369,8 +369,7 @@ object JooqConfigStoreCompanion {
name = record.getValue(ESCALATIONS.NAME),
action = record.getValue(ESCALATIONS.ACTION),
actionType = record.getValue(ESCALATIONS.ACTION_TYPE),
- event = record.getValue(ESCALATIONS.EVENT),
- origin = record.getValue(ESCALATIONS.ORIGIN),
+ rule = record.getValue(ESCALATIONS.RULE),
delay = record.getValue(ESCALATIONS.DELAY))
}
@@ -353,7 +353,7 @@ class JooqDomainConfigStore(jooq:JooqDatabaseFacade,
insertOrUpdate(t, ESCALATIONS,
Map(ESCALATIONS.DOMAIN -> domain, ESCALATIONS.PAIR_KEY -> pair.key, ESCALATIONS.NAME -> e.name),
Map(ESCALATIONS.ACTION -> e.action, ESCALATIONS.ACTION_TYPE -> e.actionType,
- ESCALATIONS.EVENT -> e.event, ESCALATIONS.ORIGIN -> e.origin, ESCALATIONS.DELAY -> e.delay))
+ ESCALATIONS.RULE -> e.rule, ESCALATIONS.DELAY -> e.delay))
})
upgradeConfigVersion(t, domain)
@@ -150,7 +150,9 @@ case class DifferenceEvent(
def this() = this(seqId = null)
def sequenceId = Integer.parseInt(seqId)
-
+
+ /** Proxy to the object id. Added to allow escalation rules like "id LIKE 'id123*'" to be applied. **/
+ def id = objId.id
}
abstract class DifferenceEventStatus
@@ -24,7 +24,6 @@ import net.lshift.diffa.kernel.client.{ActionableRequest, ActionsClient}
import org.slf4j.LoggerFactory
import net.lshift.diffa.kernel.lifecycle.{NotificationCentre, AgentLifecycleAware}
import net.lshift.diffa.kernel.differencing._
-import net.lshift.diffa.kernel.config.{DiffaPairRef, DomainConfigStore}
import net.lshift.diffa.kernel.reporting.ReportManager
import net.lshift.diffa.kernel.util.AlertCodes._
import java.io.Closeable
@@ -34,6 +33,9 @@ import scala.collection.JavaConversions._
import java.util.{Timer, TimerTask}
import net.lshift.diffa.kernel.frontend.EscalationDef
import net.lshift.diffa.kernel.config.system.SystemConfigStore
+import org.josql.filters.DefaultObjectFilter
+import net.lshift.diffa.kernel.config.{ConfigValidationException, DiffaPairRef, DomainConfigStore}
+import org.josql.QueryParseException
/**
* This deals with escalating mismatches based on configurable escalation policies.
@@ -122,15 +124,28 @@ class EscalationManager(val config:DomainConfigStore,
}
def escalatePairEvent(pairRef: DiffaPairRef, eventType:String) = {
- findEscalations(pairRef, eventType, REPORT).foreach(e => {
+ findEscalationsForPair(pairRef, eventType).foreach(e => {
log.debug("Escalating pair event as report %s".format(e.name))
reportManager.executeReport(pairRef, e.action)
})
}
- def findEscalations(pair: DiffaPairRef, eventType:String, actionTypes:String*) =
+ def findEscalations(pair: DiffaPairRef, diff:DifferenceEvent) = {
config.getPairDef(pair).escalations.
- filter(e => e.event == eventType && (actionTypes.length == 0 || actionTypes.contains(e.actionType)))
+ filter(e => {
+ if (e.rule == null) {
+ true
+ } else {
+ val filter = new DefaultObjectFilter(e.rule, classOf[DifferenceEventRuleView])
+ filter.accept(DifferenceEventRuleView(diff))
+ }
+ })
+ }
+
+ def findEscalationsForPair(pair: DiffaPairRef, eventType:String) = {
+ config.getPairDef(pair).escalations.
+ filter(e => e.rule == eventType && e.actionType == REPORT)
+ }
def findEscalation(pair: DiffaPairRef, name:String) =
config.getPairDef(pair).escalations.find(_.name == name)
@@ -147,7 +162,7 @@ class EscalationManager(val config:DomainConfigStore,
def progressDiff(diff:DifferenceEvent) {
val diffType = DifferenceUtils.differenceType(diff.upstreamVsn, diff.downstreamVsn)
- val escalations = orderEscalations(findEscalations(diff.objId.pair, mapDifferenceType(diffType)).toSeq)
+ val escalations = orderEscalations(findEscalations(diff.objId.pair, diff).toSeq)
val selectedEscalation = diff.nextEscalation match {
case null => escalations.headOption
@@ -169,12 +184,33 @@ class EscalationManager(val config:DomainConfigStore,
diffs.scheduleEscalation(diff, esc.name, escalateTime)
}
}
+}
- def mapDifferenceType(t:DifferenceType) = t match {
- case UpstreamMissing => UPSTREAM_MISSING
- case DownstreamMissing => DOWNSTREAM_MISSING
- case ConflictingVersions => MISMATCH
+object EscalationManager {
+ def validateRule(rule:String, path:String) {
+ if (rule == null) return
+
+ try {
+ new DefaultObjectFilter(rule, classOf[DifferenceEventRuleView])
+ } catch {
+ case e:QueryParseException => throw new ConfigValidationException(path,
+ "invalid rule '%s': %s".format(rule, e.getMessage))
+ }
}
}
-case class Escalate(e:DifferenceEvent)
+case class Escalate(e:DifferenceEvent)
+
+case class DifferenceEventRuleView(event:DifferenceEvent) {
+ def getId = event.objId.id
+ def getUpstream = event.upstreamVsn
+ def getDownstream = event.downstreamVsn
+ def getDetectedAt = event.detectedAt
+ def getLastSeen = event.lastSeen
+
+ def getHasUpstream = event.upstreamVsn != null
+ def getUpstreamMissing = !getHasUpstream
+ def getHasDownstream = event.downstreamVsn != null
+ def getDownstreamMissing = !getHasDownstream
+ def isMismatch = getHasUpstream && getHasDownstream && getUpstream != getDownstream
+}
@@ -23,6 +23,7 @@ import java.util.HashMap
import scala.collection.JavaConversions._
import net.lshift.diffa.kernel.util.{DownstreamEndpoint, UpstreamEndpoint, EndpointSide}
import net.lshift.diffa.participant.scanning.Collation
+import net.lshift.diffa.kernel.escalation.EscalationManager
/**
* Describes a complete Diffa configuration in the context of a domain - this means that all of the objects
@@ -383,12 +384,10 @@ case class EscalationDef (
@BeanProperty var name: String = null,
@BeanProperty var action: String = null,
@BeanProperty var actionType: String = null,
- @BeanProperty var event: String = null,
- @BeanProperty var origin: String = null,
+ @BeanProperty var rule: String = null,
@BeanProperty var delay: Int = 0
) {
import EscalationEvent._
- import EscalationOrigin._
import EscalationActionType._
def this() = this(name = null)
@@ -397,34 +396,22 @@ case class EscalationDef (
val escalationPath = ValidationUtil.buildPath(path, "escalation", Map("name" -> name))
action = ValidationUtil.maybeNullify(action)
+ rule = ValidationUtil.maybeNullify(rule)
ValidationUtil.ensureLengthLimit(escalationPath, "name", name, DefaultLimits.KEY_LENGTH_LIMIT)
ValidationUtil.ensureLengthLimit(escalationPath, "action", action, DefaultLimits.KEY_LENGTH_LIMIT)
+ ValidationUtil.ensureLengthLimit(escalationPath, "rule", rule, DefaultLimits.URL_LENGTH_LIMIT)
// Ensure that the action type is supported, and validate the parameters that depend on it
actionType match {
case REPAIR | IGNORE =>
- // Ensure that the origin is supported
- origin match {
- case SCAN =>
- case _ => throw new ConfigValidationException(escalationPath, "Invalid escalation origin: " + origin)
- }
- event match {
- case UPSTREAM_MISSING | DOWNSTREAM_MISSING | MISMATCH => event
- case _ =>
- throw new ConfigValidationException(escalationPath,
- "Invalid escalation event source type %s for action type %s".format(event, actionType))
- }
+ EscalationManager.validateRule(rule, escalationPath)
case REPORT =>
- // We don't support origins for reports
- if (origin != null)
- throw new ConfigValidationException(escalationPath, "Origin not supported on report escalations.")
-
- event match {
- case SCAN_FAILED | SCAN_COMPLETED => event
+ rule match {
+ case SCAN_FAILED | SCAN_COMPLETED => rule
case _ =>
throw new ConfigValidationException(escalationPath,
- "Invalid escalation event source type %s for action type %s".format(event, actionType))
+ "Invalid escalation event source type %s for action type %s".format(rule, actionType))
}
case _ =>
throw new ConfigValidationException(escalationPath, "Invalid escalation action type: " + actionType)
@@ -48,7 +48,7 @@ class DocExamplesFactory {
val escalation = EscalationDef(name = "some-escalation",
action = "resend", actionType = "repair",
- event = "downstream-missing", origin = "scan")
+ rule = "downstreamVsn is null", delay = 10)
val actionable = Actionable(name = "resend",
scope = "entity",
@@ -79,9 +79,8 @@ class JooqDomainConfigStoreTest {
scope=RepairAction.ENTITY_SCOPE,
url="resend")
val escalation = EscalationDef(name="esc", action = "test_action",
- event = EscalationEvent.UPSTREAM_MISSING,
+ rule = "upstreamMissing",
actionType = EscalationActionType.REPAIR,
- origin = EscalationOrigin.SCAN,
delay = 30)
val report = PairReportDef(name = "REPORT_NAME",
reportType = "differences", target = "http://example.com/diff_listener")
@@ -166,8 +165,8 @@ class JooqDomainConfigStoreTest {
val repairAction2 = repairAction.copy(url = "resend2")
val repairAction3 = repairAction.copy(url = "resend3")
- val escalation2 = escalation.copy(event = EscalationEvent.DOWNSTREAM_MISSING)
- val escalation3 = escalation.copy(event = EscalationEvent.MISMATCH)
+ val escalation2 = escalation.copy(rule = "downstreamMissing")
+ val escalation3 = escalation.copy(rule = "upstreamMissing")
val report2 = report.copy(target = "http://example.com/diff_listener2")
val report3 = report.copy(target = "http://example.com/diff_listener3")
@@ -61,7 +61,7 @@ class JooqDomainDifferenceStoreTest {
val pairTemplate = PairDef(upstreamName = us.name, downstreamName = ds.name)
val pair1 = pairTemplate.copy(key = "pair1",
repairActions = Set(RepairActionDef(name = "r1", url = "http://localhost/repair", scope = "entity")),
- escalations = Set(EscalationDef(name = "esc1", action = "r1", actionType = "repair", event = "upstream-missing", origin = "scan")))
+ escalations = Set(EscalationDef(name = "esc1", action = "r1", actionType = "repair", rule = "upstreamMissing")))
val pair2 = pairTemplate.copy(key = "pair2")
domainConfigStore.listPairs(domainName).foreach(p => domainConfigStore.deletePair(domainName, p.key))
Oops, something went wrong.

0 comments on commit 523947b

Please sign in to comment.