Skip to content

Commit

Permalink
Validation tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@ulitol97
ulitol97 committed Apr 25, 2022
1 parent ddc9cac commit bd00e1f
Show file tree
Hide file tree
Showing 5 changed files with 287 additions and 42 deletions.
8 changes: 4 additions & 4 deletions src/main/scala/Main.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,11 @@ object Main extends IOApp {
filePaths =>
for {
// Schema for validations
schema <- TestData.mkSchemaShexIO()
schema <- TestData.mkSchemaShaclIO()
// Trigger for validations
trigger = TestData.mkTriggerShex()
trigger = TestData.mkTriggerShacl
// Validator settings
validatorConfiguration = ValidatorConfiguration(schema, trigger, haltOnErrored = true)
validatorConfiguration = ValidatorConfiguration(schema, trigger, haltOnInvalid = true, haltOnErrored = true)
// RDF extractors: all types ready to be tested
// - List extractor
listExtractor = ListExtractor(
Expand All @@ -78,7 +78,7 @@ object Main extends IOApp {
itemTimeout = None)

// Validator instance
validator = Validator(validatorConfiguration, fileExtractor)
validator = new Validator(validatorConfiguration, listExtractor)
// Open validation stream
app <- validator.validate // Init
// .delayBy(10.minute)
Expand Down
19 changes: 14 additions & 5 deletions src/main/scala/validation/Validator.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,8 @@ import org.apache.kafka.common.KafkaException
* such cases
*
*/
case class Validator[A](configuration: ValidatorConfiguration,
private val extractor: StreamExtractor[A]) {
// Shorthands for useful information
class Validator[A](configuration: ValidatorConfiguration,
private val extractor: StreamExtractor[A]) {

/**
* Schema against which this validator's data is validated
Expand All @@ -65,7 +64,7 @@ case class Validator[A](configuration: ValidatorConfiguration,
*/
val validationTrigger: ValidationTrigger = configuration.trigger

// Expose useful extractor information
// Shorthands for useful information

/**
* Source from which this validator's data arrives
Expand All @@ -77,11 +76,21 @@ case class Validator[A](configuration: ValidatorConfiguration,
*/
val dataFormat: DataFormat = extractor.format

// Expose useful extractor information

/**
* Data inference applied by this this validator
*/
val dataInference: InferenceEngine = extractor.inference

/**
* Alternative constructor, automatically build the validator configuration
* given a schema and a trigger
*/
def this(extractor: StreamExtractor[A], schema: Schema, trigger: ValidationTrigger) = {
this(ValidatorConfiguration(schema, trigger), extractor)
}

/**
* Main method of the validator, produces a Stream of validation results
* from input data, fetching and processing items as specified in
Expand Down Expand Up @@ -197,7 +206,7 @@ case class Validator[A](configuration: ValidatorConfiguration,
/**
* Helper utilities for all Validators
*/
object Validator {
private[comet] object Validator {

/**
* Auxiliary messages emitted by the validator
Expand Down
40 changes: 22 additions & 18 deletions src/test/scala/utils/Samples.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ object Samples {
* @param format Format of the output RDF string
* @param min Minimum temperature of the item
* @param max Maximum temperature of the items
* @param valid Whether the produced item should comply with the schemas
* generated for testing or not
* @return A String containing RDF data with the format and contents
* specified by the user
*
Expand All @@ -97,9 +99,11 @@ object Samples {
*/
def mkRdfItem(format: DataFormat = TURTLE,
min: Double = minValidTemperature,
max: Double = maxValidTemperature): String = {
max: Double = maxValidTemperature,
valid: Boolean = true): String = {
val dateFormatted = dateFormatter.format(new Date())
val temperature = Random.between(min, max)
// Force a too low temperature to make items invalid if necessary
val temperature = if (valid) Random.between(min, max) else minValidTemperature - 1
// Format with US locale to have dots, not commas
val temperatureFormatted = String.format(Locale.US, "%.2f", temperature)

Expand All @@ -112,8 +116,8 @@ object Samples {
| xmlns:ex="http://example.org/"
| xmlns:xsd="http://www.w3.org/2001/XMLSchema#">
| <ex:sensorReading rdf:about="http://example.org/reading">
| <ex:readingDate rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime"
| >$dateFormatted</ex:readingDate>
| <ex:readingDateTime rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime"
| >$dateFormatted</ex:readingDateTime>
| <ex:readingTemperature rdf:datatype="http://www.w3.org/2001/XMLSchema#decimal"
| >$temperatureFormatted</ex:readingTemperature>
| <ex:status>OK</ex:status>
Expand All @@ -126,12 +130,12 @@ object Samples {
|{
| "@id" : "ex:reading",
| "@type" : "ex:sensorReading",
| "readingDate" : "$dateFormatted",
| "readingDateTime" : "$dateFormatted",
| "readingTemperature" : "$temperatureFormatted",
| "status" : "OK",
| "@context" : {
| "readingDate" : {
| "@id" : "http://example.org/readingDate",
| "readingDateTime" : {
| "@id" : "http://example.org/readingDateTime",
| "@type" : "http://www.w3.org/2001/XMLSchema#dateTime"
| },
| "readingTemperature" : {
Expand All @@ -155,7 +159,7 @@ object Samples {
|@prefix ex: <http://example.org/> .
|
|ex:reading a ex:sensorReading ;
| ex:readingDatetime "$dateFormatted"^^xsd:dateTime ;
| ex:readingDateTime "$dateFormatted"^^xsd:dateTime ;
| ex:readingTemperature "$temperatureFormatted"^^xsd:decimal ;
| ex:status "OK" .
|""".stripMargin.strip
Expand Down Expand Up @@ -188,7 +192,7 @@ object Samples {
|
|# Filters of a valid sensor reading
|ex:ValidReading {
| ex:readingDatetime xsd:dateTime ; # Has a VALID timestamp
| ex:readingDateTime xsd:dateTime ; # Has a VALID timestamp
| ex:readingTemperature xsd:decimal MININCLUSIVE $minValidTemperature MAXINCLUSIVE $maxValidTemperature + ; # 1+ readings in range 18-20
| ex:status [ "OK" "RUNNING" ] # Status must be one of
|}
Expand All @@ -210,7 +214,7 @@ object Samples {
| "expressions" : [
| {
| "type" : "TripleConstraint",
| "predicate" : "http://example.org/readingDate",
| "predicate" : "http://example.org/readingDateTime",
| "valueExpr" : {
| "type" : "NodeConstraint",
| "datatype" : "http://www.w3.org/2001/XMLSchema#dateTime"
Expand Down Expand Up @@ -290,7 +294,7 @@ object Samples {
| </sh:property>
| <sh:property>
| <sh:PropertyShape>
| <sh:path rdf:resource="http://example.org/readingDate"/>
| <sh:path rdf:resource="http://example.org/readingDateTime"/>
| <sh:datatype rdf:resource="http://www.w3.org/2001/XMLSchema#dateTime"/>
| </sh:PropertyShape>
| </sh:property>
Expand All @@ -313,7 +317,7 @@ object Samples {
case JSONLD =>
f"""
|{
| "@graph" : [ {"
| "@graph" : [ {
| "@id" : "_:b0",
| "@type" : "sh:PropertyShape",
| "datatype" : "xsd:decimal",
Expand All @@ -332,7 +336,7 @@ object Samples {
| "@id" : "_:b2",
| "@type" : "sh:PropertyShape",
| "datatype" : "xsd:dateTime",
| "path" : "ex:readingDate"
| "path" : "ex:readingDateTime"
| }, {
| "@id" : "ex:ValidReading",
| "@type" : "sh:NodeShape",
Expand All @@ -348,7 +352,7 @@ object Samples {
| "@id" : "http://www.w3.org/ns/shacl#maxInclusive",
| "@type" : "http://www.w3.org/2001/XMLSchema#decimal"
| },
| "miSHaclEXnInclusive" : {
| "minInclusive" : {
| "@id" : "http://www.w3.org/ns/shacl#minInclusive",
| "@type" : "http://www.w3.org/2001/XMLSchema#decimal"
| },
Expand Down Expand Up @@ -395,14 +399,14 @@ object Samples {
|ex:ValidReading a sh:NodeShape ;
| sh:targetClass ex:sensorReading ;
| sh:property [
| sh:path ex:readingDate ;
| sh:path ex:readingDateTime ;
| sh:datatype xsd:dateTime ;
| ] ;
| sh:property [
| sh:path ex:readingTemperature ;
| sh:datatype xsd:decimal ;
| sh:minInclusive 18.00;
| sh:maxInclusive 20.00 ;
| sh:minInclusive $minValidTemperature;
| sh:maxInclusive $maxValidTemperature ;
| sh:minCount 1 ; # 1+ readings
| ] ;
| sh:property [
Expand All @@ -413,7 +417,7 @@ object Samples {
| ] .
|""".stripMargin.strip
}
Schemas.fromString(schemaText, format.name, Schemas.shEx.name)
Schemas.fromString(schemaText, format.name, Schemas.shaclex.name)
}
}

Expand Down
15 changes: 0 additions & 15 deletions src/test/scala/validation/ouputs/DummyTest.scala
View file

This file was deleted.

Loading

0 comments on commit bd00e1f

Please sign in to comment.