Skip to content

Commit

Permalink
local writing works
Browse files Browse the repository at this point in the history
  • Loading branch information
@arnegevaert
arnegevaert committed Oct 9, 2017
1 parent cc6b02c commit 12a2eed
Show file tree
Hide file tree
Showing 5 changed files with 74 additions and 14 deletions.
5 changes: 4 additions & 1 deletion cron.php
View file
Expand Up @@ -46,11 +46,14 @@ function acquire_data() {
$interval = 60*60*3; // 3 hour interval results in files of a few 100 KB
// TODO add out and resources directories to .env. Right now Dutch dataset class has dependency on this.
$fs = new Filesystem\FileWriter(__DIR__ . "/out", __DIR__ . "/resources", $interval, $processor);
echo "Fetching graph for " . $processor->getName() . "\n";
$graph = $processor->getDynamicGraph();
$now = time();
echo "Writing data for " . $processor->getName() . "\n";
$fs->writeToFile($now, $graph);
// Temporarily disabling range gates, semantically incorrect
//$fs->updateStatisticalSummary($now, $graph);
echo "Updating statistical summary for " . $processor->getName() . "\n";
$fs->updateStatisticalSummary($now, $graph);
}
}
}
45 changes: 42 additions & 3 deletions src/Filesystem/FileReader.php
View file
Expand Up @@ -60,7 +60,9 @@ public function getStatisticalSummary($interval) {
$sortedStatistics = $this->sortStatisticTriples($this->getAllStatisticsForInterval($interval));

// Take median of medians, means of the rest
foreach($sortedStatistics as $parking => $stats) {
foreach($sortedStatistics as $summ => $stats) {
$parking = $stats['rdf:subject'];
// TODO FIRST FIX THIS
$medians = $stats[$buildingBlocks['median']];
$statCalc = new Helpers\Statistics($medians);
$median = $statCalc->median();
Expand Down Expand Up @@ -92,6 +94,11 @@ public function getStaticData() {

private function sortStatisticTriples($statistics) {
$buildingBlocks = $this->statisticBuildingBlocks;
$buildingBlocks['rdf:type'] = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type';
$buildingBlocks['rdf:predicate'] = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#predicate';
$buildingBlocks['rdf:subject'] = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#subject';
$buildingBlocks['time:hasBeginning'] = 'https://www.w3.org/TR/owl-time/hasBeginning';
$buildingBlocks['time:hasEnd'] = 'https://www.w3.org/TR/owl-time/hasEnd';

$sortedStatistics = array();

Expand All @@ -105,8 +112,40 @@ private function sortStatisticTriples($statistics) {
$buildingBlocks['thirdQuartile'] => array()
);
}
$value = doubleval(Util::getLiteralValue($triple["object"]));
array_push($sortedStatistics[$triple["subject"]][$triple['predicate']], $value);
if ($triple["predicate"] === $buildingBlocks['rdf:type']) {
$sortedStatistics[$triple['subject']]['rdf:type'] = $triple['object'];
} else if ($triple["predicate"] === $buildingBlocks['rdf:predicate']) {
$sortedStatistics[$triple['subject']]['rdf:predicate'] = $triple['object'];
} else if ($triple["predicate"] === $buildingBlocks['rdf:subject']) {
$sortedStatistics[$triple['subject']]['rdf:subject'] = $triple['object'];
} else if ($triple["predicate"] === $buildingBlocks['time:hasBeginning']) {
$curBeginning = $sortedStatistics['time:hasBeginning'];
if ($curBeginning === null) {
$sortedStatistics['time:hasBeginning'] = $triple['object'];
} else {
$curBeginningDateTime = new \DateTime(Util::getLiteralValue($curBeginning));
$beginningDateTime = new \DateTime(Util::getLiteralValue($triple['object']));
if ($beginningDateTime < $curBeginningDateTime) {
$lit = Util::createLiteral($triple['object'], 'http://www.w3.org/2001/XMLSchema#dateTime');
$sortedStatistics['time:hasBeginning'] = $lit;
}
}
} else if ($triple["predicate"] === $buildingBlocks['time:hasEnd']) {
$curEnd = $sortedStatistics['time:hasEnd'];
if ($curEnd === null) {
$sortedStatistics['time:hasEnd'] = $triple['object'];
} else {
$curEndDateTime = new \DateTime(Util::getLiteralValue($curEnd));
$endDateTime = new \DateTime(Util::getLiteralValue($triple['object']));
if ($endDateTime > $curEndDateTime) {
$lit = Util::createLiteral($triple['object'], 'http://www.w3.org/2001/XMLSchema#dateTime');
$sortedStatistics['time:hasEnd'] = $lit;
}
}
} else {
$value = doubleval(Util::getLiteralValue($triple["object"]));
array_push($sortedStatistics[$triple["subject"]][$triple['predicate']], $value);
}
}
return $sortedStatistics;
}
Expand Down
29 changes: 23 additions & 6 deletions src/Filesystem/FileWriter.php
View file
Expand Up @@ -49,6 +49,7 @@ public function updateStatisticalSummary($timestamp, $graph) {

// Get all relevant triples from files
$measurements = array();
$oldest = null; $latest = null;
foreach ($files as $file) {
$relevantSubgraphs = array();
$parser = new TriGParser();
Expand All @@ -57,6 +58,9 @@ public function updateStatisticalSummary($timestamp, $graph) {
foreach($triples as $triple) {
if ($triple["predicate"] === 'http://www.w3.org/ns/prov#generatedAtTime') {
if (substr(Util::getLiteralValue($triple["object"]), 0, 10) === $filename) {
$datetime = new \DateTime(Util::getLiteralValue($triple["object"]));
if ($oldest === null || $datetime < $oldest) $oldest = $datetime;
if ($latest === null || $datetime > $latest) $latest = $datetime;
array_push($relevantSubgraphs, $triple["subject"]);
}
}
Expand All @@ -80,23 +84,36 @@ public function updateStatisticalSummary($timestamp, $graph) {

// Calculate statistics for each parking
$output = array();
$index = 0;
foreach ($sortedMeasurements as $p => $ms) {
$summaryURL = "#summary" . $index;
$stat = new Helpers\Statistics($ms);
$median = Util::createLiteral($stat->median());
$mean = Util::createLiteral($stat->mean());
$var = Util::createLiteral($stat->variance());
$firstq = Util::createLiteral($stat->percentile(0.25));
$thirdq = Util::createLiteral($stat->percentile(0.75));
array_push($output, ["subject" => $p, "predicate" => "stat:median", "object" => $median]);
array_push($output, ["subject" => $p, "predicate" => "stat:mean", "object" => $mean]);
array_push($output, ["subject" => $p, "predicate" => "stat:variance", "object" => $var]);
array_push($output, ["subject" => $p, "predicate" => "stat:firstQuartile", "object" => $firstq]);
array_push($output, ["subject" => $p, "predicate" => "stat:thirdQuartile", "object" => $thirdq]);
$beginning = Util::createLiteral($oldest->format('Y-m-d\TH:i:s'), 'http://www.w3.org/2001/XMLSchema#dateTime');
$end = Util::createLiteral($latest->format('Y-m-d\TH:i:s'), 'http://www.w3.org/2001/XMLSchema#dateTime');
array_push($output, ["subject" => $summaryURL, "predicate" => "rdf:type", "object" => "ts:Summary"]);
array_push($output, ["subject" => $summaryURL, "predicate" => "rdf:predicate", "object" => "datex:numberOfVacantSpaces"]);
array_push($output, ["subject" => $summaryURL, "predicate" => "rdf:subject", "object" => $p]);
array_push($output, ["subject" => $summaryURL, "predicate" => "ts:median", "object" => $median]);
array_push($output, ["subject" => $summaryURL, "predicate" => "ts:mean", "object" => $mean]);
array_push($output, ["subject" => $summaryURL, "predicate" => "ts:variance", "object" => $var]);
array_push($output, ["subject" => $summaryURL, "predicate" => "ts:firstQuartile", "object" => $firstq]);
array_push($output, ["subject" => $summaryURL, "predicate" => "ts:thirdQuartile", "object" => $thirdq]);
array_push($output, ["subject" => $summaryURL, "predicate" => "time:hasBeginning", "object" => $beginning]);
array_push($output, ["subject" => $summaryURL, "predicate" => "time:hasEnd", "object" => $end]);
$index++;
}

// Write statistics to file
$writer = new TriGWriter();
$writer->addPrefix('stat', 'http://datapiloten.be/vocab/timeseries#');
$writer->addPrefix('ts', 'http://datapiloten.be/vocab/timeseries#');
$writer->addPrefix('rdf', 'http://www.w3.org/1999/02/22-rdf-syntax-ns#');
$writer->addPrefix('time', 'https://www.w3.org/TR/owl-time/');
$writer->addPrefix('datex', 'http://vocab.datex.org/terms#');
$writer->addTriples($output);
$this->stat_fs->put($filename, $writer->end());
}
Expand Down
5 changes: 4 additions & 1 deletion src/Helpers/Statistics.php
View file
Expand Up @@ -40,6 +40,9 @@ public function variance() {
foreach($this->data as $d) {
$numerator += ($d-$mean)**2;
}
return $numerator/(count($this->data)-1);
if (count($this->data) > 1) {
return $numerator/(count($this->data)-1);
}
return $numerator/count($this->data);
}
}
4 changes: 1 addition & 3 deletions src/Router.php
View file
Expand Up @@ -59,8 +59,6 @@ function() use ($found, $dataset, $out_dirname, $res_dirname, $second_interval,
}
);

// Temporarily disabling range gates, semantically incorrect
/*
$this->router->get('/parking/rangegate',
function() use ($found, $dataset, $fs) {
if ($found) {
Expand All @@ -81,7 +79,7 @@ function($gatename) use ($found, $dataset, $fs){
echo "Dataset not found.<br>";
}
}
);*/
);
}

public function run() {
Expand Down

0 comments on commit 12a2eed

Please sign in to comment.