Skip to content

Commit

Permalink
Merge pull request #1122 from emoncms/fix_cleardata_feed_issue
Browse files Browse the repository at this point in the history 
fix cleardata feed issue.
  • Loading branch information
@TrystanLea
TrystanLea authored Nov 16, 2018
2 parents 6ff07e8 + d977f45 commit 2055eb3
Showing 1 changed file with 38 additions and 35 deletions.
73 changes: 38 additions & 35 deletions Modules/feed/engine/PHPFina.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ class PHPFina implements engine_methods
private $dir = "/var/lib/phpfina/";
private $log;
private $writebuffer = array();
private $maxpadding = 3153600; // 1 year @ 10s

/**
* Constructor.
Expand Down Expand Up @@ -82,7 +83,6 @@ public function delete($feedid)
if (!$meta = $this->get_meta($feedid)) return false;
unlink($this->dir.$feedid.".meta");
unlink($this->dir.$feedid.".dat");
if (isset($metadata_cache[$feedid])) { unset($metadata_cache[$feedid]); } // Clear static cache
}

/**
Expand All @@ -100,31 +100,25 @@ public function get_meta($feedid)
return false;
}

static $metadata_cache = array(); // Array to hold the cache
if (isset($metadata_cache[$feedid])) {
return $metadata_cache[$feedid]; // Retrieve from static cache
} else {
// Open and read meta data file
// The start_time and interval are saved as two consecutive unsigned integers
$meta = new stdClass();
$metafile = fopen($this->dir.$feedname, 'rb');
fseek($metafile,8);
$tmp = unpack("I",fread($metafile,4));
$meta->interval = $tmp[1];
$tmp = unpack("I",fread($metafile,4));
$meta->start_time = $tmp[1];
fclose($metafile);

$meta->npoints = $this->get_npoints($feedid);
// Open and read meta data file
// The start_time and interval are saved as two consecutive unsigned integers
$meta = new stdClass();
$metafile = fopen($this->dir.$feedname, 'rb');
fseek($metafile,8);
$tmp = unpack("I",fread($metafile,4));
$meta->interval = $tmp[1];
$tmp = unpack("I",fread($metafile,4));
$meta->start_time = $tmp[1];
fclose($metafile);

$meta->npoints = $this->get_npoints($feedid);

if ($meta->start_time>0 && $meta->npoints==0) {
$this->log->warn("PHPFina:get_meta start_time already defined but npoints is 0");
return false;
}

$metadata_cache[$feedid] = $meta; // Cache it
return $meta;
if ($meta->start_time>0 && $meta->npoints==0) {
$this->log->warn("PHPFina:get_meta start_time already defined but npoints is 0");
return false;
}

return $meta;
}

/**
Expand Down Expand Up @@ -199,10 +193,7 @@ public function post($id,$timestamp,$value,$padding_mode=null)
// Write padding
$padding = ($pos - $last_pos)-1;

// Max padding = 1 million datapoints ~4mb gap of 115 days at 10s
$maxpadding = 1000000;

if ($padding>$maxpadding) {
if ($padding>$this->maxpadding) {
$this->log->warn("post() padding max block size exeeded id=$id, $padding dp");
return false;
}
Expand Down Expand Up @@ -669,11 +660,17 @@ public function post_bulk_prepare($feedid,$timestamp,$value,$padding_mode)
$timestamp = floor($timestamp / $meta->interval) * $meta->interval;

// If this is a new feed (npoints == 0) then set the start time to the current datapoint
if ($meta->npoints == 0 && $meta->start_time==0) {
$meta->start_time = $timestamp;
$this->create_meta($feedid,$meta);
if ($meta->start_time==0) {
if ($meta->npoints == 0) {
$meta->start_time = $timestamp;
$this->create_meta($feedid,$meta);
$this->log->info("post_bulk_prepare() start_time=0 setting meta start_time=$timestamp");
} else {
$this->log->error("post_bulk_prepare() start_time=0, npoints>0");
return false;
}
}

if ($timestamp < $meta->start_time) {
$this->log->warn("post_bulk_prepare() timestamp=$timestamp older than feed starttime=$meta->start_time feedid=$feedid");
return false; // in the past
Expand All @@ -688,6 +685,11 @@ public function post_bulk_prepare($feedid,$timestamp,$value,$padding_mode)
if ($pos>$last_pos) {
$npadding = ($pos - $last_pos)-1;

if ($npadding>$this->maxpadding) {
$this->log->warn("post() padding max block size exeeded id=$feedid, $npadding dp");
return false;
}

if (!isset($this->writebuffer[$feedid])) {
$this->writebuffer[$feedid] = "";
}
Expand Down Expand Up @@ -812,7 +814,7 @@ private function get_npoints($feedid)
clearstatcache($this->dir.$feedid.".dat");
$bytesize += filesize($this->dir.$feedid.".dat");
}

if (isset($this->writebuffer[$feedid]))
$bytesize += strlen($this->writebuffer[$feedid]);

Expand Down Expand Up @@ -1280,7 +1282,8 @@ public function clear($feedid) {
ftruncate($f, 0);
fclose($f);
}
if (isset($metadata_cache[$feedid])) { unset($metadata_cache[$feedid]); } // Clear static cache
if (isset($this->writebuffer[$feedid])) $this->writebuffer[$feedid] = "";

$this->create_meta($feedid, $meta); // create meta first to avoid $this->create() from creating new one
$this->create($feedid,array('interval'=>$meta->interval));

Expand Down Expand Up @@ -1333,7 +1336,7 @@ public function trim($feedid,$start_time) {
fclose($fh);

$this->log->info(".data file trimmed to $writtenBytes bytes");
if (isset($metadata_cache[$feedid])) { unset($metadata_cache[$feedid]); } // Clear static cache
if (isset($this->writebuffer[$feedid])) $this->writebuffer[$feedid] = "";
$meta->start_time = $start_time;
$this->create_meta($feedid, $meta); // set the new start time in the feed's meta
return array('success'=>true,'message'=>"$writtenBytes bytes written");
Expand Down

0 comments on commit 2055eb3

Please sign in to comment.