Skip to content

Commit

Permalink
Implemented Predictive Analytics based Check
Browse files Browse the repository at this point in the history
  • Loading branch information
@jpatapoff
jpatapoff committed May 24, 2012
1 parent 870a356 commit a6b07c1
Show file tree
Hide file tree
Showing 14 changed files with 643 additions and 45 deletions.
24 changes: 19 additions & 5 deletions check.php
View file
Expand Up @@ -6,7 +6,7 @@
fRequest::overrideAction();
$action = fRequest::getValid('action', array('list', 'add', 'edit', 'delete'));
$check_type = fRequest::getValid('type', array('predictive', 'threshold'));
$sort = fCRUD::getSortColumn(array('name','target','warn','error','status','timestamp','count'));
$sort = fCRUD::getSortColumn(array('name','target','warn','error','status','timestamp','count','regression_type','sample','baseline','over_under','visibility','number_of_regressions'));
$sort_dir = fCRUD::getSortDirection('asc');

$check_id = fRequest::get('check_id', 'integer');
Expand Down Expand Up @@ -66,7 +66,11 @@
fMessaging::create('error', fURL::get(), $e->getMessage());
}

include VIEW_PATH . '/add_edit.php';
if ($check_type == 'threshold') {
include VIEW_PATH . '/add_edit.php';
} elseif ($check_type == 'predictive') {
include VIEW_PATH . 'add_edit_predictive_check.php';
}

// --------------------------------- //
} elseif ('add' == $action) {
Expand All @@ -86,9 +90,19 @@
}
}

include VIEW_PATH . '/add_edit.php';

if ($check_type == 'threshold') {
include VIEW_PATH . '/add_edit.php';
} elseif ($check_type == 'predictive') {
include VIEW_PATH . 'add_edit_predictive_check.php';
}

} else {
$checks = Check::findAll($check_type,$sort,$sort_dir);
include VIEW_PATH .'/list_checks.php';

if ($check_type == 'threshold') {
include VIEW_PATH . '/list_checks.php';
} elseif ($check_type == 'predictive') {
include VIEW_PATH . 'list_predictive_checks.php';
}

}
3 changes: 2 additions & 1 deletion graphite_tattle_schema_alpha.sql
View file
Expand Up @@ -41,7 +41,7 @@ CREATE TABLE `checks` (
`target` varchar(1000) NOT NULL,
`error` decimal(20,3) NOT NULL,
`warn` decimal(20,3) NOT NULL,
`sample` varchar(255) NOT NULL DEFAULT '5',
`sample` varchar(255) NOT NULL DEFAULT '10',
`baseline` varchar(255) NOT NULL DEFAULT 'average',
`visibility` int(11) NOT NULL DEFAULT '0',
`over_under` int(11) NOT NULL DEFAULT '0',
Expand All @@ -52,6 +52,7 @@ CREATE TABLE `checks` (
`repeat_delay` int(11) NOT NULL DEFAULT '60',
`type` varchar(255) NOT NULL,
`regression_type` varchar(255) DEFAULT NULL,
`number_of_regressions` int(11) DEFAULT NULL,
PRIMARY KEY (`check_id`),
UNIQUE KEY `user_id` (`user_id`,`name`)
) CHARSET=utf8;
Expand Down
179 changes: 155 additions & 24 deletions inc/classes/Check.php
View file
@@ -1,6 +1,11 @@
<?
class Check extends fActiveRecord
{

const MINUTES_PER_DAY = 1440;
const MINUTES_PER_WEEK = 10080;
const MINUTES_PER_MONTH = 40320; //TODO: adapt number of minutes per month depending on which month it is. Using 4 weeks for now

protected function configure()
{
}
Expand All @@ -19,7 +24,19 @@ static function findAll($type, $sort_column = 'name', $sort_dir = 'desc')
array('type=' => $type, 'enabled=' => true,'user_id=|visibility=' => array(fSession::get('user_id'),0)),
array($sort_column => $sort_dir)
);
}
}

static public function constructTarget($check)
{
if($check->getType() == 'threshold') {
if($check->getBaseline() == 'average') {
return 'movingAverage(' . $check->prepareTarget() . ',' . $check->getSample() . ')';
} elseif($check->getBaseline() == 'median') {
return 'movingMedian(' . $check->prepareTarget() . ',' . $check->getSample() . ')';
}
}
return $check->prepareTarget();
}

/**
* Returns all active checks on the system
Expand Down Expand Up @@ -96,24 +113,73 @@ static public function acknowledgeCheck($check=NULL,$result=NULL,$ackAll=false)
*/
static public function getData($obj=NULL)
{
if ( $GLOBALS['PRIMARY_SOURCE'] == "GANGLIA" ) {
$check_url = $GLOBALS['GANGLIA_URL'] . '/graph.php/?' .
'target=' . $obj->prepareTarget() .
'&cs=-'. $obj->prepareSample() . 'minutes' .
'&ce=now&format=json';
} else {
$check_url = $GLOBALS['GRAPHITE_URL'] . '/render/?' .
'target=movingAverage(' . $obj->prepareTarget() . ',' . $obj->prepareSample() . ')' .
'&from=-'. $obj->prepareSample() . 'minutes' .
'&format=json';
}
$json_data = @file_get_contents($check_url);
if ($json_data) {
$data = json_decode($json_data);
} else {
if($obj->getType() == 'threshold') {
if ( $GLOBALS['PRIMARY_SOURCE'] == "GANGLIA" ) {
$check_url = $GLOBALS['GANGLIA_URL'] . '/graph.php/?' .
'target=' . $obj->prepareTarget() .
'&cs=-'. $obj->prepareSample() . 'minutes' .
'&ce=now&format=json';
} else {
$check_url = $GLOBALS['GRAPHITE_URL'] . '/render/?' .
'target=' . Check::constructTarget($obj) .
'&from=-'. $obj->prepareSample() . 'minutes' .
'&format=json';
}
$json_data = @file_get_contents($check_url);
if ($json_data) {
$data = json_decode($json_data);
} else {
$data = array();
}
return $data;
} elseif($obj->getType() == 'predictive') {
$data = array();
for($i = $obj->getNumberOfRegressions(); $i >= 0; $i--) {
$regression_size = 0;

if($obj->getRegressionType() == 'daily') {
$regression_size = self::MINUTES_PER_DAY * $i;
} elseif($obj->getRegressionType() == 'weekly') {
$regression_size = self::MINUTES_PER_WEEK * $i;
} elseif($obj->getRegressionType() == 'monthly') {
$regression_size = self::MINUTES_PER_MONTH * $i;
}

$from = $regression_size + $obj->getSample();
$until = $regression_size;

$check_url = $GLOBALS['GRAPHITE_URL'] . '/render/?' .
'target=' . $obj->prepareTarget() .
'&from=-' . $from . 'minutes' .
'&until=-' . $until . 'minutes' .
'&format=json';

$json_data = @file_get_contents($check_url);
if($json_data) {
$temp_data = json_decode($json_data);
$value = 0;

if($obj->getBaseline() == 'average') {
$value = subarray_average($temp_data[0]->datapoints);
} elseif($obj->getBaseline() == 'median') {
$value = subarray_median($temp_data[0]->datapoints);
}

array_push($data, $value);

//$temp_data = $temp_data[0]->datapoints;
//fCore::debug("Iteration: " . $i,FALSE);
//for($j=0; $j < count($temp_data); $j++) {
// if($temp_data[$j][0] != 0) {
// fCore::debug($temp_data[$j][0],FALSE);
// }
//}
//fCore::debug("\n",FALSE);
//$data = array_merge($data, $temp_data);
}
}
return $data;
}
return $data;
}

/**
Expand All @@ -131,8 +197,73 @@ static public function getResultValue($data,$obj=NULL)
} elseif ($obj->getBaseline() == 'median') {
$value = subarray_median($data[0]->datapoints);
}
return $value;
}
return $value;
}

static public function getResultHistoricalValue($data,$obj)
{
$value = 0;
if(count($data) <= 1) {
return $value; //TODO: Handle cases where amount of data returned is less than expected
}
$historical_data = array_slice($data, 0, count($data) - 1);
if($obj->getBaseline() == 'average') {
$value = average($historical_data);
} elseif($obj->getBaseline() == 'median') {
$value = median($historical_data);
}
return $value;
}

static public function getResultStandardDeviation($data,$obj)
{
$value = 0;
if(count($data) <= 1) {
return $value; //TODO: Handle cases where amount of data returned is less than expected
}
$historical_data = array_slice($data, 0, count($data) - 1);
return sd($historical_data);
}

static public function getResultCurrentValue($data)
{
$value = 0;
if(count($data) == 0) {
return $value;
}
return end($data);
}


static public function setPredictiveResultsLevel($current_value,$historical_value,$stdev,$check)
{
$upper_error = $historical_value + ($check->getError() * $stdev);
$upper_warn = $historical_value + ($check->getWarn() * $stdev);
$lower_error = $historical_value - ($check->getError() * $stdev);
$lower_warn = $historical_value - ($check->getWarn() * $stdev);
$state = 0;

if ($check->getOverUnder() == 0 || $check->getOverUnder() == 2) {
if ($current_value >= $upper_error) {
$state = 1;
} elseif ($current_value >= $upper_warn) {
$state = 2;
}

if($state != 0) {
return $state;
}
}
if ($check->getOverUnder() == 1 || $check->getOverUnder() == 2) {
if ($current_value <= $lower_error) {
$state = 1;
} elseif ($current_value <= $lower_warn) {
$state = 2;
}
}
return $state;
}


/**
* Creates all Check related URLs for the site
Expand All @@ -156,13 +287,13 @@ static public function setResultsLevel($value,$obj=NULL)
}

if ($obj->getOverUnder() == 1) {
if ($value >= $obj->getWarn()) {
if ($value > $obj->getWarn()) {
$state = 0;
} elseif ($value >= $obj->getError()) {
$state = 1;
} elseif ($value > $obj->getError()) {
$state = 2;
} else {
fCore::debug('error state' . " $value compared to " . $obj->getError() . "<br />",FALSE);
$state = 2;
$state = 1;
}
return $state;
}
Expand Down Expand Up @@ -198,7 +329,7 @@ static public function showGraph($obj=NULL,$img=true,$sample=false,$width=false,
} else {

$link .= $GLOBALS['GRAPHITE_URL'] . '/render/?';
$link .= 'target=legendValue(alias(movingAverage(' . $obj->prepareTarget() . ',' . $obj->prepareSample() . ')%2C%22Check : ' . $obj->prepareName() .'%22),%22last%22)';
$link .= 'target=legendValue(alias(' . Check::constructTarget($obj) . '%2C%22Check : ' . $obj->prepareName() .'%22),%22last%22)';
if ($sample !== False) {
$link .= '&from=' . $sample;
} else {
Expand Down
5 changes: 5 additions & 0 deletions inc/config.php
View file
Expand Up @@ -5,6 +5,7 @@
$GLOBALS['DATABASE_NAME'] = 'tattle';
$GLOBALS['DATABASE_USER'] = 'dbuser';
$GLOBALS['DATABASE_PASS'] = 'dbpass';
$GLOBALS['TATTLE_DOMAIN'] = 'http://localhost';

// GRAPHITE and GANGLIA Settings
$GLOBALS['PRIMARY_SOURCE'] = 'GRAPHITE'; //Currently can be GRAPHITE or GANGLIA
Expand Down Expand Up @@ -95,6 +96,10 @@
$status_array = array('0' => 'OK', '1' => 'Error', '2' => 'Warning');
$visibility_array = array('0' => 'Public', '1' => 'Private');
$over_under_array = array('0' => 'Over', '1' => 'Under');
$over_under_both_array = array('0' => 'Over', '1' => 'Under', '2' => 'Both');
$average_median_array = array('average' => 'Average', 'median' => 'Median');
$regression_type_array = array('daily' => 'Daily', 'weekly' => 'Weekly', 'monthly' => 'Monthly');
$number_of_regressions_array = array('1' => '1','2' =>'2','3' =>'3','4' =>'4','5' =>'5','6' =>'6','7' =>'7','8' => '8','9' => '9','10' => '10', '11' => '11', '12' => '12', '13' => '13', '14' => '14', '15' => '15');
$breadcrumbs = array();
$breadcrumbs[] = array('name' => 'Home', 'url' => 'index.php', 'active'=> false);

Expand Down
44 changes: 44 additions & 0 deletions inc/functions.php
View file
Expand Up @@ -17,6 +17,20 @@ function subarray_median($data_array) {
return $median;
}

function median($data_array) {
sort($data_array);
$count = count($data_array);
$middleval = floor(($count-1)/2);
if($count % 2) {
$median = $data_array[$middleval];
} else {
$low = $data_array[$middleval];
$high = $data_array[$middleval+1];
$median = (($low+$high)/2);
}
return $median;
}

function usr_var($setting_name,$user_id=NULL) {
try{
$setting = New Setting(array('name' => $setting_name,'owner_id' => $user_id));
Expand Down Expand Up @@ -66,9 +80,39 @@ function subarray_average($data_array) {
return $average;
}

function average($data_array) {
$count = count($data_array);
$total = '';
foreach ($data_array as $value) {
$total = $total + $value;
}
$average = ($total/$count);
return $average;
}

function subarray_endvalue($data_array) {
$lastDataPoint = end($data_array);
return($lastDataPoint[0]);
}

function subarray_standard_deviation($data_array) {
//Find N
$N = count($data_array);
//Compute the mean for the distribution
$mean = subarray_average($data_array);

$sum_of_squared_differences = 0;

foreach($data_array as $value) {
//Compute the difference between each score and the mean
$difference = $value[0] - $mean;
//Square these differences
$difference_squared = pow($difference,2);
//Add up all the squared differences
$sum_of_squared_differences = $sum_of_squared_differences + $difference_squared;
}
//Divide the sum of the squared differences by the number of cases, minus one
$sd_squared = $sum_of_squared_differences / ($N - 1);
//Take the square root of the result from line 5
return sqrt($sd_squared);
}
4 changes: 3 additions & 1 deletion inc/init.php
View file
@@ -1,6 +1,7 @@
<?
// Include main class loading config file
define('TATTLE_ROOT', str_replace(array('ajax'),'',getcwd()));
//define('TATTLE_ROOT', str_replace(array('ajax'),'',getcwd()));
define('TATTLE_ROOT','/var/www/Tattle');
$web_root = dirname($_SERVER['PHP_SELF']);
if ($web_root != '/') {
$web_root .= '/';
Expand All @@ -12,6 +13,7 @@

include TATTLE_ROOT . '/inc/includes.php';
include TATTLE_ROOT . '/inc/functions.php';
include TATTLE_ROOT . '/statistical_functions.php';
include TATTLE_ROOT . '/inc/config.php';
include TATTLE_ROOT . '/inc/constructor_functions.php';

Expand Down

0 comments on commit a6b07c1

Please sign in to comment.