Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Modify time validation for API and MQTT input #821

Merged
merged 1 commit into from
Jul 15, 2018
Merged

Modify time validation for API and MQTT input #821

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 16 additions & 9 deletions Modules/input/Views/input_api.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,25 +25,32 @@

<h3><?php echo _('Posting data to EmonCMS'); ?></h3>

<p><?php echo _('The EmonCMS input API provides three ways of sending data to emoncms:'); ?></p>
<p><?php echo _('The EmonCMS HTTP input API provides three ways of sending data to EmonCMS:'); ?></p>
<ul>
<li><?php echo _('<b>input/post</b> - Post a single update from a node.'); ?></li>
<li><?php echo _('<b>input/post</b> - Post a single update from a node as either one data item or as a JSON data structure.'); ?></li>
<li><?php echo _('<b>input/bulk</b> - Bulk upload historic data from multiple nodes in a single update.'); ?></li>
<li><?php echo _('<b>Encrypted</b> - An encrypted version of both of the above'); ?></li>
</ul>

<p><?php echo _("If your starting out with EmonCMS 'input/post' is a good starting point for testing, this was the original input method when EmonCMS . The EmonPi/EmonBase uses the 'input/bulk' input method to post to a remote emoncms server as this method provides the option to efficiently bulk upload buffered data after an internet connection outage. Combining multiple updates in a single input/bulk request also reduces bandwidth requirements. " ); ?></p>
<p><?php echo _("If you're starting out with EmonCMS, 'input/post' is a good starting point for testing. This was the original input method for EmonCMS. The EmonPi/EmonBase uses the 'input/bulk' input method to post to a remote EmonCMS server as this method provides the option to efficiently bulk upload buffered data after an internet connection outage. Combining multiple updates in a single input/bulk request also reduces bandwidth requirements. " ); ?></p>

<p><?php echo _("For applications where HTTPS or TLS is not available, emoncms offers an in-built transport layer encryption solution where the emoncms apikey is used as the pre-shared key for encrypting the data with AES-128-CBC." ); ?></p>
<p><?php echo _("For applications where HTTPS or TLS is not available, EmonCMS offers an in-built transport layer encryption solution where the EmonCMS apikey is used as the pre-shared key for encrypting the data with AES-128-CBC." ); ?></p>

<h4><?php echo _('input/post'); ?></h4>

<p><?php echo _('The "fulljson" format is recommended for new integrations, it uses the PHP JSON decoder and answer is also in json.<br>The "json like" format is based on the CSV input parsing implementation and maintained for backwards compatibility.'); ?><br><?php echo _('A node name can be a name e.g: emontx or a number e.g: 10.'); ?><br><?php echo _('The input/post API is compatible with both GET and POST request methods (POST examples given use curl).'); ?></p>

<ul>
<li><?php echo _('The <b>fulljson</b> format is recommended for new integrations. It uses the PHP JSON decoder and answer is also in json.');?></li>
<li><?php echo _('The <b>json like</b> format is based on the CSV input parsing implementation and maintained for backwards compatibility.'); ?></li>
<li><?php echo _('The <b>node</b> parameter can be an unquoted string e.g: emontx or a number e.g: 10.'); ?></li>
<li><?php echo _('Time is set as system time unless a <b>time</b> element is included. It can be either a parameter &time (unquoted) or as part of the JSON data structure. If both are included the parameter value will take precedence. Time is a UNIX timestamp and can be in seconds or a string PHP can decode (ISO8061 recommended). If you are having problems check you are using seconds not milliseconds. If part of the JSON data structure as a string, the node value will report NULL'); ?></li>
<li><?php echo _('The input/post API is compatible with both GET and POST request methods (POST examples given use curl).'); ?></li>
</ul>
<table class="table">
<tr><th><?php echo _('Description'); ?></th><th><?php echo _('Method'); ?></th><th><?php echo _('Example'); ?></th></tr>
<tr><th><?php echo _('Description'); ?></th><th><?php echo _('HTTP Method'); ?></th><th><?php echo _('Example'); ?></th></tr>

<tr><td><?php echo _('JSON format'); ?></td><td>GET</td><td><a href="<?php echo $path; ?>input/post?node=emontx&fulljson={%22power1%22:100,%22power2%22:200,%22power3%22:300}"><?php echo $path; ?>input/post?<b>node=emontx</b>&fulljson={"power1":100,"power2":200,"power3":300}</a></td></tr>

<tr><td><?php echo _('JSON format - with time (as a string in this example)'); ?></td><td>GET</td><td><a href="<?php echo $path; ?>input/post?node=emontx&fulljson={%22power1%22:100,%22power2%22:200,%22power3%22:300,%22time%22:%22<?php echo date(DATE_ATOM);?>%22}"><?php echo $path; ?>input/post?<b>node=emontx</b>&fulljson={"power1":100,"power2":200,"power3":300,"time":"<?php echo date(DATE_ATOM);?>"}</a></td></tr>

<tr><td><?php echo _('JSON like format'); ?></td><td>GET</td><td><a href="<?php echo $path; ?>input/post?node=emontx&json={power1:100,power2:200,power3:300}"><?php echo $path; ?>input/post?<b>node=emontx</b>&json={power1:100,power2:200,power3:300}</a></td></tr>

Expand Down Expand Up @@ -99,15 +106,15 @@

<h4><?php echo _('Encryption'); ?></h4>

<p><?php echo _("For applications where HTTPS or TLS is not available, emoncms offers an in-built transport layer encryption solution where the emoncms apikey is used as the pre-shared key for encrypting the data with AES-128-CBC." ); ?><br><?php echo _("There is a PHP example of how to generate an encrypted request here: "); ?><a href="https://github.com/emoncms/emoncms/blob/input-improvements/docs/input_encrypted.md">PHP Example source code.</a></p>
<p><?php echo _("For applications where HTTPS or TLS is not available, EmonCMS offers an in-built transport layer encryption solution where the emoncms apikey is used as the pre-shared key for encrypting the data with AES-128-CBC." ); ?><br><?php echo _("There is a PHP example of how to generate an encrypted request here: "); ?><a href="https://github.com/emoncms/emoncms/blob/input-improvements/docs/input_encrypted.md">PHP Example source code.</a></p>

<p>
1. Start with a request string conforming with the API options above e.g: node=emontx&data={power1:100,power2:200,power3:300}<br>
2. Create an initialization vector.<br>
3. Encrypt using AES-128-CBC.<br>
4. Create a single string starting with the initialization vector followed by the cipher-text result of the AES-128-CBC encryption.<br>
5. Convert to a base64 encoded string.<br>
6. Generate a HMAC_HASH of the data string together, using the emoncms apikey for authorization.<br>
6. Generate a HMAC_HASH of the data string together, using the EmonCMS apikey for authorization.<br>
7. Send the encrypted string in the POST body of a request to either input/post or input/bulk with headers properties 'Content-type' and 'Authorization' set as below<br>
8. Verify the result. The result is a base64 encoded sha256 hash of the json data string.
</p>
Expand Down
72 changes: 65 additions & 7 deletions Modules/input/input_methods.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public function __construct($mysqli,$redis,$user,$input,$feed,$process,$device)
public function post($userid)
{
// Nodeid
global $route,$param;
global $route,$param,$log;

// Default nodeid is zero
$nodeid = 0;
Expand All @@ -53,7 +53,30 @@ public function post($userid)
if ($nodeid=="") $nodeid = 0;

// Time
if ($param->exists('time')) $time = (int) $param->val('time'); else $time = time();
//if ($param->exists('time')) $time = (int) $param->val('time'); else $time = time();
if ($param->exists('time')) {
$inputtime = $param->val('time');

// validate time
if (is_numeric($inputtime)){
$log->info("Valid time in seconds used ".$inputtime);
$time = (int) $inputtime;
} elseif (is_string($inputtime)){
if (($timestamp = strtotime($inputtime)) === false) {
//If time string is not valid, use system time.
$log->warn("Time string not valid ".$inputtime);
$time = time();
} else {
$log->info("Valid time string used ".$inputtime);
$time = $timestamp;
}
} else {
$log->warn("Time parameter not valid ".$inputtime);
$time = time();
}
} else {
$time = time();
}

// Data
$datain = false;
Expand All @@ -70,11 +93,46 @@ public function post($userid)
if ($datain=="") return "Request contains no data via csv, json or data tag";

if ($param->exists('fulljson')) {
$inputs = json_decode($datain, true, 2);
if (is_null($inputs)) {
return "Error decoding JSON string (invalid or too deeply nested)";
} else if (!is_array($inputs)) {
return "Input must be a JSON object";
$jsondata = null;
$jsondata = json_decode($datain,true,2);
if ((json_last_error() === JSON_ERROR_NONE) && is_array($jsondata)) {
// JSON is valid - is it an array
//$jsoninput = true;
$log->info("Valid JSON found ");
//Create temporary array and change all keys to lower case to look for a 'time' key
$jsondataLC = array_change_key_case($jsondata);

// If JSON, check to see if there is a time value else set to time now.
// Time set as a parameter takes precedence.
if ($param->exists('time')) {
$log->info("Time from parameter used");
} elseif (array_key_exists('time',$jsondataLC)){
$inputtime = $jsondataLC['time'];

// validate time
if (is_numeric($inputtime)){
$log->info("Valid time in seconds used ".$inputtime);
$time = (int) $inputtime;
} elseif (is_string($inputtime)){
if (($timestamp = strtotime($inputtime)) === false) {
//If time string is not valid, use system time.
$log->warn("Time string not valid ".$inputtime);
$time = time();
} else {
$log->info("Valid time string used ".$inputtime);
$time = $timestamp;
}
} else {
$log->warn("Time not valid ".$inputtime);
$time = time();
}
} else {
$log->info("No time element found in JSON - System time used");
$time = time();
}
$inputs = $jsondata;
} else {
return "Input in not a valid JSON object";
}
} else {
$json = preg_replace('/[^\p{N}\p{L}_\s-.:,]/u','',$datain);
Expand Down
27 changes: 16 additions & 11 deletions scripts/phpmqtt_input.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,8 @@ function message($message)
$jsoninput = false;
$topic = $message->topic;
$value = $message->payload;

$time = time();

global $mqtt_server, $user, $input, $process, $device, $log, $count;

//Check and see if the input is a valid JSON and when decoded is an array. A single number is valid JSON.
Expand All @@ -194,29 +195,33 @@ function message($message)
//Create temporary array and change all keys to lower case to look for a 'time' key
$jsondataLC = array_change_key_case($jsondata);

#If JSON check to see if there is a time value else set to time now.
// If JSON, check to see if there is a time value else set to time now.
if (array_key_exists('time',$jsondataLC)){
$time = $jsondataLC['time'];
if (is_string($time)){
if (($timestamp = strtotime($time)) === false) {
$inputtime = $jsondataLC['time'];

// validate time
if (is_numeric($inputtime)){
$log->info("Valid time in seconds used ".$inputtime);
$time = (int) $inputtime;
} elseif (is_string($inputtime)){
if (($timestamp = strtotime($inputtime)) === false) {
//If time string is not valid, use system time.
$log->warn("Time string not valid ".$inputtime);
$time = time();
$log->warn("Time string not valid ".$time);
} else {
$log->info("Valid time string used ".$time);
$log->info("Valid time string used ".$inputtime);
$time = $timestamp;
}
} else {
$log->info("Valid time in seconds used ".$time);
//Do nothings as it has been assigned to $time as a value
$log->warn("Time value not valid ".$inputtime);
$time = time();
}
} else {
$log->info("No time element found in JSON - System time used");
$time = time();
}
} else {
$jsoninput = false;
$log->info("No JSON found - System time used");
$time = time();
}

Expand Down Expand Up @@ -329,4 +334,4 @@ function exceptions_error_handler($severity, $message, $filename, $lineno) {
if (error_reporting() & $severity) {
throw new ErrorException($message, 0, $severity, $filename, $lineno);
}
}
}