Skip to content

Commit

Permalink
Merge pull request #67 from beepnl/add-weather-data
Browse files Browse the repository at this point in the history 
Add weather data
  • Loading branch information
@pvgennip
pvgennip committed Nov 24, 2020
2 parents d946932 + 11d6716 commit ee507cb
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 16 deletions.
73 changes: 61 additions & 12 deletions app/Http/Controllers/ResearchController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ public function show($id, Request $request)

//die(print_r([$request->input('user_ids'), $consent_users_selected, $users]));
// Fill dates array
$assets = ["users"=>0, "apiaries"=>0, "hives"=>0, "inspections"=>0, "devices"=>0, "measurements"=>0];
$assets = ["users"=>0, "apiaries"=>0, "hives"=>0, "inspections"=>0, "devices"=>0, "measurements"=>0, "weather"=>0];

$moment = $moment_start;
while($moment < $moment_end)
Expand Down Expand Up @@ -316,6 +316,14 @@ public function show($id, Request $request)
'Data file']
];

$spreadsheet_array['Weather data'] = [
['User_id',
'Device_id',
'Date from',
'Date to',
'Data file']
];

// Add item names to header row of inspections
// first combine all user's itemnames
$item_ancs = [];
Expand Down Expand Up @@ -368,21 +376,31 @@ public function show($id, Request $request)
$user_devices = Device::where('user_id', $user_id)->where('created_at', '<', $date_until)->orderBy('created_at')->get();
$user_inspections = User::find($user_id)->inspections()->with('items')->where('created_at', '>=', $date_start)->where('created_at', '<', $date_until)->orderBy('created_at')->get();
$user_measurements = [];
$user_weather_data = [];

//die(print_r([$user_apiaries->toArray(), $user_hives->toArray()]));

if ($user_devices->count() > 0)
{
// get daily counts of sensor measurements
$points = [];
$weather = [];
$user_device_keys = [];
$user_dloc_coords = [];

// Add sensor data
foreach ($user_devices as $device)
{
$user_device_keys[]= '"key" = \''.$device->key.'\' OR "key" = \''.strtolower($device->key).'\' OR "key" = \''.strtoupper($device->key).'\'';
$loc = $device->location();
if ($loc && isset($loc->coordinate_lat) && isset($loc->coordinate_lon))
$user_dloc_coords[] = '("lat" = \''.$loc->coordinate_lat.'\' AND "lon" = \''.$loc->coordinate_lon.'\')';
}

$user_device_keys = '('.implode(' OR ', $user_device_keys).')';

try{
$points = $influx::query('SELECT COUNT("bv") as "count" FROM "sensors" WHERE '.$user_device_keys.' AND time >= \''.$user_consents[0]->updated_at.'\' AND time <= \''.$moment_end->format('Y-m-d H:i:s').'\' GROUP BY time(1d) fill(null)')->getPoints();
$points = $influx::query('SELECT COUNT("bv") as "count" FROM "sensors" WHERE '.$user_device_keys.' AND time >= \''.$user_consents[0]->updated_at.'\' AND time <= \''.$moment_end->format('Y-m-d H:i:s').'\' GROUP BY time(1d)')->getPoints();
} catch (InfluxDB\Exception $e) {
// return Response::json('influx-group-by-query-error', 500);
}
Expand All @@ -391,6 +409,19 @@ public function show($id, Request $request)
foreach ($points as $point)
$user_measurements[substr($point['time'],0,10)] = $point['count'];
}

// Add weather data
$user_location_coord_where = '('.implode(' OR ', $user_dloc_coords).')';
try{
$weather = $influx::query('SELECT COUNT("temperature") as "count" FROM "weather" WHERE '.$user_location_coord_where.' AND time >= \''.$user_consents[0]->updated_at.'\' AND time <= \''.$moment_end->format('Y-m-d H:i:s').'\' GROUP BY time(1d)')->getPoints(); // get first weather date
} catch (InfluxDB\Exception $e) {
// return Response::json('influx-group-by-query-error', 500);
}
if (count($weather) > 0)
{
foreach ($weather as $point)
$user_weather_data[substr($point['time'],0,10)] = $point['count'];
}
}

// go over dates, compare consent dates
Expand Down Expand Up @@ -448,21 +479,36 @@ public function show($id, Request $request)

if ($sensordata && $user_devices->count() > 0)
{
foreach ($user_devices as $dev)
foreach ($user_devices as $device)
{
// Add device to spreadsheet
if ($dev->created_at < $date_next_consent)
if ($device->created_at < $date_next_consent)
{
$spreadsheet_array[__('export.devices')][] = $this->getDevice($user_id, $dev);
$spreadsheet_array[__('export.devices')][] = $this->getDevice($user_id, $device);

// Export data to file per device / period
$fileName = strtolower(env('APP_NAME')).'-export-'.$research->name.'-device-id-'.$dev->id.'-sensor-data-'.substr($date_curr_consent,0,10).'-'.substr($date_next_consent,0,10).'-'.Str::random(10).'.csv';
$filePath = $this->exportCsvFromInflux($dev, $date_curr_consent, $date_next_consent, $fileName, '*');
$where = 'WHERE ("key" = \''.$device->key.'\' OR "key" = \''.strtolower($device->key).'\' OR "key" = \''.strtoupper($device->key).'\') AND time >= \''.$date_curr_consent.'\' AND time <= \''.$date_next_consent.'\'';
$fileName = strtolower(env('APP_NAME')).'-export-'.$research->name.'-device-id-'.$device->id.'-sensor-data-'.substr($date_curr_consent,0,10).'-'.substr($date_next_consent,0,10).'-'.Str::random(10).'.csv';
$filePath = $this->exportCsvFromInflux($where, $fileName, '*', 'sensors');
if ($filePath)
{
$spreadsheet_array['Sensor data'][] = [$user_id, $dev->id, $date_curr_consent, $date_next_consent, $filePath];
$spreadsheet_array['Sensor data'][] = [$user_id, $device->id, $date_curr_consent, $date_next_consent, $filePath];
$sensor_urls[$fileName] = $filePath;
}

// Export data to file per device / period
$loc = $device->location();
if ($loc && isset($loc->coordinate_lat) && isset($loc->coordinate_lon))
{
$where = 'WHERE "lat" = \''.$loc->coordinate_lat.'\' AND "lon" = \''.$loc->coordinate_lon.'\' AND time >= \''.$date_curr_consent.'\' AND time <= \''.$date_next_consent.'\'';
$fileName = strtolower(env('APP_NAME')).'-export-'.$research->name.'-device-id-'.$device->id.'-weather-data-'.substr($date_curr_consent,0,10).'-'.substr($date_next_consent,0,10).'-'.Str::random(10).'.csv';
$filePath = $this->exportCsvFromInflux($where, $fileName, '*', 'weather');
if ($filePath)
{
$spreadsheet_array['Weather data'][] = [$user_id, $device->id, $date_curr_consent, $date_next_consent, $filePath];
$sensor_urls[$fileName] = $filePath;
}
}
}
}
}
Expand All @@ -485,6 +531,9 @@ public function show($id, Request $request)
if (in_array($d, array_keys($user_measurements)))
$dates[$d]['measurements']= $v['measurements'] + $user_measurements[$d];

if (in_array($d, array_keys($user_weather_data)))
$dates[$d]['weather']= $v['weather'] + $user_weather_data[$d];

}

$i++;
Expand All @@ -501,7 +550,7 @@ public function show($id, Request $request)
{
foreach ($day_arr as $asset => $count)
{
if ($asset == 'inspections' || $asset == 'measurements')
if ($asset == 'inspections' || $asset == 'measurements' || $asset == 'weather')
$totals[$asset] += $count;
else
$totals[$asset] = max($totals[$asset], $count);
Expand Down Expand Up @@ -804,16 +853,16 @@ private function getInspections($user_id, $inspections, $item_names, $date_start
return $table;
}

private function exportCsvFromInflux(Device $device, $start, $end, $fileName='research-export-', $measurements='*', $separator=',')
private function exportCsvFromInflux($where, $fileName='research-export-', $measurements='*', $database='sensors', $separator=',')
{
$options= ['precision'=>'rfc3339', 'format'=>'csv'];

if ($measurements == null || $measurements == '' || $measurements === '*')
$sensor_measurements = '*';
else
$sensor_measurements = '"'.implode('","',$measurements).'"';
$sensor_measurements = $measurements;

$query = 'SELECT '.$sensor_measurements.' FROM "sensors" WHERE ("key" = \''.$device->key.'\' OR "key" = \''.strtolower($device->key).'\' OR "key" = \''.strtoupper($device->key).'\') AND time >= \''.$start.'\' AND time <= \''.$end.'\'';
$query = 'SELECT '.$sensor_measurements.' FROM "'.$database.'" '.$where;

try{
$client = new \Influx;
Expand Down
16 changes: 12 additions & 4 deletions resources/views/research/show.blade.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
<div class="col-md-1"></div>
<div class="col-xs-12 col-md-3">
<div class="form-group {{ $errors->has('date_start') ? 'has-error' : ''}}">
<label for="date_start" control-label>{{ 'From date (filters inspections/measurements)' }}</label>
<label for="date_start" control-label>{{ 'From date (filters inspections/measurements/weather data)' }}</label>
<div>
<input class="form-control" name="date_start" type="date" id="date_start" min="{{substr($research->start_date, 0, 10)}}" max="{{substr($research->end_date, 0, 10)}}" value="{{ isset($date_start) ? substr($date_start, 0, 10) : '' }}" >
{!! $errors->first('date_start', '<p class="help-block">:message</p>') !!}
Expand All @@ -40,7 +40,7 @@
<div class="col-md-1"></div>
<div class="col-xs-12 col-md-3">
<div class="form-group {{ $errors->has('date_until') ? 'has-error' : ''}}">
<label for="date_until" control-label>{{ 'Until date (filters inspections/measurements)' }}</label>
<label for="date_until" control-label>{{ 'Until date (filters inspections/measurements/weather data)' }}</label>
<div>
<input class="form-control" name="date_until" type="date" id="date_until" min="{{substr($research->start_date, 0, 10)}}" max="{{substr($research->end_date, 0, 10)}}" value="{{ isset($date_until) ? substr($date_until, 0, 10) : '' }}" >
{!! $errors->first('date_until', '<p class="help-block">:message</p>') !!}
Expand Down Expand Up @@ -83,6 +83,9 @@
<tr>
<th class="row-header"><span><i class="fa fa-2x fa-line-chart"></i> Measurements ({{ $totals['measurements'] }})</span></th>
</tr>
<tr>
<th class="row-header"><span><i class="fa fa-2x fa-thermometer"></i> Weather data ({{ $totals['weather'] }})</span></th>
</tr>
</tbody>
</table>
</div>
Expand Down Expand Up @@ -126,6 +129,11 @@
<td>{{ $d['measurements'] > 0 ? $d['measurements'] : '' }}</td>
@endforeach
</tr>
<tr>
@foreach($dates as $date => $d)
<td>{{ $d['weather'] > 0 ? $d['weather'] : '' }}</td>
@endforeach
</tr>
</tbody>
</table>
</div>
Expand Down Expand Up @@ -162,8 +170,8 @@
<a href="{{$download_url}}" target="_blank"><i class="fa fa-download"></i> Download selected consent data set</a>
<div style="display:block; height: 10px;"></div>
@if(count($sensor_urls) > 0)
<h4>Sensor data</h4>
<p>Export files are saved per device per consent period.
<h4>Sensor and weather data</h4>
<p>Export files are saved per device/location per consent period.
All data per device in the highest possible resolution as comma separated (,) .csv file that you can open in Excel, or SPSS.
<br>
<em>NB: The date time data in the 'time' column is in GMT time (this differs from what you see in the BEEP app), formatted by the RFC 3339 date-time standard.</em>
Expand Down

0 comments on commit ee507cb

Please sign in to comment.