Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add weather data #67

Merged
merged 8 commits into from
Nov 24, 2020
Merged

Add weather data #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5fe2dbe
Add weather data to research data
pvgennip Nov 24, 2020
fbb8bbc
Merge branch 'master' into add-weather-data
pvgennip Nov 24, 2020
6d26713
Update ResearchController.php
pvgennip Nov 24, 2020
6cceac4
Update ResearchController.php
pvgennip Nov 24, 2020
5411ebb
Update ResearchController.php
pvgennip Nov 24, 2020
960088e
Update ResearchController.php
pvgennip Nov 24, 2020
25a6b2f
Update ResearchController.php
pvgennip Nov 24, 2020
11d6716
Update ResearchController.php
pvgennip Nov 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 61 additions & 12 deletions app/Http/Controllers/ResearchController.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ public function show($id, Request $request)

//die(print_r([$request->input('user_ids'), $consent_users_selected, $users]));
// Fill dates array
$assets = ["users"=>0, "apiaries"=>0, "hives"=>0, "inspections"=>0, "devices"=>0, "measurements"=>0];
$assets = ["users"=>0, "apiaries"=>0, "hives"=>0, "inspections"=>0, "devices"=>0, "measurements"=>0, "weather"=>0];

$moment = $moment_start;
while($moment < $moment_end)
Expand Down Expand Up @@ -316,6 +316,14 @@ public function show($id, Request $request)
'Data file']
];

$spreadsheet_array['Weather data'] = [
['User_id',
'Device_id',
'Date from',
'Date to',
'Data file']
];

// Add item names to header row of inspections
// first combine all user's itemnames
$item_ancs = [];
Expand Down Expand Up @@ -368,21 +376,31 @@ public function show($id, Request $request)
$user_devices = Device::where('user_id', $user_id)->where('created_at', '<', $date_until)->orderBy('created_at')->get();
$user_inspections = User::find($user_id)->inspections()->with('items')->where('created_at', '>=', $date_start)->where('created_at', '<', $date_until)->orderBy('created_at')->get();
$user_measurements = [];
$user_weather_data = [];

//die(print_r([$user_apiaries->toArray(), $user_hives->toArray()]));

if ($user_devices->count() > 0)
{
// get daily counts of sensor measurements
$points = [];
$weather = [];
$user_device_keys = [];
$user_dloc_coords = [];

// Add sensor data
foreach ($user_devices as $device)
{
$user_device_keys[]= '"key" = \''.$device->key.'\' OR "key" = \''.strtolower($device->key).'\' OR "key" = \''.strtoupper($device->key).'\'';
$loc = $device->location();
if ($loc && isset($loc->coordinate_lat) && isset($loc->coordinate_lon))
$user_dloc_coords[] = '("lat" = \''.$loc->coordinate_lat.'\' AND "lon" = \''.$loc->coordinate_lon.'\')';
}

$user_device_keys = '('.implode(' OR ', $user_device_keys).')';

try{
$points = $influx::query('SELECT COUNT("bv") as "count" FROM "sensors" WHERE '.$user_device_keys.' AND time >= \''.$user_consents[0]->updated_at.'\' AND time <= \''.$moment_end->format('Y-m-d H:i:s').'\' GROUP BY time(1d) fill(null)')->getPoints();
$points = $influx::query('SELECT COUNT("bv") as "count" FROM "sensors" WHERE '.$user_device_keys.' AND time >= \''.$user_consents[0]->updated_at.'\' AND time <= \''.$moment_end->format('Y-m-d H:i:s').'\' GROUP BY time(1d)')->getPoints();
} catch (InfluxDB\Exception $e) {
// return Response::json('influx-group-by-query-error', 500);
}
Expand All @@ -391,6 +409,19 @@ public function show($id, Request $request)
foreach ($points as $point)
$user_measurements[substr($point['time'],0,10)] = $point['count'];
}

// Add weather data
$user_location_coord_where = '('.implode(' OR ', $user_dloc_coords).')';
try{
$weather = $influx::query('SELECT COUNT("temperature") as "count" FROM "weather" WHERE '.$user_location_coord_where.' AND time >= \''.$user_consents[0]->updated_at.'\' AND time <= \''.$moment_end->format('Y-m-d H:i:s').'\' GROUP BY time(1d)')->getPoints(); // get first weather date
} catch (InfluxDB\Exception $e) {
// return Response::json('influx-group-by-query-error', 500);
}
if (count($weather) > 0)
{
foreach ($weather as $point)
$user_weather_data[substr($point['time'],0,10)] = $point['count'];
}
}

// go over dates, compare consent dates
Expand Down Expand Up @@ -448,21 +479,36 @@ public function show($id, Request $request)

if ($sensordata && $user_devices->count() > 0)
{
foreach ($user_devices as $dev)
foreach ($user_devices as $device)
{
// Add device to spreadsheet
if ($dev->created_at < $date_next_consent)
if ($device->created_at < $date_next_consent)
{
$spreadsheet_array[__('export.devices')][] = $this->getDevice($user_id, $dev);
$spreadsheet_array[__('export.devices')][] = $this->getDevice($user_id, $device);

// Export data to file per device / period
$fileName = strtolower(env('APP_NAME')).'-export-'.$research->name.'-device-id-'.$dev->id.'-sensor-data-'.substr($date_curr_consent,0,10).'-'.substr($date_next_consent,0,10).'-'.Str::random(10).'.csv';
$filePath = $this->exportCsvFromInflux($dev, $date_curr_consent, $date_next_consent, $fileName, '*');
$where = 'WHERE ("key" = \''.$device->key.'\' OR "key" = \''.strtolower($device->key).'\' OR "key" = \''.strtoupper($device->key).'\') AND time >= \''.$date_curr_consent.'\' AND time <= \''.$date_next_consent.'\'';
$fileName = strtolower(env('APP_NAME')).'-export-'.$research->name.'-device-id-'.$device->id.'-sensor-data-'.substr($date_curr_consent,0,10).'-'.substr($date_next_consent,0,10).'-'.Str::random(10).'.csv';
$filePath = $this->exportCsvFromInflux($where, $fileName, '*', 'sensors');
if ($filePath)
{
$spreadsheet_array['Sensor data'][] = [$user_id, $dev->id, $date_curr_consent, $date_next_consent, $filePath];
$spreadsheet_array['Sensor data'][] = [$user_id, $device->id, $date_curr_consent, $date_next_consent, $filePath];
$sensor_urls[$fileName] = $filePath;
}

// Export data to file per device / period
$loc = $device->location();
if ($loc && isset($loc->coordinate_lat) && isset($loc->coordinate_lon))
{
$where = 'WHERE "lat" = \''.$loc->coordinate_lat.'\' AND "lon" = \''.$loc->coordinate_lon.'\' AND time >= \''.$date_curr_consent.'\' AND time <= \''.$date_next_consent.'\'';
$fileName = strtolower(env('APP_NAME')).'-export-'.$research->name.'-device-id-'.$device->id.'-weather-data-'.substr($date_curr_consent,0,10).'-'.substr($date_next_consent,0,10).'-'.Str::random(10).'.csv';
$filePath = $this->exportCsvFromInflux($where, $fileName, '*', 'weather');
if ($filePath)
{
$spreadsheet_array['Weather data'][] = [$user_id, $device->id, $date_curr_consent, $date_next_consent, $filePath];
$sensor_urls[$fileName] = $filePath;
}
}
}
}
}
Expand All @@ -485,6 +531,9 @@ public function show($id, Request $request)
if (in_array($d, array_keys($user_measurements)))
$dates[$d]['measurements']= $v['measurements'] + $user_measurements[$d];

if (in_array($d, array_keys($user_weather_data)))
$dates[$d]['weather']= $v['weather'] + $user_weather_data[$d];

}

$i++;
Expand All @@ -501,7 +550,7 @@ public function show($id, Request $request)
{
foreach ($day_arr as $asset => $count)
{
if ($asset == 'inspections' || $asset == 'measurements')
if ($asset == 'inspections' || $asset == 'measurements' || $asset == 'weather')
$totals[$asset] += $count;
else
$totals[$asset] = max($totals[$asset], $count);
Expand Down Expand Up @@ -804,16 +853,16 @@ private function getInspections($user_id, $inspections, $item_names, $date_start
return $table;
}

private function exportCsvFromInflux(Device $device, $start, $end, $fileName='research-export-', $measurements='*', $separator=',')
private function exportCsvFromInflux($where, $fileName='research-export-', $measurements='*', $database='sensors', $separator=',')
{
$options= ['precision'=>'rfc3339', 'format'=>'csv'];

if ($measurements == null || $measurements == '' || $measurements === '*')
$sensor_measurements = '*';
else
$sensor_measurements = '"'.implode('","',$measurements).'"';
$sensor_measurements = $measurements;

$query = 'SELECT '.$sensor_measurements.' FROM "sensors" WHERE ("key" = \''.$device->key.'\' OR "key" = \''.strtolower($device->key).'\' OR "key" = \''.strtoupper($device->key).'\') AND time >= \''.$start.'\' AND time <= \''.$end.'\'';
$query = 'SELECT '.$sensor_measurements.' FROM "'.$database.'" '.$where;

try{
$client = new \Influx;
Expand Down
16 changes: 12 additions & 4 deletions resources/views/research/show.blade.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
<div class="col-md-1"></div>
<div class="col-xs-12 col-md-3">
<div class="form-group {{ $errors->has('date_start') ? 'has-error' : ''}}">
<label for="date_start" control-label>{{ 'From date (filters inspections/measurements)' }}</label>
<label for="date_start" control-label>{{ 'From date (filters inspections/measurements/weather data)' }}</label>
<div>
<input class="form-control" name="date_start" type="date" id="date_start" min="{{substr($research->start_date, 0, 10)}}" max="{{substr($research->end_date, 0, 10)}}" value="{{ isset($date_start) ? substr($date_start, 0, 10) : '' }}" >
{!! $errors->first('date_start', '<p class="help-block">:message</p>') !!}
Expand All @@ -40,7 +40,7 @@
<div class="col-md-1"></div>
<div class="col-xs-12 col-md-3">
<div class="form-group {{ $errors->has('date_until') ? 'has-error' : ''}}">
<label for="date_until" control-label>{{ 'Until date (filters inspections/measurements)' }}</label>
<label for="date_until" control-label>{{ 'Until date (filters inspections/measurements/weather data)' }}</label>
<div>
<input class="form-control" name="date_until" type="date" id="date_until" min="{{substr($research->start_date, 0, 10)}}" max="{{substr($research->end_date, 0, 10)}}" value="{{ isset($date_until) ? substr($date_until, 0, 10) : '' }}" >
{!! $errors->first('date_until', '<p class="help-block">:message</p>') !!}
Expand Down Expand Up @@ -83,6 +83,9 @@
<tr>
<th class="row-header"><span><i class="fa fa-2x fa-line-chart"></i> Measurements ({{ $totals['measurements'] }})</span></th>
</tr>
<tr>
<th class="row-header"><span><i class="fa fa-2x fa-thermometer"></i> Weather data ({{ $totals['weather'] }})</span></th>
</tr>
</tbody>
</table>
</div>
Expand Down Expand Up @@ -126,6 +129,11 @@
<td>{{ $d['measurements'] > 0 ? $d['measurements'] : '' }}</td>
@endforeach
</tr>
<tr>
@foreach($dates as $date => $d)
<td>{{ $d['weather'] > 0 ? $d['weather'] : '' }}</td>
@endforeach
</tr>
</tbody>
</table>
</div>
Expand Down Expand Up @@ -162,8 +170,8 @@
<a href="{{$download_url}}" target="_blank"><i class="fa fa-download"></i> Download selected consent data set</a>
<div style="display:block; height: 10px;"></div>
@if(count($sensor_urls) > 0)
<h4>Sensor data</h4>
<p>Export files are saved per device per consent period.
<h4>Sensor and weather data</h4>
<p>Export files are saved per device/location per consent period.
All data per device in the highest possible resolution as comma separated (,) .csv file that you can open in Excel, or SPSS.
<br>
<em>NB: The date time data in the 'time' column is in GMT time (this differs from what you see in the BEEP app), formatted by the RFC 3339 date-time standard.</em>
Expand Down