Skip to content

Commit

Permalink
Separate multi-value fields into columns
Browse files Browse the repository at this point in the history
  • Loading branch information
@jasonmule
jasonmule committed Apr 19, 2016
1 parent 9a8f43a commit 70f87b1
Show file tree
Hide file tree
Showing 6 changed files with 144 additions and 114 deletions.
154 changes: 82 additions & 72 deletions application/classes/Ushahidi/Formatter/Post/CSV.php
View file
Expand Up @@ -22,87 +22,131 @@ class Ushahidi_Formatter_Post_CSV implements Formatter
// Formatter
public function __invoke($records)
{
return $this->generateCSVRecords($records);
$this->generateCSVRecords($records);
}

/**
* Generates records that are suitable to save in CSV format.
* Records are padded with missing column headings as keys.
*
* Since search records will have mixed forms, rows that
* do not have a matching form field will be padded.
*
* @param array $records
*
* @return array
*/
protected function generateCSVRecords($records)
{
$csv_records = [];

// Get CSV heading
$heading = $this->getCSVHeading($records);

// Sort the columns from the heading so that they match with the record keys
sort($heading);

// Create filename from deployment name
$site_name = Kohana::$config->load('site.name');
$filename = $site_name.'.csv';

// Send response as CSV download
header('Content-Type: text/csv; charset=utf-8');
header('Content-Disposition: attachment; filename='.$filename);
header('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate');

$fp = fopen('php://output', 'w');

// Add heading
array_push($csv_records, $heading);
fputcsv($fp, $heading);

foreach ($records as $record)
{
$record = $record->asArray();

foreach ($record as $key => $val)
{
// Are these form values?
if ($key === 'values')
// Assign form values
if ($key == 'values')
{
// Remove 'values' column
unset($record['values']);
unset($record[$key]);

foreach ($val as $key => $val)
{
// XXX: Is this always a single value array?
$val = $val[0];

// Is it a location?
if ($this->isLocation($val))
{
// then create separate lat and lon fields
$record[$key.'.lat'] = $val['lat'];
$record[$key.'.lon'] = $val['lon'];
}

// else assign value as single string or csv string
else {
$record[$key] = $this->valueToString($val);
}
$this->assignRowValue($record, $key, $val[0]);
}
}

// If not form values then assign value as single string or CSV string
// Assign post values
else
{
$record[$key] = $this->valueToString($val);
unset($record[$key]);
$this->assignRowValue($record, $key, $val);
}
}

// Pad record with missing column headings as keys
// Pad record
$missing_keys = array_diff($heading, array_keys($record));
$record = array_merge($record, array_fill_keys($missing_keys, null));

// Sort the keys so that they match with columns from the CSV heading
ksort($record);
array_push($csv_records, $record);

fputcsv($fp, $record);
}

return $csv_records;
fclose($fp);

// No need for further processing
exit;
}

private function assignRowValue(&$record, $key, $value)
{
if (is_array($value))
{
// Assign in multiple columns
foreach ($value as $sub_key => $sub_value)
{
$record[$key.'.'.$sub_key] = $sub_value;
}
}

// ... else assign value as single string
else
{
$record[$key] = $value;
}
}

private function assignColumnHeading(&$columns, $key, $value)
{
if (is_array($value))
{
// Assign in multiple columns
foreach ($value as $sub_key => $sub_value)
{
$multivalue_key = $key.'.'.$sub_key;

if (! in_array($multivalue_key, $columns))
{
$columns[] = $multivalue_key;
}
}
}

// ... else assign single key
else
{
if (! in_array($key, $columns))
{
$columns[] = $key;
}
}
}

/**
* Extracts column names shared across posts to create a CSV heading
*
* @param array $records
*
*
* @return array
*/
protected function getCSVHeading($records)
Expand All @@ -116,58 +160,24 @@ protected function getCSVHeading($records)

foreach ($record as $key => $val)
{
// Are these form values?
if ($key === 'values')
// Assign form keys
if ($key == 'values')
{
foreach ($val as $key => $val)
{
// Get value from single value array
$val = $val[0];

// Is it a location?
if ($this->isLocation($val))
{
// then create separate lat and lon columns
array_push($columns, $key.'.lat', $key.'.lon');
}

// ...else add it as single column
else
{
array_push($columns, $key);
}
$this->assignColumnHeading($columns, $key, $val[0]);
}
}

// ...else add the key as is if not a form value key
// Assign post keys
else
{
array_push($columns, $key);
$this->assignColumnHeading($columns, $key, $val);
}
}
}

// Finally, return a list of unique column names found in all posts
return array_unique($columns);
}

/**
* Converts post values to strings
*
* @param mixed $value
*
* @return string
*/

protected function valueToString($value)
{
// Convert array to csv string
if (is_array($value)) {
return implode(',', $value);
}

// or return value as string
return (string) $value;
return $columns;
}

/**
Expand Down
78 changes: 43 additions & 35 deletions application/classes/Ushahidi/Transformer/CSVPostTransformer.php
View file
Expand Up @@ -11,7 +11,7 @@

use Ushahidi\Core\Tool\MappingTransformer;
use Ushahidi\Core\Entity\PostRepository;

class Ushahidi_Transformer_CSVPostTransformer implements MappingTransformer
{
protected $map;
Expand Down Expand Up @@ -53,24 +53,38 @@ public function interact(Array $record)
// Remap record columns
$record = array_combine($columns, $record);

// Trim
$record = array_map('trim', $record);
// Trim and remove empty values
foreach ($record as $key => $val)
{
$record[$key] = trim($val);

if (empty($record[$key])) {
unset($record[$key]);
}
}

// Merge multi-value columns
$this->mergeMultiValueFields($record);

// Filter post fields from the record
$post_entity = $this->repo->getEntity();
$post_fields = array_intersect_key($record, $post_entity->asArray());

// Remove post fields from the record and leave form values
foreach ($post_fields as $key => $val) {
foreach ($post_fields as $key => $val)
{
unset($record[$key]);
}

// Generate location point if any
$record = $this->mergeLocationCoordinates($record);

// Put values in array
array_walk($record, function (&$val) {
$val = [$val];
if ($this->isLocation($val)) {
$val = [$val];
}

if (! is_array($val)) {
$val = [$val];
}
});

$form_values = ['values' => $record];
Expand All @@ -81,44 +95,38 @@ public function interact(Array $record)
}

/**
* Merge location coordinates in the record
*
* We expect that coordinates are mapped to column.lat
* and column.lon for latitude and longitude respectively.
* Multi-value columns use dot notation to add sub-keys
* e.g. 'location.lat' refers to a field called 'location'
* and 'lat' is a sub-key of the field.
*
* @param Array $record
* @return Array
* @param array &$record
*/
private function mergeLocationCoordinates($record)
private function mergeMultiValueFields(&$record)
{
$locations = [];
$location_field = '';

// Get location point
foreach ($record as $column => $val)
{
// Look for latitude 'lat'
if (preg_match('/\.lat$/', $column)) {
// Get location field name
$location_field = explode('.', $column)[0];
$keys = explode('.', $column);

$locations[$location_field]['lat'] = $val;
// Get column name
$column_name = array_shift($keys);

// Remove from record
// Assign sub-key to multi-value column
if (! empty($keys))
{
unset($record[$column]);
}

// Look for longitude 'lon'
elseif (preg_match('/\.lon$/', $column)) {
// Get location field name
$location_field = explode('.', $column)[0];

$locations[$location_field]['lon'] = $val;

unset($record[$column]);
foreach ($keys as $key)
{
$record[$column_name][$key] = $val;
}
}
}
}

return array_merge($locations, $record);
private function isLocation($value)
{
return is_array($value) &&
array_key_exists('lon', $value) &&
array_key_exists('lat', $value);
}
}
12 changes: 12 additions & 0 deletions application/tests/datasets/ushahidi/Base.yml
View file
Expand Up @@ -225,6 +225,18 @@ form_attributes:
priority: 5
cardinality: 0
form_stage_id: 2
-
id: 14
label: "Possible actions"
key: "possible_actions"
type: "varchar"
input: "checkbox"
required: 0
options: '["ground_search","medical_evacuation"]'
priority: 5
cardinality: 0
config: '[]'
form_stage_id: 1
posts:
-
id: 1
Expand Down
6 changes: 3 additions & 3 deletions application/tests/datasets/ushahidi/sample.csv
View file
@@ -1,3 +1,3 @@
title, name, date, location, details, lat, lon
Missing person, Joseph Pata, 2015-11-20, Maputo, Last seen in Central Market, 33.755, -84.39
Missing person, Yusuf Hajj, 2015-08-20, Kampala, last seen in the mall, -1.2921, 36.8219
title, name, date, location, details, lat, lon, actions.0, actions.1
Missing person, Joseph Pata, 2015-11-20, Maputo, Last seen in Central Market, 33.755, -84.39, ground_search, medical_evacuation
Missing person, Yusuf Hajj, 2015-08-20, Kampala, last seen in the mall, -1.2921, 36.8219, medical_evacuation,
4 changes: 2 additions & 2 deletions application/tests/features/api.csv.feature
View file
Expand Up @@ -16,8 +16,8 @@ Feature: Testing the CSV API
And that the request "data" is:
"""
{
"columns":["title", "name", "date", "location", "details", "lat", "lon"],
"maps_to":["title", "full_name", null, "last_location", null, "last_location_point.lat", "last_location_point.lon"],
"columns":["title", "name", "date", "location", "details", "lat", "lon", "actions"],
"maps_to":["title", "full_name", null, "last_location", null, "last_location_point.lat", "last_location_point.lon", "possible_actions.0", "possible_actions.1"],
"fixed":
{
"form":1,
Expand Down

0 comments on commit 70f87b1

Please sign in to comment.