From 70f87b1d55b6d8a50e0ca25469759ee2f1fd3141 Mon Sep 17 00:00:00 2001 From: Jason Mule Date: Tue, 19 Apr 2016 12:10:09 +0300 Subject: [PATCH] Separate multi-value fields into columns --- .../classes/Ushahidi/Formatter/Post/CSV.php | 154 ++++++++++-------- .../Transformer/CSVPostTransformer.php | 78 +++++---- application/tests/datasets/ushahidi/Base.yml | 12 ++ .../tests/datasets/ushahidi/sample.csv | 6 +- application/tests/features/api.csv.feature | 4 +- .../features/forms/api.attributes.feature | 4 +- 6 files changed, 144 insertions(+), 114 deletions(-) diff --git a/application/classes/Ushahidi/Formatter/Post/CSV.php b/application/classes/Ushahidi/Formatter/Post/CSV.php index e516189c26..4ee2bd1f00 100644 --- a/application/classes/Ushahidi/Formatter/Post/CSV.php +++ b/application/classes/Ushahidi/Formatter/Post/CSV.php @@ -22,12 +22,14 @@ class Ushahidi_Formatter_Post_CSV implements Formatter // Formatter public function __invoke($records) { - return $this->generateCSVRecords($records); + $this->generateCSVRecords($records); } /** * Generates records that are suitable to save in CSV format. - * Records are padded with missing column headings as keys. + * + * Since search records will have mixed forms, rows that + * do not have a matching form field will be padded. * * @param array $records * @@ -35,16 +37,25 @@ public function __invoke($records) */ protected function generateCSVRecords($records) { - $csv_records = []; - // Get CSV heading $heading = $this->getCSVHeading($records); - + // Sort the columns from the heading so that they match with the record keys sort($heading); + // Create filename from deployment name + $site_name = Kohana::$config->load('site.name'); + $filename = $site_name.'.csv'; + + // Send response as CSV download + header('Content-Type: text/csv; charset=utf-8'); + header('Content-Disposition: attachment; filename='.$filename); + header('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'); + + $fp = fopen('php://output', 'w'); + // Add heading - array_push($csv_records, $heading); + fputcsv($fp, $heading); foreach ($records as $record) { @@ -52,57 +63,90 @@ protected function generateCSVRecords($records) foreach ($record as $key => $val) { - // Are these form values? - if ($key === 'values') + // Assign form values + if ($key == 'values') { - // Remove 'values' column - unset($record['values']); + unset($record[$key]); foreach ($val as $key => $val) { - // XXX: Is this always a single value array? - $val = $val[0]; - - // Is it a location? - if ($this->isLocation($val)) - { - // then create separate lat and lon fields - $record[$key.'.lat'] = $val['lat']; - $record[$key.'.lon'] = $val['lon']; - } - - // else assign value as single string or csv string - else { - $record[$key] = $this->valueToString($val); - } + $this->assignRowValue($record, $key, $val[0]); } } - // If not form values then assign value as single string or CSV string + // Assign post values else { - $record[$key] = $this->valueToString($val); + unset($record[$key]); + $this->assignRowValue($record, $key, $val); } } - // Pad record with missing column headings as keys + // Pad record $missing_keys = array_diff($heading, array_keys($record)); $record = array_merge($record, array_fill_keys($missing_keys, null)); // Sort the keys so that they match with columns from the CSV heading ksort($record); - - array_push($csv_records, $record); + + fputcsv($fp, $record); } - return $csv_records; + fclose($fp); + + // No need for further processing + exit; + } + + private function assignRowValue(&$record, $key, $value) + { + if (is_array($value)) + { + // Assign in multiple columns + foreach ($value as $sub_key => $sub_value) + { + $record[$key.'.'.$sub_key] = $sub_value; + } + } + + // ... else assign value as single string + else + { + $record[$key] = $value; + } + } + + private function assignColumnHeading(&$columns, $key, $value) + { + if (is_array($value)) + { + // Assign in multiple columns + foreach ($value as $sub_key => $sub_value) + { + $multivalue_key = $key.'.'.$sub_key; + + if (! in_array($multivalue_key, $columns)) + { + $columns[] = $multivalue_key; + } + } + } + + // ... else assign single key + else + { + if (! in_array($key, $columns)) + { + $columns[] = $key; + } + } } /** * Extracts column names shared across posts to create a CSV heading * * @param array $records - * + * * @return array */ protected function getCSVHeading($records) @@ -116,58 +160,24 @@ protected function getCSVHeading($records) foreach ($record as $key => $val) { - // Are these form values? - if ($key === 'values') + // Assign form keys + if ($key == 'values') { foreach ($val as $key => $val) { - // Get value from single value array - $val = $val[0]; - - // Is it a location? - if ($this->isLocation($val)) - { - // then create separate lat and lon columns - array_push($columns, $key.'.lat', $key.'.lon'); - } - - // ...else add it as single column - else - { - array_push($columns, $key); - } + $this->assignColumnHeading($columns, $key, $val[0]); } } - // ...else add the key as is if not a form value key + // Assign post keys else { - array_push($columns, $key); + $this->assignColumnHeading($columns, $key, $val); } } } - // Finally, return a list of unique column names found in all posts - return array_unique($columns); - } - - /** - * Converts post values to strings - * - * @param mixed $value - * - * @return string - */ - - protected function valueToString($value) - { - // Convert array to csv string - if (is_array($value)) { - return implode(',', $value); - } - - // or return value as string - return (string) $value; + return $columns; } /** diff --git a/application/classes/Ushahidi/Transformer/CSVPostTransformer.php b/application/classes/Ushahidi/Transformer/CSVPostTransformer.php index f83cc5fbe9..2629df8a7b 100644 --- a/application/classes/Ushahidi/Transformer/CSVPostTransformer.php +++ b/application/classes/Ushahidi/Transformer/CSVPostTransformer.php @@ -11,7 +11,7 @@ use Ushahidi\Core\Tool\MappingTransformer; use Ushahidi\Core\Entity\PostRepository; - + class Ushahidi_Transformer_CSVPostTransformer implements MappingTransformer { protected $map; @@ -53,24 +53,38 @@ public function interact(Array $record) // Remap record columns $record = array_combine($columns, $record); - // Trim - $record = array_map('trim', $record); + // Trim and remove empty values + foreach ($record as $key => $val) + { + $record[$key] = trim($val); + + if (empty($record[$key])) { + unset($record[$key]); + } + } + + // Merge multi-value columns + $this->mergeMultiValueFields($record); // Filter post fields from the record $post_entity = $this->repo->getEntity(); $post_fields = array_intersect_key($record, $post_entity->asArray()); // Remove post fields from the record and leave form values - foreach ($post_fields as $key => $val) { + foreach ($post_fields as $key => $val) + { unset($record[$key]); } - // Generate location point if any - $record = $this->mergeLocationCoordinates($record); - // Put values in array array_walk($record, function (&$val) { - $val = [$val]; + if ($this->isLocation($val)) { + $val = [$val]; + } + + if (! is_array($val)) { + $val = [$val]; + } }); $form_values = ['values' => $record]; @@ -81,44 +95,38 @@ public function interact(Array $record) } /** - * Merge location coordinates in the record - * - * We expect that coordinates are mapped to column.lat - * and column.lon for latitude and longitude respectively. + * Multi-value columns use dot notation to add sub-keys + * e.g. 'location.lat' refers to a field called 'location' + * and 'lat' is a sub-key of the field. * - * @param Array $record - * @return Array + * @param array &$record */ - private function mergeLocationCoordinates($record) + private function mergeMultiValueFields(&$record) { - $locations = []; - $location_field = ''; - - // Get location point foreach ($record as $column => $val) { - // Look for latitude 'lat' - if (preg_match('/\.lat$/', $column)) { - // Get location field name - $location_field = explode('.', $column)[0]; + $keys = explode('.', $column); - $locations[$location_field]['lat'] = $val; + // Get column name + $column_name = array_shift($keys); - // Remove from record + // Assign sub-key to multi-value column + if (! empty($keys)) + { unset($record[$column]); - } - - // Look for longitude 'lon' - elseif (preg_match('/\.lon$/', $column)) { - // Get location field name - $location_field = explode('.', $column)[0]; - $locations[$location_field]['lon'] = $val; - - unset($record[$column]); + foreach ($keys as $key) + { + $record[$column_name][$key] = $val; + } } } + } - return array_merge($locations, $record); + private function isLocation($value) + { + return is_array($value) && + array_key_exists('lon', $value) && + array_key_exists('lat', $value); } } diff --git a/application/tests/datasets/ushahidi/Base.yml b/application/tests/datasets/ushahidi/Base.yml index ebaf7e36c3..963659f1be 100644 --- a/application/tests/datasets/ushahidi/Base.yml +++ b/application/tests/datasets/ushahidi/Base.yml @@ -225,6 +225,18 @@ form_attributes: priority: 5 cardinality: 0 form_stage_id: 2 + - + id: 14 + label: "Possible actions" + key: "possible_actions" + type: "varchar" + input: "checkbox" + required: 0 + options: '["ground_search","medical_evacuation"]' + priority: 5 + cardinality: 0 + config: '[]' + form_stage_id: 1 posts: - id: 1 diff --git a/application/tests/datasets/ushahidi/sample.csv b/application/tests/datasets/ushahidi/sample.csv index 270ad48ff6..9a34339583 100644 --- a/application/tests/datasets/ushahidi/sample.csv +++ b/application/tests/datasets/ushahidi/sample.csv @@ -1,3 +1,3 @@ -title, name, date, location, details, lat, lon -Missing person, Joseph Pata, 2015-11-20, Maputo, Last seen in Central Market, 33.755, -84.39 -Missing person, Yusuf Hajj, 2015-08-20, Kampala, last seen in the mall, -1.2921, 36.8219 +title, name, date, location, details, lat, lon, actions.0, actions.1 +Missing person, Joseph Pata, 2015-11-20, Maputo, Last seen in Central Market, 33.755, -84.39, ground_search, medical_evacuation +Missing person, Yusuf Hajj, 2015-08-20, Kampala, last seen in the mall, -1.2921, 36.8219, medical_evacuation, diff --git a/application/tests/features/api.csv.feature b/application/tests/features/api.csv.feature index 14af5cf082..446aed9834 100644 --- a/application/tests/features/api.csv.feature +++ b/application/tests/features/api.csv.feature @@ -16,8 +16,8 @@ Feature: Testing the CSV API And that the request "data" is: """ { - "columns":["title", "name", "date", "location", "details", "lat", "lon"], - "maps_to":["title", "full_name", null, "last_location", null, "last_location_point.lat", "last_location_point.lon"], + "columns":["title", "name", "date", "location", "details", "lat", "lon", "actions"], + "maps_to":["title", "full_name", null, "last_location", null, "last_location_point.lat", "last_location_point.lon", "possible_actions.0", "possible_actions.1"], "fixed": { "form":1, diff --git a/application/tests/features/forms/api.attributes.feature b/application/tests/features/forms/api.attributes.feature index 771db2a806..0359e460e7 100644 --- a/application/tests/features/forms/api.attributes.feature +++ b/application/tests/features/forms/api.attributes.feature @@ -176,7 +176,7 @@ Feature: Testing the Form Attributes API Then the response is JSON And the response has a "count" property And the type of the "count" property is "numeric" - And the "count" property equals "15" + And the "count" property equals "16" Then the guzzle status code should be 200 Scenario: Listing All Attributes @@ -185,7 +185,7 @@ Feature: Testing the Form Attributes API Then the response is JSON And the response has a "count" property And the type of the "count" property is "numeric" - And the "count" property equals "15" + And the "count" property equals "16" Then the guzzle status code should be 200 Scenario: Search for point attributes