Permalink
Browse files

Added flat CSV downloading option for Census (#1156)

* Started the work

* Added fallback_questionset to settings.json.example

* Added Current (Flat CSV) downloading link

* Added Current (Flat CSV) implementation

* Added Current (Flat CSV) tests
  • Loading branch information...
roll committed Jul 4, 2018
1 parent d7549c5 commit fb3ebb39e11608ad55ec565e5849b9cec793505f
Showing with 211 additions and 2 deletions.
  1. +4 −0 .eslintrc
  2. +0 −1 README.md
  3. +180 −1 census/controllers/api.js
  4. +2 −0 census/routes/api.js
  5. +1 −0 census/views/base.html
  6. +1 −0 settings.json.example
  7. +23 −0 tests/api.js
View
@@ -10,6 +10,10 @@ env:
parserOptions:
sourceType: script
rules:
max-len:
- error
- code: 100
ignoreStrings: true
strict:
- error
- safe
View
@@ -115,4 +115,3 @@ createdb opendatacensus_test
npm test
```
View
@@ -291,6 +291,174 @@ let _outputEntryResults = function(results, questions, format, res) {
}
};
let _outputFlatEntryResults = function(results, questions, format, res) {
// Count
let maxFormats = 0
let maxCharacts = 0
let maxLocations = 0
for (const item of results) {
let answers = item.getSimpleAnswersForQuestions(questions);
let getAnswerById = id => answers.find(answer => answer.id === id) || {}
maxFormats = Math.max(maxFormats, (getAnswerById('format').value || []).length)
maxCharacts = Math.max(maxCharacts, (getAnswerById('characteristics').value || []).length)
maxLocations = Math.max(maxLocations, (getAnswerById('location').value || []).length)
}
// Mapper
let mapper = function(item) {
let answers = item.getSimpleAnswersForQuestions(questions);
let getAnswerById = id => answers.find(answer => answer.id === id) || {}
let result = {}
// General
Object.assign(result, {
'id': item.id,
'site': item.site,
'timestamp': moment(item.createdAt).format('YYYY-MM-DDTHH:mm:ss'),
'year': item.year,
'place': item.place,
'dataset': item.dataset,
})
// License
let openLicence = getAnswerById('open_licence')
let licenceURL = getAnswerById('licence_url')
Object.assign(result, {
'Openly licenced?': openLicence.value,
'Comments licence': openLicence.commentValue,
'Licence URL': licenceURL.value,
'Comments Licence URL': licenceURL.commentValue,
})
// Publicly available
let onlineFree = getAnswerById('online_free')
Object.assign(result, {
'Publicly available online (no access controls)': onlineFree.value,
'Comments publicly available': onlineFree.commentValue,
})
// Online otherwise
let onlineOtherwise = getAnswerById('online_otherwise')
Object.assign(result, {
'Online_otherwise?': onlineOtherwise.value,
'Comments online_otherwise': onlineOtherwise.commentValue,
})
// Collector gov
let collectorGov = getAnswerById('collector_gov')
let collectorName = getAnswerById('collector_name')
Object.assign(result, {
'Collector_Gov': collectorGov.value,
'Comment_gov collector': collectorGov.commentValue,
'Collector name': collectorName.value,
'Comments collector name': collectorName.commentValue,
})
// Collector non-gov
let collectorNonGov = getAnswerById('collector_non_gov')
Object.assign(result, {
'Collector_non-gov': collectorNonGov.value,
'Comment_non-gov': collectorNonGov.commentValue,
})
// Findable
let findable = getAnswerById('findable')
let findableSteps = getAnswerById('findable_steps')
Object.assign(result, {
'Findable': findable.value,
'Findable_comment': findable.commentValue,
'Findable_steps taken': findableSteps.value,
'Comments_steps taken': findableSteps.commentValue,
})
// Timely
let timely = getAnswerById('timely')
Object.assign(result, {
'Timely': timely.value,
'Timely_comment': timely.commentValue,
})
// Free
let free = getAnswerById('free')
Object.assign(result, {
'Free': free.value,
'Free_comment': free.commentValue,
})
// Usability
let usability = getAnswerById('usability')
Object.assign(result, {
'Usability': usability.value,
'Usability_comment': usability.commentValue,
})
// Bulk
let bulk = getAnswerById('bulk')
Object.assign(result, {
'Downloadable at once': bulk.value,
'Downloadable at once comment': bulk.commentValue,
})
// Format
let format = getAnswerById('format')
for (const index of _.range(maxFormats)) {
result[`Format ${index + 1}`] = format.value[index]
}
result['Format comment'] = format.commentValue
// Characteristics
let characts = getAnswerById('characteristics')
for (const index of _.range(maxCharacts)) {
result[`Data element ${index + 1}`] = characts.value[index]
}
result['Data element comment'] = characts.commentValue
// Location
let location = getAnswerById('location')
for (const index of _.range(maxLocations)) {
const data = location.value[index]
result[`URL ${index + 1}`] = data ? data.urlValue : ''
result[`Description URL ${index + 1}`] = data ? data.descValue : ''
}
result['URL comment'] = location.commentValue
// Metadata
Object.assign(result, {
'reviewed': item.reviewed ? 'Yes' : 'No',
'reviewResult': item.reviewResult ? 'Yes' : 'No',
'reviewComments': item.reviewComments,
'details': item.details,
'isCurrent': item.isCurrent ? 'Yes' : 'No',
'isOpen': item.isOpenForQuestions(questions) ? 'Yes' : 'No',
'submitter': item.Submitter ? item.Submitter.fullName() : '',
'reviewer': item.Reviewer ? item.Reviewer.fullName() : '',
'score': item.computedScore,
'relativeScore': item.computedRelativeScore
})
return result
};
// Response
switch (format) {
case 'json': {
outputItemsAsJson(res, results, mapper);
break;
}
case 'csv': {
const columns = (results.length) ? Object.keys(mapper(results[0])) : []
outputItemsAsCsv(res, results, mapper, columns);
break;
}
default: {
res.sendStatus(404);
break;
}
}
};
let pendingEntries = function(req, res, next) {
const format = req.params.format;
@@ -317,6 +485,7 @@ let entries = function(req, res, next) {
// Get request params
const format = req.params.format;
const strategy = req.params.strategy;
const modifier = req.params.modifier;
// Initial data options
let dataOptions = _.merge(modelUtils.getDataOptions(req), {
@@ -339,11 +508,21 @@ let entries = function(req, res, next) {
return res.sendStatus(404);
}
// Modifier can be only `flat` or `undefined`
let outputFunction
if (modifier === 'flat') {
outputFunction = _outputFlatEntryResults
} else if (modifier === undefined) {
outputFunction = _outputEntryResults
} else {
return res.sendStatus(404);
}
// Make request for data, return it
modelUtils.getData(dataOptions).then(function(data) {
const results = data.entries;
const questions = data.questions;
_outputEntryResults(results, questions, format, res);
outputFunction(results, questions, format, res);
}).catch(console.trace.bind(console));
};
View
@@ -11,6 +11,8 @@ var apiRoutes = function(coreMiddlewares) {
router.use(coreMiddlewares);
router.get(utils.scoped('/entries.:strategy.:modifier.:format'), coreMixins,
api.entries);
router.get(utils.scoped('/entries.:strategy.:format'), coreMixins,
api.entries);
router.get(utils.scoped('/entries/:year.:strategy.:format'), coreMixins,
View
@@ -195,6 +195,7 @@
{% if not is_index %}
<p class="pull-right">
<strong>{{ gettext("Download") }}:</strong> <a href="/api/entries.cascade.csv">{{ gettext("Current (CSV)") }}</a> |
<a href="/api/entries.cascade.flat.csv">{{ gettext("Current (Flat CSV)") }}</a> |
<a href="/api/entries.csv">{{ gettext("All (CSV)") }}</a> |
<a href="/api/entries.cascade.json">{{ gettext("Current (JSON)") }}</a> |
<a href="/api/entries.json">{{ gettext("All (JSON)") }}</a>
View
@@ -23,6 +23,7 @@
"app_id": "{FACEBOOK_APP_ID}",
"app_secret": "{FACEBOOK_APP_SECRET}"
},
"fallback_questionset": "https://docs.google.com/spreadsheets/d/1UKpnHBqwKfpZCTRPTTwpYHE3Dx6QV7M8uWmEeDTNWPM/edit#gid=1999457872",
"auth_subdomain": "id",
"system_subdomain": "system",
"sentry_dsn": "",
View
@@ -81,6 +81,14 @@ describe('API', function() {
});
});
it('Current cascaded flat', done => {
let browser = testUtils.browser;
browser.visit('/api/entries.cascade.flat.' + format, () => {
checkResponse(browser, 4);
done();
});
});
it('All current, year: ' + year, done => {
let browser = testUtils.browser;
browser.visit('/api/entries/' + year + '.' + format, () => {
@@ -137,6 +145,21 @@ describe('API', function() {
});
});
describe('Flat entries (data)', function() {
it('Cascade current', done => {
let browser = testUtils.browser;
browser.visit('/api/entries.cascade.flat.json', () => {
browser.assert.success();
const data = JSON.parse(browser.text());
const item = data.results[0];
assert.deepEqual(item['Format 1'], 'AsciiDoc')
assert.deepEqual(item['Format 2'], 'CSV')
assert.deepEqual(item['Format 3'], 'HTML')
done();
});
});
});
describe('Places', function() {
_.forEach(_.keys(responseFormats), format => {
let checkResponse = responseFormats[format];

0 comments on commit fb3ebb3

Please sign in to comment.