diff --git a/.eslintrc b/.eslintrc index 319c266e..a752cd56 100644 --- a/.eslintrc +++ b/.eslintrc @@ -10,6 +10,10 @@ env: parserOptions: sourceType: script rules: + max-len: + - error + - code: 100 + ignoreStrings: true strict: - error - safe diff --git a/README.md b/README.md index ec87f5a4..6116c221 100644 --- a/README.md +++ b/README.md @@ -115,4 +115,3 @@ createdb opendatacensus_test npm test ``` - diff --git a/census/controllers/api.js b/census/controllers/api.js index 3e2dea10..fcf86030 100644 --- a/census/controllers/api.js +++ b/census/controllers/api.js @@ -291,6 +291,174 @@ let _outputEntryResults = function(results, questions, format, res) { } }; +let _outputFlatEntryResults = function(results, questions, format, res) { + + // Count + let maxFormats = 0 + let maxCharacts = 0 + let maxLocations = 0 + for (const item of results) { + let answers = item.getSimpleAnswersForQuestions(questions); + let getAnswerById = id => answers.find(answer => answer.id === id) || {} + maxFormats = Math.max(maxFormats, (getAnswerById('format').value || []).length) + maxCharacts = Math.max(maxCharacts, (getAnswerById('characteristics').value || []).length) + maxLocations = Math.max(maxLocations, (getAnswerById('location').value || []).length) + } + + // Mapper + let mapper = function(item) { + let answers = item.getSimpleAnswersForQuestions(questions); + let getAnswerById = id => answers.find(answer => answer.id === id) || {} + let result = {} + + // General + Object.assign(result, { + 'id': item.id, + 'site': item.site, + 'timestamp': moment(item.createdAt).format('YYYY-MM-DDTHH:mm:ss'), + 'year': item.year, + 'place': item.place, + 'dataset': item.dataset, + }) + + // License + let openLicence = getAnswerById('open_licence') + let licenceURL = getAnswerById('licence_url') + Object.assign(result, { + 'Openly licenced?': openLicence.value, + 'Comments licence': openLicence.commentValue, + 'Licence URL': licenceURL.value, + 'Comments Licence URL': licenceURL.commentValue, + }) + + // Publicly available + let onlineFree = getAnswerById('online_free') + Object.assign(result, { + 'Publicly available online (no access controls)': onlineFree.value, + 'Comments publicly available': onlineFree.commentValue, + }) + + // Online otherwise + let onlineOtherwise = getAnswerById('online_otherwise') + Object.assign(result, { + 'Online_otherwise?': onlineOtherwise.value, + 'Comments online_otherwise': onlineOtherwise.commentValue, + }) + + // Collector gov + let collectorGov = getAnswerById('collector_gov') + let collectorName = getAnswerById('collector_name') + Object.assign(result, { + 'Collector_Gov': collectorGov.value, + 'Comment_gov collector': collectorGov.commentValue, + 'Collector name': collectorName.value, + 'Comments collector name': collectorName.commentValue, + }) + + // Collector non-gov + let collectorNonGov = getAnswerById('collector_non_gov') + Object.assign(result, { + 'Collector_non-gov': collectorNonGov.value, + 'Comment_non-gov': collectorNonGov.commentValue, + }) + + // Findable + let findable = getAnswerById('findable') + let findableSteps = getAnswerById('findable_steps') + Object.assign(result, { + 'Findable': findable.value, + 'Findable_comment': findable.commentValue, + 'Findable_steps taken': findableSteps.value, + 'Comments_steps taken': findableSteps.commentValue, + }) + + // Timely + let timely = getAnswerById('timely') + Object.assign(result, { + 'Timely': timely.value, + 'Timely_comment': timely.commentValue, + }) + + // Free + let free = getAnswerById('free') + Object.assign(result, { + 'Free': free.value, + 'Free_comment': free.commentValue, + }) + + // Usability + let usability = getAnswerById('usability') + Object.assign(result, { + 'Usability': usability.value, + 'Usability_comment': usability.commentValue, + }) + + // Bulk + let bulk = getAnswerById('bulk') + Object.assign(result, { + 'Downloadable at once': bulk.value, + 'Downloadable at once comment': bulk.commentValue, + }) + + // Format + let format = getAnswerById('format') + for (const index of _.range(maxFormats)) { + result[`Format ${index + 1}`] = format.value[index] + } + result['Format comment'] = format.commentValue + + // Characteristics + let characts = getAnswerById('characteristics') + for (const index of _.range(maxCharacts)) { + result[`Data element ${index + 1}`] = characts.value[index] + } + result['Data element comment'] = characts.commentValue + + // Location + let location = getAnswerById('location') + for (const index of _.range(maxLocations)) { + const data = location.value[index] + result[`URL ${index + 1}`] = data ? data.urlValue : '' + result[`Description URL ${index + 1}`] = data ? data.descValue : '' + } + result['URL comment'] = location.commentValue + + // Metadata + Object.assign(result, { + 'reviewed': item.reviewed ? 'Yes' : 'No', + 'reviewResult': item.reviewResult ? 'Yes' : 'No', + 'reviewComments': item.reviewComments, + 'details': item.details, + 'isCurrent': item.isCurrent ? 'Yes' : 'No', + 'isOpen': item.isOpenForQuestions(questions) ? 'Yes' : 'No', + 'submitter': item.Submitter ? item.Submitter.fullName() : '', + 'reviewer': item.Reviewer ? item.Reviewer.fullName() : '', + 'score': item.computedScore, + 'relativeScore': item.computedRelativeScore + }) + + return result + }; + + // Response + switch (format) { + case 'json': { + outputItemsAsJson(res, results, mapper); + break; + } + case 'csv': { + const columns = (results.length) ? Object.keys(mapper(results[0])) : [] + outputItemsAsCsv(res, results, mapper, columns); + break; + } + default: { + res.sendStatus(404); + break; + } + } + +}; + let pendingEntries = function(req, res, next) { const format = req.params.format; @@ -317,6 +485,7 @@ let entries = function(req, res, next) { // Get request params const format = req.params.format; const strategy = req.params.strategy; + const modifier = req.params.modifier; // Initial data options let dataOptions = _.merge(modelUtils.getDataOptions(req), { @@ -339,11 +508,21 @@ let entries = function(req, res, next) { return res.sendStatus(404); } + // Modifier can be only `flat` or `undefined` + let outputFunction + if (modifier === 'flat') { + outputFunction = _outputFlatEntryResults + } else if (modifier === undefined) { + outputFunction = _outputEntryResults + } else { + return res.sendStatus(404); + } + // Make request for data, return it modelUtils.getData(dataOptions).then(function(data) { const results = data.entries; const questions = data.questions; - _outputEntryResults(results, questions, format, res); + outputFunction(results, questions, format, res); }).catch(console.trace.bind(console)); }; diff --git a/census/routes/api.js b/census/routes/api.js index b9918749..77afc9a2 100644 --- a/census/routes/api.js +++ b/census/routes/api.js @@ -11,6 +11,8 @@ var apiRoutes = function(coreMiddlewares) { router.use(coreMiddlewares); + router.get(utils.scoped('/entries.:strategy.:modifier.:format'), coreMixins, + api.entries); router.get(utils.scoped('/entries.:strategy.:format'), coreMixins, api.entries); router.get(utils.scoped('/entries/:year.:strategy.:format'), coreMixins, diff --git a/census/views/base.html b/census/views/base.html index 8055f5ac..f05fe0a3 100644 --- a/census/views/base.html +++ b/census/views/base.html @@ -195,6 +195,7 @@ {% if not is_index %}

{{ gettext("Download") }}: {{ gettext("Current (CSV)") }} | + {{ gettext("Current (Flat CSV)") }} | {{ gettext("All (CSV)") }} | {{ gettext("Current (JSON)") }} | {{ gettext("All (JSON)") }} diff --git a/settings.json.example b/settings.json.example index 248b9b89..e85faaab 100644 --- a/settings.json.example +++ b/settings.json.example @@ -23,6 +23,7 @@ "app_id": "{FACEBOOK_APP_ID}", "app_secret": "{FACEBOOK_APP_SECRET}" }, + "fallback_questionset": "https://docs.google.com/spreadsheets/d/1UKpnHBqwKfpZCTRPTTwpYHE3Dx6QV7M8uWmEeDTNWPM/edit#gid=1999457872", "auth_subdomain": "id", "system_subdomain": "system", "sentry_dsn": "", diff --git a/tests/api.js b/tests/api.js index 0f6c1b19..eda72b5c 100644 --- a/tests/api.js +++ b/tests/api.js @@ -81,6 +81,14 @@ describe('API', function() { }); }); + it('Current cascaded flat', done => { + let browser = testUtils.browser; + browser.visit('/api/entries.cascade.flat.' + format, () => { + checkResponse(browser, 4); + done(); + }); + }); + it('All current, year: ' + year, done => { let browser = testUtils.browser; browser.visit('/api/entries/' + year + '.' + format, () => { @@ -137,6 +145,21 @@ describe('API', function() { }); }); + describe('Flat entries (data)', function() { + it('Cascade current', done => { + let browser = testUtils.browser; + browser.visit('/api/entries.cascade.flat.json', () => { + browser.assert.success(); + const data = JSON.parse(browser.text()); + const item = data.results[0]; + assert.deepEqual(item['Format 1'], 'AsciiDoc') + assert.deepEqual(item['Format 2'], 'CSV') + assert.deepEqual(item['Format 3'], 'HTML') + done(); + }); + }); + }); + describe('Places', function() { _.forEach(_.keys(responseFormats), format => { let checkResponse = responseFormats[format];