executable: add support for passing options to --simplify either as JSON or key=value

Author: maxlath

docs/how_to.md

@@ -131,9 +131,16 @@ cat entities.json | wikidata-filter --languages en,fr,de,zh,eo > subset.ndjson
 ```
 
 ### Simplify entity data
-Uses [wikidata-sdk `simplify.entity` function](https://github.com/maxlath/wikidata-sdk#simplify-entity) to parse the labels, descriptions, aliases, claims, and sitelinks.
+Uses [wikidata-sdk `simplify.entity` function](https://github.com/maxlath/wikidata-sdk/blob/master/docs/simplify_entities_data.md) to parse the labels, descriptions, aliases, claims, and sitelinks.
 ```sh
-cat entities.json | wikidata-filter --simplified > simplified_dump.ndjson
+# Default simplify options
+cat entities.json | wikidata-filter --simplify > simplified_dump.ndjson
+# Custom options, see wdk.simplify.entity documentation https://github.com/maxlath/wikidata-sdk/blob/master/docs/simplify_entities_data.md
+# and specifically for claims options, see https://github.com/maxlath/wikidata-sdk/blob/master/docs/simplify_claims.md#options
+cat entities.json | wikidata-filter --simplify '{"keepRichValues":"true","keepQualifiers":"true","keepReferences":"true"}' > simplified_dump.ndjson
+# The options can also be passed in a lighter, urlencoded-like, key=value format
+# that's simpler than typing all those JSON double quotes
+cat entities.json | wikidata-filter --simplify 'keepRichValues=true&keepQualifiers=true&keepReferences=true' > simplified_dump.ndjson
 ```
 
 ## Other options
@@ -194,5 +201,5 @@ The [equivalent SPARQL query](https://query.wikidata.org/#SELECT%20%3Fs%20%3FsLa
 
 ```sh
 DUMP='https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.json.gz'
-curl $DUMP | gzip -d | wikidata-filter --sitelink 'zhwiki&frwiki' --keep id,labels,sitelinks --languages zh,fr --simplified > subset.ndjson
+curl $DUMP | gzip -d | wikidata-filter --sitelink 'zhwiki&frwiki' --keep id,labels,sitelinks --languages zh,fr --simplify > subset.ndjson
 ```

lib/get_format_data.js

@@ -3,6 +3,7 @@ const difference = require('lodash.difference')
 const isPlainObject = require('lodash.isplainobject')
 const lists = require('./lists')
 const validate = require('./validate_arguments')
+const querystring = require('querystring')
 
 module.exports = (options) => {
   var { omit, languages, simplify } = options
@@ -16,6 +17,10 @@ module.exports = (options) => {
 
   var simplifyOptions
   if (simplify) {
+    if (typeof simplify === 'string') {
+      if (simplify.match(/=/)) simplify = querystring.parse(simplify)
+      else if (simplify[0] === '{') simplify = JSON.parse(simplify)
+    }
     if (isPlainObject(simplify)) {
       simplifyOptions = simplify
       simplify = true

lib/program.js

@@ -12,7 +12,7 @@ module.exports = function () {
   .option('-o, --omit <attributes>', help.omit, list)
   .option('-k, --keep <attributes>', help.keep, list)
   .option('-l, --languages <languages>', help.languages, list)
-  .option('-s, --simplified', help.simplified)
+  .option('-s, --simplify <boolean|options>', help.simplify)
   // misc
   .option('-p, --progress <boolean>', help.progress)
   .parse(process.argv)
@@ -47,8 +47,9 @@ const help = {
     Specify for which languages labels, descriptions and aliases should be kept.
     `,
 
-  simplified: `
+  simplify: `
     Flag to simplify claims values. Defaults to false.
+    Can also accept wikidata-sdk simplify.entity option object as JSON or key=values: wikidata-filter --simplify 'keepQualifiers=true&keepRichValues=true'
     `,
 
   progress: `

test/integration.js

@@ -1,4 +1,4 @@
-require('should')
+const should = require('should')
 const { exec } = require('child_process')
 
 describe('integration', () => {
@@ -12,4 +12,28 @@ describe('integration', () => {
       done()
     })
   })
+
+  it('should parse json simplify options', done => {
+    exec(`cat ./test/fixtures/Q22.json | ./bin/wikidata-filter -s '{"keepQualifiers":true}'`, (err, stdout, stderr) => {
+      if (err) return done(err)
+      const entity = JSON.parse(stdout)
+      entity.id.should.equal('Q22')
+      should(entity.claims.P1549[2].mainsnak).not.be.ok()
+      entity.claims.P1549[2].qualifiers.should.be.an.Object()
+      entity.claims.P1549[2].qualifiers.P518.should.be.an.Object()
+      done()
+    })
+  })
+
+  it('should parse url-encoded simplify options', done => {
+    exec('cat ./test/fixtures/Q22.json | ./bin/wikidata-filter -s "keepQualifiers=true"', (err, stdout, stderr) => {
+      if (err) return done(err)
+      const entity = JSON.parse(stdout)
+      entity.id.should.equal('Q22')
+      should(entity.claims.P1549[2].mainsnak).not.be.ok()
+      entity.claims.P1549[2].qualifiers.should.be.an.Object()
+      entity.claims.P1549[2].qualifiers.P518.should.be.an.Object()
+      done()
+    })
+  })
 })