claim filters: add support for logical operators

fix #17

Author: maxlath

docs/how_to.md

@@ -8,6 +8,7 @@
 
 - [Filter entities](#filter-entities)
   - [By claims](#by-claims)
+    - [claims logical operators](#claims-logical-operators)
     - [Long claim option](#long-claim-option)
   - [By sitelinks](#by-sitelinks)
   - [By type](#by-type)
@@ -45,6 +46,13 @@ this can be quite convinient when you don't have enough space to keep the whole
 
 Of course, **this probably only make sense if the kind of entities you are looking for is somewhere above 100 000 units(?)**, given that under this level, it would probably be faster/more efficient to get the list of ids from [Wikidata Query](http://query.wikidata.org/), then [get the entities data from the API](https://www.wikidata.org/w/api.php?action=help&modules=wbgetentities) ([wikidata-sdk](https://github.com/maxlath/wikidata-sdk#get-entities-by-id) can be helpful there).
 
+#### claims logical operators
+You can use logical operators:
+```sh
+// the claim filter is equivalent to (P31:Q571 && (P50 || P110))
+cat entities.json | wikidata-filter --claim 'P31:Q571&P50|P110' > books_with_an_author_or_an_illustrator.ndjson
+```
+
 #### Long claim option
 If [your claim is too long and triggers a `Argument list too long` error](https://github.com/maxlath/wikidata-filter/issues/13), you can pass a file instead:
 ```sh

lib/get_filter_by_claims_data.js

@@ -3,19 +3,29 @@ const path = require('path')
 const validate = require('./validate_arguments')
 const hash = require('./hash')
 
-module.exports = claimOption => {
-  if (claimOption == null) return
+module.exports = conjonctiveClaimsFilter => {
+  if (conjonctiveClaimsFilter == null) return
 
   // Accept a path as claim option to work around long arguments limitations
   try {
-    claimOption = fs.readFileSync(path.resolve(claimOption)).toString().trim()
+    conjonctiveClaimsFilter = fs.readFileSync(path.resolve(conjonctiveClaimsFilter)).toString().trim()
   } catch (err) {
     if (err.code !== 'ENOENT') throw err
   }
 
-  validate.claim(claimOption)
+  validate.claims(conjonctiveClaimsFilter)
 
-  var [ P, Q ] = claimOption.split(':')
+  return conjonctiveClaimsFilter
+  .split(/&/g)
+  .map(disjonctiveClaimsFilter => {
+    return disjonctiveClaimsFilter
+    .split(/\|/g)
+    .map(claimsData)
+  })
+}
+
+const claimsData = claim => {
+  var [ P, Q ] = claim.split(':')
   const negatedProp = P[0] === '~'
   const filterByClaimValue = Q != null
   if (negatedProp) P = P.slice(1)

lib/valid_claims.js

@@ -1,10 +1,21 @@
 const wdk = require('wikidata-sdk')
 const haveAMatch = require('./have_a_match')
+const every = require('lodash.every')
+const some = require('lodash.some')
 
-module.exports = (claims, filterByClaimData) => {
-  if (!filterByClaimData) return true
+module.exports = (claims, conjonctiveClaimsFilter) => {
+  if (!conjonctiveClaimsFilter) return true
+  // console.log({ conjonctiveClaimsFilter })
+  // ex: P31:Q571&P50|P110 => P31:Q571 && (P50 || P110)
+  return every(conjonctiveClaimsFilter, anyDisjonctiveFilter(claims))
+}
+
+const anyDisjonctiveFilter = claims => disjonctiveClaimsFilter => {
+  return some(disjonctiveClaimsFilter, validClaim(claims))
+}
 
-  const { P, QHash, negatedProp, filterByClaimValue } = filterByClaimData
+const validClaim = claims => claimsFilterData => {
+  const { P, QHash, negatedProp, filterByClaimValue } = claimsFilterData
   // filter-out this entity unless it has claims of the desired property
   var propClaims = claims[P]
 

lib/validate_arguments.js

@@ -3,6 +3,17 @@ const languageCode = /^[a-z]{2}(-[a-z]{2})?$/
 // ~ stands for negated properties
 const isPropertyId = (str) => /^~?P\d+$/.test(str)
 
+const validateClaim = claim => {
+  const [ P, Q ] = claim.split(':')
+  if (!(isPropertyId(P))) throw new Error(`invalid claim property: ${P}`)
+  if (Q) {
+    const Qs = Q.split(',')
+    for (let q of Qs) {
+      if (!(isItemId(q))) throw new Error(`invalid claim value: ${q}`)
+    }
+  }
+}
+
 module.exports = {
   value: function (label, values, list) {
     for (let attr of values) {
@@ -14,14 +25,5 @@ module.exports = {
       if (!(languageCode.test(lang))) throw new Error(`invalid language: ${lang}`)
     }
   },
-  claim: claim => {
-    const [ P, Q ] = claim.split(':')
-    if (!(isPropertyId(P))) throw new Error(`invalid claim property: ${P}`)
-    if (Q) {
-      const Qs = Q.split(',')
-      for (let q of Qs) {
-        if (!(isItemId(q))) throw new Error(`invalid claim value: ${q}`)
-      }
-    }
-  }
+  claims: claims => claims.split(/\||&/g).forEach(validateClaim)
 }

package-lock.json

@@ -25,7 +25,9 @@
     "check-node-version": "^2.1.0",
     "commander": "^2.9.0",
     "lodash.difference": "^4.2.0",
+    "lodash.every": "^4.6.0",
     "lodash.pick": "^4.2.0",
+    "lodash.some": "^4.6.0",
     "split": "^1.0.0",
     "through": "^2.3.8",
     "wikidata-sdk": "^5.1.0"

package.json

@@ -47,4 +47,44 @@ describe('claims', function () {
       done()
     })
   })
+
+  describe('disjonctive claims', function () {
+    it('should return the entity if it has one of the specified claim', function (done) {
+      const result = wdFilter({ claim: 'P31:Q571|P300' })(parsedEntity)
+      should(result).equal(parsedEntity)
+      done()
+    })
+
+    it('should support multiple claim values', function (done) {
+      const result = wdFilter({ claim: 'P6214214|P31:Q571,Q3336843' })(parsedEntity)
+      should(result).equal(parsedEntity)
+      done()
+    })
+
+    it('should support negative claims', function (done) {
+      const result = wdFilter({ claim: '~P31|~P2002' })(parsedEntity)
+      should(result).equal(parsedEntity)
+      done()
+    })
+  })
+
+  describe('conjonctive claims', function () {
+    it("should not return the entity if it doesn't have all the specified claim", function (done) {
+      const result = wdFilter({ claim: 'P31:Q571&P300' })(parsedEntity)
+      should(result).be.null()
+      done()
+    })
+
+    it('should return the entity if it has all the specified claim', function (done) {
+      const result = wdFilter({ claim: 'P31:Q3336843&P300' })(parsedEntity)
+      should(result).equal(parsedEntity)
+      done()
+    })
+
+    it('should support negative claims', function (done) {
+      const result = wdFilter({ claim: 'P31:Q3336843&P300&~P2002' })(parsedEntity)
+      should(result).equal(parsedEntity)
+      done()
+    })
+  })
 })