+ get_wikidata_humans_dump script

inventaire · Jul 20, 2018 · 2b5310c · 2b5310c
1 parent d7cb02c
commit 2b5310c
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 1 deletion.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -112,7 +112,8 @@
     "shell-quote": "^1.4.3",
     "should": "^11.1.1",
     "sinon": "^1.17.1",
-    "supervisor": "^0.10.0"
+    "supervisor": "^0.10.0",
+    "wikidata-filter": "^2.3.1"
   },
   "engines": {
     "node": ">= 6.4"

diff --git a/scripts/dumps/get_wikidata_humans_dump b/scripts/dumps/get_wikidata_humans_dump
@@ -0,0 +1,10 @@
+#!/usr/bin/env zsh
+# Generate a pre-filtered dump of humans in Wikidata to ease development setup
+# cf https://github.com/inventaire/inventaire-deploy/blob/d280055/install_entities_search_engine#L24-L28
+
+curl -s https://dumps.wikimedia.org/wikidatawiki/entities/latest-all.json.gz |
+  gzip -d |
+  wikidata-filter --claim P31:Q5 --omit type,sitelinks |
+  gzip -c9 > humans.ndjson.gz
+
+chmod 664 humans.ndjson.gz