Merge 1de1e4a into 5f63765

mediachain · Jan 11, 2017 · ab40687 · ab40687
2 parents 5f63765 + 1de1e4a
commit ab40687
Show file tree

Hide file tree

Showing 7 changed files with 25 additions and 8 deletions.
diff --git a/scripts/publish-500px.sh b/scripts/publish-500px.sh
@@ -11,7 +11,8 @@ SKIP_VALIDATION='--skipSchemaValidation'
 
 mcclient publish ${SKIP_VALIDATION} \
     --jqFilter '._source | del(.aesthetics)' \
-    --idFilter '.native_id' \
+    --prefix '500px'
+    --idFilter '.native_id | sub("500px[^_]*_(?<id>\\d+)"; "\(.id)")' \
     ${NAMESPACE} \
     ${SCHEMA_HASH} \
     $1 > /dev/null

diff --git a/scripts/publish-dpla.sh b/scripts/publish-dpla.sh
@@ -12,7 +12,8 @@ SKIP_VALIDATION='--skipSchemaValidation'
 
 mcclient publish ${SKIP_VALIDATION} \
     --jqFilter '._source | del(.aesthetics)' \
-    --idFilter '.native_id | sub("dpla_http://dp.la/api/items/(?<id>.+)"; "dpla_\(.id)")' \
+    --idFilter '.native_id | sub("dpla_http://dp.la/api/items/(?<id>.+)"; "\(.id)")' \
+    --prefix 'dpla'
     ${NAMESPACE} \
     ${SCHEMA_HASH} \
     $1 > /dev/null

diff --git a/scripts/publish-flickr-c10.sh b/scripts/publish-flickr-c10.sh
@@ -7,7 +7,8 @@ COMPOUND="--compound 10"
 mcclient publish ${COMPOUND} \
     --skipSchemaValidation \
     --jqFilter '._source | del(.aesthetics)' \
-    --idFilter '.native_id' \
+    --idFilter '.native_id | sub("flickr[^_]*_(?<id>\\d+)"; "\(.id)")' \
+    --prefix 'flickr'
     ${NAMESPACE} \
     ${SCHEMA_HASH} \
     $1 > /dev/null

diff --git a/scripts/publish-flickr-c100.sh b/scripts/publish-flickr-c100.sh
@@ -7,7 +7,8 @@ COMPOUND="--compound 100"
 mcclient publish ${COMPOUND} \
     --skipSchemaValidation \
     --jqFilter '._source | del(.aesthetics)' \
-    --idFilter '.native_id' \
+    --idFilter '.native_id | sub("flickr[^_]*_(?<id>\\d+)"; "\(.id)")' \
+    --prefix 'flickr'
     ${NAMESPACE} \
     ${SCHEMA_HASH} \
     $1 > /dev/null

diff --git a/scripts/publish-flickr.sh b/scripts/publish-flickr.sh
@@ -12,7 +12,8 @@ SKIP_VALIDATION='--skipSchemaValidation'
 
 mcclient publish ${SKIP_VALIDATION} \
     --jqFilter '._source | del(.aesthetics)' \
-    --idFilter '.native_id' \
+    --idFilter '.native_id | sub("flickr[^_]*_(?<id>\\d+)"; "\(.id)")' \
+    --prefix 'flickr'
     ${NAMESPACE} \
     ${SCHEMA_HASH} \
     $1 > /dev/null

diff --git a/scripts/publish-pexels.sh b/scripts/publish-pexels.sh
@@ -11,7 +11,8 @@ SKIP_VALIDATION='--skipSchemaValidation'
 
 mcclient publish ${SKIP_VALIDATION} \
     --jqFilter '._source | del(.aesthetics)' \
-    --idFilter '.native_id' \
+    --idFilter '.native_id | sub("flickr[^_]*_(?<id>\\d+)"; "\(.id)")' \
+    --prefix 'pexels'
     ${NAMESPACE} \
     ${SCHEMA_HASH} \
     $1 > /dev/null

diff --git a/src/client/cli/commands/publish.js b/src/client/cli/commands/publish.js
@@ -41,6 +41,12 @@ module.exports = {
       description: 'A jq filter that produces a mediachain identifier from your input object.  ' +
       'Will be applied after `jqFilter`, and does not modify the object itself.\n'
     },
+    prefix: {
+      required: true,
+      type: 'string',
+      description: 'Prefix for the WKI.  ' +
+      'The WKI is formed by joining the `prefix` (e.g. "moma") and the identifier extracted by `idFilter` with a colon (e.g. "moma:67350")'
+    },
     jqFilter: {
       type: 'string',
       description: 'A jq filter string to use to pre-process your input data.\n',
@@ -71,7 +77,7 @@ module.exports = {
   },
 
   handler: subcommand((opts: HandlerOptions) => {
-    const {client, namespace, schemaReference, batchSize, filename, dryRun, skipSchemaValidation, jqFilter, idFilter, compound} = opts
+    const {client, namespace, schemaReference, batchSize, filename, dryRun, skipSchemaValidation, jqFilter, prefix, idFilter, compound} = opts
     const publishOpts = {namespace, compound}
 
     let stream: Readable
@@ -114,6 +120,7 @@ module.exports = {
           publishOpts,
           schemaReference,
           schema,
+          prefix,
           jqFilter: composeJQFilters(jqFilter, idFilter)})
       )
   })
@@ -129,6 +136,7 @@ function publishStream (opts: {
   publishOpts: {namespace: string, compound?: number},
   schemaReference?: string,
   schema: ?SelfDescribingSchema,
+  prefix: string,
   jqFilter: string
 }): Promise<*> {
   const {
@@ -141,6 +149,7 @@ function publishStream (opts: {
     skipSchemaValidation,
     schemaReference,
     schema,
+    prefix,
     jqFilter
   } = opts
 
@@ -216,7 +225,9 @@ function publishStream (opts: {
 
         wki = wki.toString()
 
-        const refs = [ wki ]
+        wkiWithPrefix = [ prefix, wki ].join(':')
+
+        const refs = [ wkiWithPrefix ]
         const tags = [] // TODO: support extracting tags
         const deps = []