[import] Allow importing from array (#228)

packages/@sanity/import/README.md

@@ -22,6 +22,7 @@ const client = sanityClient({
   useCdn: false
 })
 
+// Input can either be a stream or an array of documents
 const input = fs.createReadStream('my-documents.ndjson')
 sanityImport(input, {
   client: client,

packages/@sanity/import/src/documentHasErrors.js

@@ -1,4 +1,4 @@
-module.exports = function documentHasErrors(doc) {
+function documentHasErrors(doc) {
   if (typeof doc._id !== 'undefined' && typeof doc._id !== 'string') {
     return `Document contained an invalid "_id" property - must be a string`
   }
@@ -9,3 +9,12 @@ module.exports = function documentHasErrors(doc) {
 
   return null
 }
+
+documentHasErrors.validate = (doc, index) => {
+  const err = documentHasErrors(doc)
+  if (err) {
+    throw new Error(`Failed to parse document at index #${index}: ${err}`)
+  }
+}
+
+module.exports = documentHasErrors

packages/@sanity/import/src/import.js

@@ -5,6 +5,7 @@ const streamToArray = require('./streamToArray')
 const {getAssetRefs, unsetAssetRefs} = require('./assetRefs')
 const assignArrayKeys = require('./assignArrayKeys')
 const uploadAssets = require('./uploadAssets')
+const documentHasErrors = require('./documentHasErrors')
 const batchDocuments = require('./batchDocuments')
 const importBatches = require('./importBatches')
 const {
@@ -14,17 +15,22 @@ const {
   strengthenReferences
 } = require('./references')
 
-async function importFromStream(stream, opts) {
-  const options = validateOptions(stream, opts)
+async function importDocuments(input, opts) {
+  const options = validateOptions(input, opts)
 
-  // Get raw documents from the stream
-  debug('Streaming input source to array of documents')
   options.onProgress({step: 'Reading/validating data file'})
-  const raw = await streamToArray(stream)
+  const isStream = typeof input.pipe === 'function'
+  let documents = input
+  if (isStream) {
+    debug('Streaming input source to array of documents')
+    documents = await streamToArray(input)
+  } else {
+    documents.some(documentHasErrors.validate)
+  }
 
   // User might not have applied `_key` on array elements which are objects;
   // if this is the case, generate random keys to help realtime engine
-  const keyed = raw.map(doc => assignArrayKeys(doc))
+  const keyed = documents.map(doc => assignArrayKeys(doc))
 
   // Sanity prefers to have a `_type` on every object. Make sure references
   // has `_type` set to `reference`.
@@ -62,4 +68,4 @@ async function importFromStream(stream, opts) {
   return docsImported
 }
 
-module.exports = importFromStream
+module.exports = importDocuments

packages/@sanity/import/src/validateOptions.js

@@ -4,27 +4,21 @@ const clientMethods = ['fetch', 'transaction', 'config']
 const allowedOperations = ['create', 'createIfNotExists', 'createOrReplace']
 const defaultOperation = allowedOperations[0]
 
-function validateOptions(stream, opts) {
+function validateOptions(input, opts) {
   const options = defaults({}, opts, {
     operation: defaultOperation,
     onProgress: noop
   })
 
-  if (typeof stream.pipe !== 'function') {
-    throw new Error(
-      'Stream does not seem to be a readable stream - no "pipe" method found'
-    )
+  if (!input || (typeof input.pipe !== 'function' && !Array.isArray(input))) {
+    throw new Error('Stream does not seem to be a readable stream or an array')
   }
 
   if (!options.client) {
-    throw new Error(
-      '`options.client` must be set to an instance of @sanity/client'
-    )
+    throw new Error('`options.client` must be set to an instance of @sanity/client')
   }
 
-  const missing = clientMethods.find(
-    key => typeof options.client[key] !== 'function'
-  )
+  const missing = clientMethods.find(key => typeof options.client[key] !== 'function')
 
   if (missing) {
     throw new Error(

packages/@sanity/import/test/__snapshots__/import.test.js.snap

@@ -0,0 +1,43 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`employee creation 1`] = `
+Object {
+  "mutations": Array [
+    Object {
+      "create": Object {
+        "_id": "espen",
+        "_type": "employee",
+        "name": "Espen",
+      },
+    },
+    Object {
+      "create": Object {
+        "_id": "pk",
+        "_type": "employee",
+        "name": "Per-Kristian",
+      },
+    },
+  ],
+}
+`;
+
+exports[`employee creation 2`] = `
+Object {
+  "mutations": Array [
+    Object {
+      "create": Object {
+        "_id": "espen",
+        "_type": "employee",
+        "name": "Espen",
+      },
+    },
+    Object {
+      "create": Object {
+        "_id": "pk",
+        "_type": "employee",
+        "name": "Per-Kristian",
+      },
+    },
+  ],
+}
+`;

packages/@sanity/import/test/helpers/helpers.js

@@ -0,0 +1,22 @@
+const noop = require('lodash/noop')
+const sanityClient = require('@sanity/client')
+const {injectResponse} = require('get-it/middleware')
+
+const defaultClientOptions = {
+  projectId: 'foo',
+  dataset: 'bar',
+  token: 'abc123',
+  useCdn: false
+}
+
+const getSanityClient = (inject = noop, opts = {}) => {
+  const requester = sanityClient.requester.clone()
+  requester.use(injectResponse({inject}))
+  const req = {requester: requester}
+  const client = sanityClient(Object.assign(defaultClientOptions, req, opts))
+  return client
+}
+
+module.exports = {
+  getSanityClient
+}

packages/@sanity/import/test/helpers/index.js

@@ -0,0 +1 @@
+module.exports = require('./helpers')

packages/@sanity/import/test/import.test.js

@@ -1,39 +1,98 @@
+/* eslint-disable no-sync */
 const fs = require('fs')
 const path = require('path')
 const sanityClient = require('@sanity/client')
+const {getSanityClient} = require('./helpers')
 const importer = require('../')
 
-const client = sanityClient({
+const defaultClient = sanityClient({
   projectId: 'foo',
   dataset: 'bar',
   useCdn: false,
   token: 'foo'
 })
 
-const options = {client}
+const importOptions = {client: defaultClient}
 const fixturesDir = path.join(__dirname, 'fixtures')
-const getFixture = fix => {
-  const fixPath = path.join(fixturesDir, `${fix}.ndjson`)
-  return fs.createReadStream(fixPath, 'utf8')
-}
+const getFixturePath = fix => path.join(fixturesDir, `${fix}.ndjson`)
+const getFixtureStream = fix => fs.createReadStream(getFixturePath(fix), 'utf8')
+const getFixtureArray = fix =>
+  fs
+    .readFileSync(getFixturePath(fix), 'utf8')
+    .trim()
+    .split('\n')
+    .map(JSON.parse)
+
+test('rejects on invalid input type (null/undefined)', async () => {
+  await expect(importer(null, importOptions)).rejects.toHaveProperty(
+    'message',
+    'Stream does not seem to be a readable stream or an array'
+  )
+})
+
+test('rejects on invalid input type (non-array)', async () => {
+  await expect(importer({}, importOptions)).rejects.toHaveProperty(
+    'message',
+    'Stream does not seem to be a readable stream or an array'
+  )
+})
 
 test('rejects on invalid JSON', async () => {
-  await expect(importer(getFixture('invalid-json'), options)).rejects.toHaveProperty(
+  await expect(importer(getFixtureStream('invalid-json'), importOptions)).rejects.toHaveProperty(
     'message',
     'Failed to parse line #3: Unexpected token _ in JSON at position 1'
   )
 })
 
-test('rejects on missing `_id` property', async () => {
-  await expect(importer(getFixture('invalid-id'), options)).rejects.toHaveProperty(
+test('rejects on invalid `_id` property', async () => {
+  await expect(importer(getFixtureStream('invalid-id'), importOptions)).rejects.toHaveProperty(
     'message',
     'Failed to parse line #2: Document contained an invalid "_id" property - must be a string'
   )
 })
 
 test('rejects on missing `_type` property', async () => {
-  await expect(importer(getFixture('missing-type'), options)).rejects.toHaveProperty(
+  await expect(importer(getFixtureStream('missing-type'), importOptions)).rejects.toHaveProperty(
     'message',
     'Failed to parse line #3: Document did not contain required "_type" property of type string'
   )
 })
+
+test('rejects on missing `_type` property (from array)', async () => {
+  await expect(importer(getFixtureArray('missing-type'), importOptions)).rejects.toHaveProperty(
+    'message',
+    'Failed to parse document at index #2: Document did not contain required "_type" property of type string'
+  )
+})
+
+test('accepts an array as source', async () => {
+  const docs = getFixtureArray('employees')
+  const client = getSanityClient(getMockEmployeeHandler())
+  const res = await importer(docs, {client})
+  expect(res).toBe(2)
+})
+
+test('accepts a stream as source', async () => {
+  const client = getSanityClient(getMockEmployeeHandler())
+  const res = await importer(getFixtureStream('employees'), {client})
+  expect(res).toBe(2)
+})
+
+function getMockEmployeeHandler() {
+  return req => {
+    const options = req.context.options
+    const uri = options.uri || options.url
+
+    if (uri.includes('/data/mutate')) {
+      const body = JSON.parse(options.body)
+      expect(body).toMatchSnapshot('employee creation')
+      const results = body.mutations.map(mut => ({
+        id: mut.create.id,
+        operation: 'create'
+      }))
+      return {body: {results}}
+    }
+
+    return {statusCode: 400, body: {error: `"${uri}" should not be called`}}
+  }
+}

packages/@sanity/import/test/uploadAssets.test.js

@@ -1,24 +1,9 @@
 const path = require('path')
 const fileUrl = require('file-url')
 const noop = require('lodash/noop')
-const sanityClient = require('@sanity/client')
-const {injectResponse} = require('get-it/middleware')
 const uploadAssets = require('../src/uploadAssets')
 const mockAssets = require('./fixtures/mock-assets')
-
-const defaultClientOptions = {
-  projectId: 'foo',
-  dataset: 'bar',
-  useCdn: false
-}
-
-const getSanityClient = (inject = noop, opts = {}) => {
-  const requester = sanityClient.requester.clone()
-  requester.use(injectResponse({inject}))
-  const req = {requester: requester}
-  const client = sanityClient(Object.assign(defaultClientOptions, req, opts))
-  return client
-}
+const {getSanityClient} = require('./helpers')
 
 const fixturesDir = path.join(__dirname, 'fixtures')
 const imgFileUrl = fileUrl(path.join(fixturesDir, 'img.gif'))
@@ -70,9 +55,7 @@ test('will reuse an existing asset if it exists', () => {
     return {statusCode: 400, body: {error: `"${uri}" should not be called`}}
   })
 
-  return expect(
-    uploadAssets([fileAsset], {client, onProgress: noop})
-  ).resolves.toBe(1)
+  return expect(uploadAssets([fileAsset], {client, onProgress: noop})).resolves.toBe(1)
 })
 
 test('will upload asset that do not already exist', () => {
@@ -100,9 +83,7 @@ test('will upload asset that do not already exist', () => {
     return {statusCode: 400, body: {error: `"${uri}" should not be called`}}
   })
 
-  return expect(
-    uploadAssets([fileAsset], {client, onProgress: noop})
-  ).resolves.toBe(1)
+  return expect(uploadAssets([fileAsset], {client, onProgress: noop})).resolves.toBe(1)
 })
 
 test('will upload once but batch patches', () => {