[core] Prompt for dataset on import/export (#337)

packages/@sanity/core/package.json

@@ -38,7 +38,6 @@
     "filesize": "^3.5.6",
     "fs-extra": "^4.0.2",
     "get-uri": "^2.0.1",
-    "got": "^6.7.1",
     "json-lexer": "^1.1.1",
     "linecount": "^1.0.1",
     "lodash": "^4.17.4",
@@ -48,6 +47,7 @@
     "pumpify": "^1.3.5",
     "rimraf": "^2.6.2",
     "simple-concat": "^1.0.0",
+    "simple-get": "^2.7.0",
     "split2": "^2.1.1",
     "tar-fs": "^1.15.2",
     "thenify": "^3.3.0",

packages/@sanity/core/src/actions/dataset/chooseDatasetPrompt.js

@@ -0,0 +1,31 @@
+import debug from '../../debug'
+import promptForDatasetName from './datasetNamePrompt'
+
+module.exports = async (context, options = {}) => {
+  const {apiClient, prompt} = context
+  const {message, allowCreation} = options
+  const client = apiClient()
+
+  const datasets = await client.datasets.list()
+  const hasProduction = datasets.find(dataset => dataset.name === 'production')
+  const datasetChoices = datasets.map(dataset => ({value: dataset.name}))
+  const selected = await prompt.single({
+    message: message || 'Select dataset to use',
+    type: 'list',
+    choices: allowCreation
+      ? [{value: 'new', name: 'Create new dataset'}, new prompt.Separator(), ...datasetChoices]
+      : datasetChoices
+  })
+
+  if (selected === 'new') {
+    debug('User wants to create a new dataset, prompting for name')
+    const newDatasetName = await promptForDatasetName(prompt, {
+      message: 'Name your dataset:',
+      default: hasProduction ? undefined : 'production'
+    })
+    await client.datasets.create(newDatasetName)
+    return newDatasetName
+  }
+
+  return selected
+}

packages/@sanity/core/src/actions/dataset/streamDataset.js

@@ -1,10 +1,11 @@
-import got from 'got'
+import simpleGet from 'simple-get'
 
 export default (client, dataset) => {
   // Sanity client doesn't handle streams natively since we want to support node/browser
   // with same API. We're just using it here to get hold of URLs and tokens.
   const url = client.getUrl(`/data/export/${dataset}`)
-  return got.stream(url, {
-    headers: {Authorization: `Bearer ${client.config().token}`}
+  const headers = {Authorization: `Bearer ${client.config().token}`}
+  return new Promise((resolve, reject) => {
+    simpleGet({url, headers}, (err, res) => (err ? reject(err) : resolve(res)))
   })
 }

packages/@sanity/core/src/commands/dataset/exportDatasetCommand.js

@@ -2,30 +2,25 @@ import path from 'path'
 import fse from 'fs-extra'
 import split from 'split2'
 import prettyMs from 'pretty-ms'
+import {pathTools} from '@sanity/util'
 import streamDataset from '../../actions/dataset/streamDataset'
 import skipSystemDocuments from '../../util/skipSystemDocuments'
+import chooseDatasetPrompt from '../../actions/dataset/chooseDatasetPrompt'
 
 export default {
   name: 'export',
   group: 'dataset',
   signature: '[NAME] [DESTINATION]',
   description: 'Export dataset to local filesystem',
   action: async (args, context) => {
-    const {apiClient, output, chalk} = context
+    const {apiClient, output, chalk, workDir, prompt} = context
     const client = apiClient()
-    const [dataset, destination] = args.argsWithoutOptions
-    const signature = 'sanity dataset export [dataset] [destination]'
+    const [targetDataset, targetDestination] = args.argsWithoutOptions
+    const {absolutify} = pathTools
 
+    let dataset = targetDataset
     if (!dataset) {
-      throw new Error(
-        `Dataset must be specified ("${signature}")`
-      )
-    }
-
-    if (!destination) {
-      throw new Error(
-        `Destination filename must be specified. Use "-" to print to stdout. ("${signature}")`
-      )
+      dataset = await chooseDatasetPrompt(context, {message: 'Select dataset to export'})
     }
 
     // Verify existence of dataset before trying to export from it
@@ -36,7 +31,17 @@ export default {
       )
     }
 
-    const outputPath = await getOutputPath(destination, dataset)
+    let destinationPath = targetDestination
+    if (!destinationPath) {
+      destinationPath = await prompt.single({
+        type: 'input',
+        message: 'Output path:',
+        default: path.join(workDir, `${dataset}.ndjson`),
+        filter: absolutify
+      })
+    }
+
+    const outputPath = await getOutputPath(destinationPath, dataset)
 
     // If we are dumping to a file, let the user know where it's at
     if (outputPath) {
@@ -45,7 +50,8 @@ export default {
 
     const startTime = Date.now()
 
-    streamDataset(client, dataset)
+    const stream = await streamDataset(client, dataset)
+    stream
       .pipe(split())
       .pipe(skipSystemDocuments)
       .pipe(outputPath ? fse.createWriteStream(outputPath) : process.stdout)

packages/@sanity/core/src/commands/dataset/importDatasetCommand.js

@@ -1,23 +1,26 @@
 import path from 'path'
-import got from 'got'
-import padStart from 'lodash/padStart'
+import simpleGet from 'simple-get'
 import fse from 'fs-extra'
+import sanityImport from '@sanity/import'
+import padStart from 'lodash/padStart'
 import prettyMs from 'pretty-ms'
 import linecount from 'linecount/promise'
+import chooseDatasetPrompt from '../../actions/dataset/chooseDatasetPrompt'
 import debug from '../../debug'
-import sanityImport from '@sanity/import'
 
 export default {
   name: 'import',
   group: 'dataset',
   signature: '[FILE] [TARGET_DATASET]',
   description: 'Import dataset from local filesystem',
   action: async (args, context) => {
-    const {apiClient, output, chalk} = context
+    const {apiClient, output, chalk, prompt} = context
 
+    let spinner = null
     const operation = getMutationOperation(args.extOptions)
+    const client = apiClient()
 
-    const [file, targetDataset] = args.argsWithoutOptions
+    const [file, target] = args.argsWithoutOptions
     if (!file) {
       throw new Error(
         `Source file name and target dataset must be specified ("sanity dataset import ${chalk.bold(
@@ -26,49 +29,45 @@ export default {
       )
     }
 
+    debug('[  0%] Fetching available datasets')
+    spinner = output.spinner('Fetching available datasets').start()
+    const datasets = await client.datasets.list()
+    spinner.succeed('[100%] Fetching available datasets')
+
+    let targetDataset = target
     if (!targetDataset) {
-      // @todo ask which dataset the user wants to use
-      throw new Error(
-        `Target dataset must be specified ("sanity dataset import [file] ${chalk.bold(
-          '[dataset]'
-        )}")`
-      )
+      targetDataset = await chooseDatasetPrompt(context, {
+        message: 'Select target dataset',
+        allowCreation: true
+      })
+    } else if (!datasets.find(dataset => dataset.name === targetDataset)) {
+      debug('Target dataset does not exist, prompting for creation')
+      const shouldCreate = await prompt.single({
+        type: 'confirm',
+        message: `Dataset "${targetDataset}" does not exist, would you like to create it?`,
+        default: true
+      })
+
+      if (!shouldCreate) {
+        throw new Error(`Dataset "${targetDataset}" does not exist`)
+      }
+
+      await client.datasets.create(targetDataset)
     }
 
+    debug(`Target dataset has been set to "${targetDataset}"`)
+
     const isUrl = /^https?:\/\//i.test(file)
     const sourceFile = isUrl ? file : path.resolve(process.cwd(), file)
-    const inputStream = isUrl
-      ? got.stream(sourceFile)
-      : fse.createReadStream(sourceFile)
-    const client = apiClient()
+    const inputSource = isUrl ? getUrlStream(sourceFile) : fse.createReadStream(sourceFile)
+    const inputStream = await inputSource
 
     const documentCount = isUrl ? 0 : await linecount(sourceFile)
     debug(
       documentCount
         ? 'Could not count documents in source'
         : `Found ${documentCount} lines in source file`
     )
-    debug(`Target dataset has been set to "${targetDataset}"`)
-
-    let spinner = null
-
-    // Verify existence of dataset before trying to import to it
-    debug('Verifying if dataset already exists')
-    spinner = output.spinner('Checking if destination dataset exists').start()
-    const datasets = await client.datasets.list()
-    if (!datasets.find(set => set.name === targetDataset)) {
-      // @todo ask if user wants to create it
-      spinner.fail()
-      throw new Error(
-        [
-          `Dataset with name "${targetDataset}" not found.`,
-          `Create it by running "${chalk.cyan(
-            `sanity dataset create ${targetDataset}`
-          )}" first`
-        ].join('\n')
-      )
-    }
-    spinner.succeed()
 
     const importClient = client.clone().config({dataset: targetDataset})
 
@@ -146,14 +145,22 @@ export default {
 
       endTask({success: true})
 
-      output.print(
-        'Done! Imported %d documents to dataset "%s"',
-        imported,
-        targetDataset
-      )
+      output.print('Done! Imported %d documents to dataset "%s"', imported, targetDataset)
     } catch (err) {
       endTask({success: false})
-      output.error(err)
+
+      let error = err.message
+      if (err.response && err.response.statusCode === 409) {
+        error = [
+          err.message,
+          '',
+          'You probably want either:',
+          ' --replace (replace existing documents with same IDs)',
+          ' --missing (only import documents that do not already exist)'
+        ].join('\n')
+      }
+
+      output.error(chalk.red(`\n${error}\n`))
     }
   }
 }
@@ -183,3 +190,9 @@ function getPercentage(opts) {
   const percent = Math.floor(opts.current / opts.total * 100)
   return `[${padStart(percent, 3, ' ')}%] `
 }
+
+function getUrlStream(url) {
+  return new Promise((resolve, reject) => {
+    simpleGet(url, (err, res) => (err ? reject(err) : resolve(res)))
+  })
+}

packages/@sanity/util/src/index.js

@@ -3,11 +3,13 @@ import reduceConfig from './reduceConfig'
 import lazyRequire from './lazyRequire'
 import safeJson from './safeJson'
 import getSanityVersions from './getSanityVersions'
+import * as pathTools from './pathTools'
 
 export {
   getConfig,
   reduceConfig,
   lazyRequire,
   safeJson,
-  getSanityVersions
+  getSanityVersions,
+  pathTools
 }

packages/@sanity/util/src/pathTools.js

@@ -0,0 +1,36 @@
+import fse from 'fs-extra'
+import path from 'path'
+import os from 'os'
+
+export async function pathIsEmpty(dir) {
+  try {
+    const content = await fse.readdir(absolutify(dir))
+    return content.length === 0
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return true
+    }
+
+    throw err
+  }
+}
+
+export function expandHome(filePath) {
+  if (filePath.charCodeAt(0) === 126 /* ~ */) {
+    if (filePath.charCodeAt(1) === 43 /* + */) {
+      return path.join(process.cwd(), filePath.slice(2))
+    }
+
+    const home = os.homedir()
+    return home ? path.join(home, filePath.slice(1)) : filePath
+  }
+
+  return filePath
+}
+
+export function absolutify(dir) {
+  const pathName = expandHome(dir)
+  return path.isAbsolute(pathName)
+    ? pathName
+    : path.resolve(process.cwd(), pathName)
+}

packages/@sanity/util/test/util.test.js

@@ -1,7 +1,10 @@
+/* eslint-disable no-sync */
+import fs from 'fs'
+import os from 'os'
 import path from 'path'
 import {describe, it} from 'mocha'
 import {expect} from 'chai'
-import {reduceConfig, getSanityVersions} from '../src'
+import {reduceConfig, getSanityVersions, pathTools} from '../src'
 
 describe('util', () => {
   describe('reduceConfig', () => {
@@ -59,4 +62,41 @@ describe('util', () => {
       })
     })
   })
+
+  describe('path tools', () => {
+    it('returns whether or not a path is empty (false)', async () => {
+      const {pathIsEmpty} = pathTools
+      const isEmpty = await pathIsEmpty(__dirname)
+      expect(isEmpty).to.equal(false)
+    })
+
+    it('returns whether or not a path is empty (true)', async () => {
+      const {pathIsEmpty} = pathTools
+      const emptyPath = path.join(__dirname, '__temp__')
+      fs.mkdirSync(emptyPath)
+      const isEmpty = await pathIsEmpty(emptyPath)
+      fs.rmdirSync(emptyPath)
+      expect(isEmpty).to.equal(true)
+    })
+
+    it('can expand home dirs', () => {
+      const {expandHome} = pathTools
+      expect(expandHome('~/tmp')).to.equal(path.join(os.homedir(), 'tmp'))
+    })
+
+    it('can absolutify relative paths', () => {
+      const {absolutify} = pathTools
+      expect(absolutify('./util.test.js')).to.equal(path.join(process.cwd(), 'util.test.js'))
+    })
+
+    it('can absolutify homedir paths', () => {
+      const {absolutify} = pathTools
+      expect(absolutify('~/tmp')).to.equal(path.join(os.homedir(), 'tmp'))
+    })
+
+    it('can absolutify (noop) absolute paths', () => {
+      const {absolutify} = pathTools
+      expect(absolutify('/tmp/foo')).to.equal('/tmp/foo')
+    })
+  })
 })