Merge d1d2d32 into 3a79c1b

bin/contentful-import

@@ -1,16 +1,14 @@
 #!/usr/bin/env node
-var log = require('npmlog')
 var runContentfulImport = require('../dist/index')
 var usageParams = require('../dist/usageParams')
 
-log.info('import', 'Contentful Import Tool')
-log.info('import', 'Importing data ...')
-
 runContentfulImport(usageParams)
-.then((result) => {
-  process.exit(0)
-})
-.catch(function (err) {
-  log.error('import', 'Failed within\n', err)
-  process.exit(1)
-})
+  .then((result) => {
+    process.exit(0)
+  })
+  .catch(function (err) {
+    if (err.name !== 'ContentfulMultiError') {
+      console.error(err)
+    }
+    process.exit(1)
+  })

lib/get-transformed-destination-response.js → lib/get-destination-response.js

@@ -13,12 +13,28 @@ export default function getTransformedDestinationResponse ({
   skipLocales,
   skipContentModel
 }) {
+  const entryIds = sourceResponse.entries.map((entry) => entry.sys.id)
+  const assetIds = sourceResponse.assets.map((asset) => asset.sys.id)
   return getOutdatedDestinationContent({
     managementClient,
     spaceId,
     sourceResponse,
     contentModelOnly,
+    entryIds,
+    assetIds,
     skipLocales,
     skipContentModel
   })
+    .then((destinationResponse) => {
+      if (skipContentModel) {
+        destinationResponse.contentTypes = []
+        destinationResponse.locales = []
+      }
+
+      if (skipLocales) {
+        destinationResponse.locales = []
+      }
+
+      return destinationResponse
+    })
 }

lib/index.js

@@ -1,99 +1,163 @@
-import { find, filter } from 'lodash/collection'
-import createClients from 'contentful-batch-libs/dist/utils/create-clients'
-import pushToSpace from 'contentful-batch-libs/dist/push/push-to-space'
 import Promise from 'bluebird'
-import transformSpace from 'contentful-batch-libs/dist/transform/transform-space'
-import getTransformedDestinationResponse from './get-transformed-destination-response'
-import log from 'npmlog'
-import { startCase } from 'lodash'
 import Table from 'cli-table2'
+import Listr from 'listr'
+import UpdateRenderer from 'listr-update-renderer'
+import VerboseRenderer from 'listr-verbose-renderer'
+import { startCase } from 'lodash'
 import moment from 'moment'
 
+import pushToSpace from 'contentful-batch-libs/dist/push/push-to-space'
+import transformSpace from 'contentful-batch-libs/dist/transform/transform-space'
+import createClients from 'contentful-batch-libs/dist/utils/create-clients'
+import {
+  setupLogging,
+  displayErrorLog,
+  writeErrorLogFile
+} from 'contentful-batch-libs/dist/utils/logging'
+
+import getDestinationResponse from './get-destination-response'
 import parseOptions from './parseOptions'
 
-const summary = {}
+function createListrOptions (options) {
+  if (options.useVerboseRenderer) {
+    return {
+      renderer: VerboseRenderer
+    }
+  }
+  return {
+    renderer: UpdateRenderer,
+    collapse: false
+  }
+}
 
 export default function runContentfulImport (params) {
-  summary.startTime = moment()
-
+  const log = []
   const options = parseOptions(params)
+  const listrOptions = createListrOptions(options)
 
-  const clients = createClients(options)
-  return Promise.props({
-    source: options.content,
-    destination: getTransformedDestinationResponse({
-      managementClient: clients.destination.management,
-      spaceId: clients.destination.spaceId,
-      sourceResponse: options.content,
-      contentModelOnly: options.contentModelOnly,
-      skipLocales: options.skipLocales,
-      skipContentModel: options.skipContentModel
-    })
+  // Setup custom log listener to store log messages for later
+  setupLogging(log)
+
+  const infoTable = new Table()
+
+  infoTable.push([{colSpan: 2, content: 'The following entities are going to be imported:'}])
+
+  Object.keys(options.content).forEach((type) => {
+    if (options.skipLocales && type === 'locales') {
+      return
+    }
+
+    if (options.skipContentModel && type === 'contentTypes') {
+      return
+    }
+
+    if (options.contentModelOnly && !(['contentTypes', 'locales'].includes(type))) {
+      return
+    }
+
+    infoTable.push([startCase(type), options.content[type].length])
   })
-    .then((responses) => {
-      return Promise.props({
-        source: transformSpace(responses.source, responses.destination),
-        destination: responses.destination
-      })
-    })
-    .then((responses) => {
-      responses.source.deletedContentTypes = filter(responses.destination.contentTypes, (contentType) => {
-        return !find(responses.source.contentTypes, {original: {sys: {id: contentType.sys.id}}})
-      })
-      responses.source.deletedLocales = filter(responses.destination.locales, (locale) => {
-        return !find(responses.source.locales, {original: {code: locale.code}})
-      })
-      return responses
-    })
-    // push source space content to destination space
-    .then((responses) => {
-      return pushToSpace({
-        sourceContent: responses.source,
-        destinationContent: responses.destination,
-        managementClient: clients.destination.management,
-        spaceId: clients.destination.spaceId,
-        prePublishDelay: options.prePublishDelay,
-        assetProcessDelay: options.assetProcessDelay,
-        contentModelOnly: options.contentModelOnly,
-        skipLocales: options.skipLocales,
-        skipContentModel: options.skipContentModel,
-        skipContentPublishing: options.skipContentPublishing
-      })
-    })
-    .then((response) => {
-      log.info('import', 'Finished importing all data')
 
-      const infoTable = new Table()
+  console.log(infoTable.toString())
 
-      infoTable.push([{colSpan: 2, content: 'The following entities were imported'}])
+  const tasks = new Listr([
+    {
+      title: 'Initialize clients',
+      task: (ctx) => {
+        ctx.clients = createClients(options)
+      }
+    },
+    {
+      title: 'Checking if destination space already has any content and retrieving it',
+      task: (ctx) => {
+        return Promise.props({
+          source: options.content,
+          destination: getDestinationResponse({
+            managementClient: ctx.clients.destination.management,
+            spaceId: ctx.clients.destination.spaceId,
+            sourceResponse: options.content,
+            skipLocales: options.skipLocales,
+            skipContentModel: options.skipContentModel
+          })
+        })
+          .then(({source, destination}) => {
+            ctx.source = source
+            ctx.destination = destination
+          })
+      }
+    },
+    {
+      title: 'Apply transformations to source data',
+      task: (ctx) => {
+        return transformSpace(ctx.source, ctx.destination)
+          .then((source) => {
+            ctx.source = source
+          })
+      }
+    },
+    {
+      title: 'Push content to destination space',
+      task: (ctx, task) => {
+        return pushToSpace({
+          sourceContent: ctx.source,
+          destinationContent: ctx.destination,
+          managementClient: ctx.clients.destination.management,
+          spaceId: ctx.clients.destination.spaceId,
+          assetProcessDelay: options.assetProcessDelay,
+          contentModelOnly: options.contentModelOnly,
+          skipLocales: options.skipLocales,
+          skipContentModel: options.skipContentModel,
+          skipContentPublishing: options.skipContentPublishing,
+          prePublishDelay: options.prePublishDelay,
+          listrOptions
+        })
+      }
+    }
+  ], listrOptions)
 
-      Object.keys(options.content).forEach((type) => {
-        if (options.skipLocales && type === 'locales') {
-          return
-        }
+  return tasks.run({
+    data: {}
+  })
+    .then((ctx) => {
+      console.log('Finished importing all data')
 
-        if (options.skipContentModel && type === 'contentTypes') {
-          return
-        }
+      const resultTypes = Object.keys(ctx.data)
+      if (resultTypes.length) {
+        const resultTable = new Table()
 
-        if (options.contentModelOnly && !(['contentTypes', 'locales'].includes(type))) {
-          return
-        }
+        resultTable.push([{colSpan: 2, content: 'Imported entities'}])
 
-        infoTable.push([startCase(type), options.content[type].length])
-      })
+        resultTypes.forEach((type) => {
+          resultTable.push([startCase(type), ctx.data[type].length])
+        })
 
-      console.log(infoTable.toString())
+        console.log(resultTable.toString())
+      } else {
+        console.log('No data was imported')
+      }
 
-      const durationHuman = summary.startTime.fromNow(true)
-      const durationSeconds = moment().diff(summary.startTime, 'seconds')
+      const durationHuman = options.startTime.fromNow(true)
+      const durationSeconds = moment().diff(options.startTime, 'seconds')
 
-      log.info('import', `The import took ${durationHuman} (${durationSeconds}s)`)
+      console.log(`The import took ${durationHuman} (${durationSeconds}s)`)
 
-      return response
+      return ctx.data
     })
     .catch((err) => {
-      log.error('import', err)
-      throw err
+      log.push({
+        ts: (new Date()).toJSON(),
+        level: 'error',
+        error: err
+      })
+    })
+    .then((data) => {
+      const errorLog = log.filter((logMessage) => logMessage.level !== 'info')
+
+      displayErrorLog(errorLog)
+      if (errorLog.length) {
+        return writeErrorLogFile(options.errorLogFile, errorLog)
+          .then(() => data)
+      }
+      return data
     })
 }

lib/parseOptions.js

@@ -1,13 +1,18 @@
 import fs from 'fs'
 import { resolve } from 'path'
+
+import moment from 'moment'
+
 import { version } from '../package'
 import { proxyStringToObject, agentFromProxy } from './utils/proxy'
 
 export default function parseOptions (params) {
   const defaultOptions = {
     skipContentModel: false,
     skipLocales: false,
-    skipContentPublishing: false
+    skipContentPublishing: false,
+    useVerboseRenderer: false,
+    prePublishDelay: 3000
   }
 
   const configFile = params.config
@@ -47,6 +52,14 @@ export default function parseOptions (params) {
     throw new Error('Please provide the proxy config in the following format:\nhost:port or user:password@host:port')
   }
 
+  options.startTime = moment()
+
+  if (!options.errorLogFile) {
+    options.errorLogFile = resolve(process.cwd(), `contentful-import-error-log-${options.spaceId}-${options.startTime.format('YYYY-MM-DDTHH-mm-SS')}.json`)
+  } else {
+    options.errorLogFile = resolve(process.cwd(), options.errorLogFile)
+  }
+
   // Further processing
   options.destinationSpace = options.spaceId
   options.destinationManagementToken = options.managementToken

lib/usageParams.js

@@ -53,4 +53,5 @@ export default yargs
     type: 'string'
   })
   .config('config', 'An optional configuration JSON file containing all the options for a single run')
+  .epilog('This is a Beta release')
   .argv