Merge fd84ba6 into 836bbb3

lib/get/get-full-source-space.js

@@ -1,10 +1,13 @@
 import Promise from 'bluebird'
 import Listr from 'listr'
 import verboseRenderer from 'listr-verbose-renderer'
+
+import { logEmitter, logToTaskOutput } from '../utils/logging'
 import sortEntries from '../utils/sort-entries'
 
 const MAX_ALLOWED_LIMIT = 1000
 let pageLimit = MAX_ALLOWED_LIMIT
+
 /**
  * Gets all the content from a space via the management API. This includes
  * content in draft state.
@@ -29,114 +32,128 @@ export default function getFullSourceSpace ({
   return new Listr([
     {
       title: 'Connecting to space',
-      task: (ctx) => {
+      task: (ctx, task) => {
+        const teardownTaskListeners = logToTaskOutput(task)
         return managementClient.getSpace(spaceId)
-        .then((space) => {
-          ctx.space = space
-        })
-        .catch((err) => {
-          console.error(`
-The destination space was not found. This can happen for multiple reasons:
-- If you haven't yet, you should create your space manually.
-- If your destination space is in another organization, and your user from the source space does not have access to it, you'll need to specify separate sourceManagementToken and destinationManagementToken
-
-Full error details below.
-`)
-          throw err
-        })
+          .then((space) => {
+            ctx.space = space
+            teardownTaskListeners()
+          })
       }
     },
     {
       title: 'Fetching content types data',
-      task: (ctx) => {
+      task: (ctx, task) => {
+        const teardownTaskListeners = logToTaskOutput(task)
         return pagedGet(ctx.space, 'getContentTypes')
-        .then(extractItems)
-        .then((items) => {
-          ctx.data.contentTypes = items
-        })
+          .then(extractItems)
+          .then((items) => {
+            ctx.data.contentTypes = items
+            teardownTaskListeners()
+          })
       },
       skip: () => skipContentModel
     },
     {
       title: 'Fetching editor interfaces data',
-      task: (ctx) => {
+      task: (ctx, task) => {
+        const teardownTaskListeners = logToTaskOutput(task)
         return getEditorInterfaces(ctx.data.contentTypes)
-        .then((editorInterfaces) => {
-          ctx.data.editorInterfaces = editorInterfaces.filter((editorInterface) => {
-            return editorInterface !== null
+          .then((editorInterfaces) => {
+            ctx.data.editorInterfaces = editorInterfaces.filter((editorInterface) => {
+              return editorInterface !== null
+            })
+            teardownTaskListeners()
           })
-        })
       },
       skip: (ctx) => skipContentModel || (ctx.data.contentTypes.length === 0 && 'Skipped since no content types downloaded')
     },
     {
       title: 'Fetching content entries data',
-      task: (ctx) => {
+      task: (ctx, task) => {
+        const teardownTaskListeners = logToTaskOutput(task)
         return pagedGet(ctx.space, 'getEntries')
-        .then(extractItems)
-        .then(sortEntries)
-        .then((items) => filterDrafts(items, includeDrafts))
-        .then((items) => {
-          ctx.data.entries = items
-        })
+          .then(extractItems)
+          .then(sortEntries)
+          .then((items) => filterDrafts(items, includeDrafts))
+          .then((items) => {
+            ctx.data.entries = items
+            teardownTaskListeners()
+          })
       },
       skip: () => skipContent
     },
     {
       title: 'Fetching assets data',
-      task: (ctx) => {
+      task: (ctx, task) => {
+        const teardownTaskListeners = logToTaskOutput(task)
         return pagedGet(ctx.space, 'getAssets')
-        .then(extractItems)
-        .then((items) => {
-          ctx.data.assets = items
-        })
+          .then(extractItems)
+          .then((items) => {
+            ctx.data.assets = items
+            teardownTaskListeners()
+          })
       },
       skip: () => skipContent
     },
     {
       title: 'Fetching locales data',
-      task: (ctx) => {
+      task: (ctx, task) => {
+        const teardownTaskListeners = logToTaskOutput(task)
         return pagedGet(ctx.space, 'getLocales')
-        .then(extractItems)
-        .then((items) => {
-          ctx.data.locales = items
-        })
+          .then(extractItems)
+          .then((items) => {
+            ctx.data.locales = items
+            teardownTaskListeners()
+          })
       },
       skip: () => skipContentModel
     },
     {
       title: 'Fetching webhooks data',
-      task: (ctx) => {
+      task: (ctx, task) => {
+        const teardownTaskListeners = logToTaskOutput(task)
         return pagedGet(ctx.space, 'getWebhooks')
-        .then(extractItems)
-        .then((items) => {
-          ctx.data.webhooks = items
-        })
+          .then(extractItems)
+          .then((items) => {
+            ctx.data.webhooks = items
+            teardownTaskListeners()
+          })
       },
       skip: () => skipWebhooks
     },
     {
       title: 'Fetching roles data',
-      task: (ctx) => {
+      task: (ctx, task) => {
+        const teardownTaskListeners = logToTaskOutput(task)
         return pagedGet(ctx.space, 'getRoles')
-        .then(extractItems)
-        .then((items) => {
-          ctx.data.roles = items
-        })
+          .then(extractItems)
+          .then((items) => {
+            ctx.data.roles = items
+            teardownTaskListeners()
+          })
       },
       skip: () => skipRoles
     }
   ], listrOptions)
 }
+
 function getEditorInterfaces (contentTypes) {
-  const editorInterfacePromises = contentTypes.map((contentType) => {
-    // old contentTypes may not have an editor interface but we'll handle in a later stage
-    // but it should not stop getting the data process
-    return contentType.getEditorInterface().catch(() => {
-      return Promise.resolve(null)
-    })
+  return Promise.map(contentTypes, (contentType, index, length) => {
+    return contentType.getEditorInterface()
+      .then((editorInterface) => {
+        logEmitter.emit('info', `Fetched editor interface for ${contentType.name}`)
+        return editorInterface
+      })
+      .catch(() => {
+      // old contentTypes may not have an editor interface but we'll handle in a later stage
+      // but it should not stop getting the data process
+        logEmitter.emit('warning', `No editor interface found for ${contentType}`)
+        return Promise.resolve(null)
+      })
+  }, {
+    concurrency: 6
   })
-  return Promise.all(editorInterfacePromises)
 }
 
 /**
@@ -150,17 +167,20 @@ function pagedGet (space, method, skip = 0, aggregatedResponse = null) {
     limit: pageLimit,
     order: 'sys.createdAt'
   })
-  .then((response) => {
-    if (!aggregatedResponse) {
-      aggregatedResponse = response
-    } else {
-      aggregatedResponse.items = aggregatedResponse.items.concat(response.items)
-    }
-    if (skip + pageLimit <= response.total) {
-      return pagedGet(space, method, skip + pageLimit, aggregatedResponse)
-    }
-    return aggregatedResponse
-  })
+    .then((response) => {
+      if (!aggregatedResponse) {
+        aggregatedResponse = response
+      } else {
+        aggregatedResponse.items = aggregatedResponse.items.concat(response.items)
+      }
+      const page = Math.ceil(skip / pageLimit) + 1
+      const pages = Math.ceil(response.total / pageLimit)
+      logEmitter.emit('info', `Fetched page ${page} of ${pages}`)
+      if (skip + pageLimit <= response.total) {
+        return pagedGet(space, method, skip + pageLimit, aggregatedResponse)
+      }
+      return aggregatedResponse
+    })
 }
 
 function extractItems (response) {

lib/get/get-outdated-destination-content.js

@@ -1,5 +1,4 @@
 import Promise from 'bluebird'
-import log from 'npmlog'
 
 const BATCH_CHAR_LIMIT = 1990
 const BATCH_SIZE_LIMIT = 100
@@ -12,8 +11,6 @@ const BATCH_SIZE_LIMIT = 100
  * and Locales will be retrieved.
  */
 export default function getOutdatedDestinationContent ({managementClient, spaceId, entryIds = [], assetIds = [], webhookIds = [], skipContentModel, skipContent}) {
-  log.info('Checking if destination space already has any content and retrieving it')
-
   return managementClient.getSpace(spaceId)
     .then((space) => {
       return Promise.props({
@@ -24,13 +21,6 @@ export default function getOutdatedDestinationContent ({managementClient, spaceI
         webhooks: []
       })
     }, (err) => {
-      log.error(`
-The destination space was not found. This can happen for multiple reasons:
-- If you haven't yet, you should create your space manually.
-- If your destination space is in another organization, and your user from the source space does not have access to it, you'll need to specify separate sourceManagementToken and destinationManagementToken
-
-Full error details below.
-`)
       throw err
     })
 }

lib/push/assets.js

@@ -1,12 +1,14 @@
 import Promise from 'bluebird'
-import log from 'npmlog'
-import getEntityName from './get-entity-name'
+
+import getEntityName from '../utils/get-entity-name'
+import { logEmitter } from '../utils/logging'
 
 export function processAssets (assets) {
   return Promise.map(assets, (asset) => {
-    log.info(`Processing Asset ${getEntityName(asset)}`)
-    return asset.processForAllLocales().catch(() => {
-      log.info('Error Processing this asset, continuing...')
+    logEmitter.emit('info', `Processing Asset ${getEntityName(asset)}`)
+    return asset.processForAllLocales().catch((err) => {
+      err.entity = asset
+      logEmitter.emit('error', err)
       return Promise.resolve(null)
     })
   }, {concurrency: 4})

lib/push/creation.js

@@ -1,10 +1,10 @@
 import Promise from 'bluebird'
-import log from 'npmlog'
 import {partial} from 'lodash/function'
 import {find} from 'lodash/collection'
 import {assign, get, omitBy, omit} from 'lodash/object'
-import getEntityName from './get-entity-name'
-import errorBuffer from '../utils/error-buffer'
+
+import getEntityName from '../utils/get-entity-name'
+import { logEmitter } from '../utils/logging'
 
 /**
  * Creates a list of entities
@@ -22,6 +22,7 @@ export function createEntities (context, entities, destinationEntities) {
       .then(partial(creationSuccessNotifier, operation))
       .catch(partial(handleCreationErrors, entity))
   }, {concurrency: 6})
+    .then((entries) => entries.filter((entry) => entry))
 }
 
 /**
@@ -30,7 +31,7 @@ export function createEntities (context, entities, destinationEntities) {
 export function createEntries (context, entries, destinationEntries) {
   return Promise.map(entries, (entry) => createEntry(
     entry, context.space, context.skipContentModel, destinationEntries),
-    {concurrency: 6})
+  {concurrency: 6})
     .then((entries) => entries.filter((entry) => entry))
 }
 
@@ -83,17 +84,11 @@ function handleCreationErrors (entity, err) {
       return entity
     }
   }
-  if (get(err, 'error.sys.id') === 'VersionMismatch') {
-    log.error('Content update error:')
-    log.error('Error', err.error)
-    log.error('Request', err.request)
-    log.error(`
-This probably means you are synchronizing over a space with previously existing
-content, or that you don't have the sync token for the last sync you performed
-to this space.
-    `)
-  }
-  throw err
+  err.entity = entity.original
+  logEmitter.emit('error', err)
+
+  // No need to pass this entity down to publishing if it wasn't created
+  return null
 }
 
 /**
@@ -110,7 +105,9 @@ function handleEntryCreationErrors (entry, space, skipContentModel, destinationE
   err.originalEntry = entry.original
   err.transformedEntry = entry.transformed
   err.contentModelWasSkipped = skipContentModel
-  errorBuffer.push(err)
+  err.entity = entry.original
+  logEmitter.emit('error', err)
+
   // No need to pass this entry down to publishing if it wasn't created
   return null
 }
@@ -132,7 +129,7 @@ function getDestinationEntityForSourceEntity (destinationEntities, sourceEntity)
 
 function creationSuccessNotifier (method, createdEntity) {
   const verb = method[0].toUpperCase() + method.substr(1, method.length) + 'd'
-  log.info(`${verb} ${createdEntity.sys.type} ${getEntityName(createdEntity)}`)
+  logEmitter.emit('info', `${verb} ${createdEntity.sys.type} ${getEntityName(createdEntity)}`)
   return createdEntity
 }
 

lib/push/index.js

@@ -1,12 +1,9 @@
 import * as assets from './assets'
 import * as creation from './creation'
-import * as deletion from './deletion'
 import * as publishing from './publishing'
 
 export { assets }
 export { creation }
-export { deletion }
 export { publishing }
 
-export { default as getEntityName } from './get-entity-name'
 export { default as pushToSpace } from './push-to-space'