🎨 Support LTS imports (#8498)

refs #8141

- update importer for LTS fields
- optimise for LTS export fixtures
- add image/language test for LTS import
- ensure post image is mapped to feature_image
- create mobiledoc values from markdown and html
- if mobiledoc is null, use markdown or html to create a mobiledoc markdown card
- update import mapping to use locale
- defaultLang in settings now maps to default_locale
- language for post and user models now maps to locale
- posts are not always loaded in correct same order so we select the posts we want to validate
- ensure if mobiledoc field is not in export we can still import from markdown
- map last_login to last_seen
- for users the importer maps last_login to last_seen
- add warning for legacyActiveTheme
- for export with old activeTheme key provide a warning that theme is not installed
- add importer test for LTS user long email
- add a test for LTS export where email address could be longer than alpha
- fix for importer date tests on mysql
- use valueOf in moment to compare times stored in different formats
- ignore warnings for not found settings in import
- use a flag to ignore NotFound Entries for settings during import

core/server/data/importer/importers/data/base.js

@@ -16,7 +16,15 @@ class Base {
 
         this.errorConfig = {
             allowDuplicates: true,
-            returnDuplicates: true
+            returnDuplicates: true,
+            showNotFoundWarning: true
+        };
+
+        this.legacyKeys = {};
+        this.legacyMapper = function legacyMapper(item) {
+            return _.mapKeys(item, function matchLegacyKey(value, key) {
+                return self.legacyKeys[key] || key;
+            });
         };
 
         this.dataKeyToImport = options.dataKeyToImport;
@@ -79,12 +87,14 @@ class Base {
                     }));
                 }
             } else if (err instanceof errors.NotFoundError) {
-                problems.push({
-                    message: 'Entry was not imported and ignored. Could not find entry.',
-                    help: self.modelName,
-                    context: JSON.stringify(obj),
-                    err: err
-                });
+                if (self.showNotFoundWarning) {
+                    problems.push({
+                        message: 'Entry was not imported and ignored. Could not find entry.',
+                        help: self.modelName,
+                        context: JSON.stringify(obj),
+                        err: err
+                    });
+                }
             } else {
                 if (!errors.utils.isIgnitionError(err)) {
                     err = new errors.DataImportError({

core/server/data/importer/importers/data/posts.js

@@ -13,6 +13,10 @@ class PostsImporter extends BaseImporter {
             dataKeyToImport: 'posts',
             requiredData: ['tags', 'posts_tags']
         }));
+
+        this.legacyKeys = {
+            image: 'feature_image'
+        };
     }
 
     sanitizeAttributes() {
@@ -99,10 +103,42 @@ class PostsImporter extends BaseImporter {
 
     beforeImport() {
         debug('beforeImport');
+        let mobileDocContent, self = this;
 
         this.sanitizeAttributes();
         this.addTagsToPosts();
 
+        // Remove legacy field language
+        this.dataToImport = _.filter(this.dataToImport, function (data) {
+            return _.omit(data, 'language');
+        });
+
+        this.dataToImport = this.dataToImport.map(self.legacyMapper);
+
+        // For legacy imports/custom imports with only html we can parse the markdown or html into a mobile doc card
+        // For now we can hardcode the version
+        _.each(this.dataToImport, function (model) {
+            if (!model.mobiledoc) {
+                if (model.markdown && model.markdown.length > 0) {
+                    mobileDocContent = model.markdown;
+                } else if (model.html && model.html.length > 0) {
+                    mobileDocContent = model.html;
+                } else {
+                    // Set mobileDocContent to null else it will affect empty posts
+                    mobileDocContent = null;
+                }
+                if (mobileDocContent) {
+                    model.mobiledoc = JSON.stringify({
+                        version: '0.3.1',
+                        markups: [],
+                        atoms: [],
+                        cards: [['card-markdown',{cardName: 'card-markdown',markdown: mobileDocContent}]],
+                        sections:[[10,0]]
+                    });
+                }
+            }
+        });
+
         // NOTE: do after, because model properties are deleted e.g. post.id
         return super.beforeImport();
     }

core/server/data/importer/importers/data/settings.js

@@ -14,9 +14,17 @@ class SettingsImporter extends BaseImporter {
             requiredData: []
         }));
 
-        this.legacyKeys = {
-            activePlugins: 'active_apps',
-            installedPlugins: 'installed_apps'
+        this.errorConfig = {
+            allowDuplicates: true,
+            returnDuplicates: true,
+            showNotFoundWarning: false
+        };
+
+        // Map legacy keys
+        this.legacySettingsKeyValues = {
+            isPrivate: 'is_private',
+            activeTimezone: 'active_timezone',
+            cover: 'cover_image'
         };
     }
 
@@ -27,14 +35,25 @@ class SettingsImporter extends BaseImporter {
     beforeImport() {
         debug('beforeImport');
 
-        let self = this;
+        let self = this,
+            ltsActiveTheme = _.find(this.dataToImport, {key: 'activeTheme'});
+
+        // If there is an lts we want to warn user that theme is not imported
+        if (ltsActiveTheme) {
+            self.problems.push({
+                message: 'Theme not imported, please upload in Settings - Design',
+                help: self.modelName,
+                context: JSON.stringify(ltsActiveTheme)
+            });
+        }
 
+        // Remove core and theme data types
         this.dataToImport = _.filter(this.dataToImport, function (data) {
             return ['core', 'theme'].indexOf(data.type) === -1;
         });
 
         _.each(this.dataToImport, function (obj) {
-            obj.key = self.legacyKeys[obj.key] || obj.key;
+            obj.key = self.legacySettingsKeyValues[obj.key] || obj.key;
         });
 
         return super.beforeImport();

core/server/data/importer/importers/data/tags.js

@@ -13,6 +13,11 @@ class TagsImporter extends BaseImporter {
             dataKeyToImport: 'tags',
             requiredData: []
         }));
+
+        // Map legacy keys
+        this.legacyKeys = {
+            image: 'feature_image'
+        };
     }
 
     beforeImport() {
@@ -32,6 +37,8 @@ class TagsImporter extends BaseImporter {
 
         let self = this, ops = [];
 
+        this.dataToImport = this.dataToImport.map(self.legacyMapper);
+
         _.each(this.dataToImport, function (obj) {
             ops.push(models[self.modelName].findOne({name: obj.name}, options).then(function (tag) {
                 if (tag) {

core/server/data/importer/importers/data/users.js

@@ -12,6 +12,13 @@ class UsersImporter extends BaseImporter {
             dataKeyToImport: 'users',
             requiredData: ['roles', 'roles_users']
         }));
+
+        // Map legacy keys
+        this.legacyKeys = {
+            image: 'profile_image',
+            cover: 'cover_image',
+            last_login: 'last_seen'
+        };
     }
 
     /**
@@ -25,6 +32,13 @@ class UsersImporter extends BaseImporter {
 
         let self = this, role;
 
+        // Remove legacy field language
+        this.dataToImport = _.filter(this.dataToImport, function (data) {
+            return _.omit(data, 'language');
+        });
+
+        this.dataToImport = this.dataToImport.map(self.legacyMapper);
+
         _.each(this.dataToImport, function (model) {
             model.password = globalUtils.uid(50);
             model.status = 'locked';