Added products and Stripe data to exports and imports (#14873)

- The migration path from 4.x on SQLite to 5.0 on MySQL requires an export/import - Exports don't include the Stripe info required to map members to tiers correctly on import. This change fixes that. Co-authored-by: Simon Backx <simon@ghost.org> Co-authored-by: Hannah Wolfe <github.erisds@gmail.com>
TryGhost · May 20, 2022 · eae0a6a · eae0a6a
1 parent d6d6841
commit eae0a6a
Show file tree

Hide file tree

Showing 12 changed files with 506 additions and 26 deletions.
diff --git a/core/server/data/importer/importers/data/base.js b/core/server/data/importer/importers/data/base.js
@@ -304,6 +304,19 @@ class Base {
         });
     }
 
+    /**
+     * @returns {Object}
+     */
+    mapImportedData(originalObject, importedObject) {
+        return {
+            id: importedObject.id,
+            originalId: this.originalIdMap[importedObject.id],
+            slug: importedObject.get('slug'),
+            originalSlug: originalObject.slug,
+            email: importedObject.get('email')
+        };
+    }
+
     doImport(options, importOptions) {
         debug('doImport', this.modelName, this.dataToImport.length);
 
@@ -322,13 +335,9 @@ class Base {
                         }
 
                         // for identifier lookup
-                        this.importedData.push({
-                            id: importedModel.id,
-                            originalId: this.originalIdMap[importedModel.id],
-                            slug: importedModel.get('slug'),
-                            originalSlug: obj.slug,
-                            email: importedModel.get('email')
-                        });
+                        this.importedData.push(
+                            this.mapImportedData(obj, importedModel)
+                        );
 
                         importedModel = null;
                         this.dataToImport.splice(index, 1);

diff --git a/core/server/data/importer/importers/data/data-importer.js b/core/server/data/importer/importers/data/data-importer.js
@@ -10,6 +10,9 @@ const TagsImporter = require('./tags');
 const SettingsImporter = require('./settings');
 const UsersImporter = require('./users');
 const NewslettersImporter = require('./newsletters');
+const ProductsImporter = require('./products');
+const StripeProductsImporter = require('./stripe-products');
+const StripePricesImporter = require('./stripe-prices');
 const RolesImporter = require('./roles');
 let importers = {};
 let DataImporter;
@@ -26,9 +29,12 @@ DataImporter = {
         importers.users = new UsersImporter(importData.data);
         importers.roles = new RolesImporter(importData.data);
         importers.tags = new TagsImporter(importData.data);
-        importers.posts = new PostsImporter(importData.data);
         importers.newsletters = new NewslettersImporter(importData.data);
         importers.settings = new SettingsImporter(importData.data);
+        importers.products = new ProductsImporter(importData.data);
+        importers.stripe_products = new StripeProductsImporter(importData.data);
+        importers.stripe_prices = new StripePricesImporter(importData.data);
+        importers.posts = new PostsImporter(importData.data);
 
         return importData;
     },
@@ -114,6 +120,35 @@ DataImporter = {
                 });
             });
 
+            /**
+             * @TODO: figure out how to fix this properly
+             * fixup the circular reference from
+             * stripe_prices -> stripe_products -> products -> stripe_prices
+             *
+             * Note: the product importer validates that all values are either
+             *   - being imported, or
+             *   - already exist in the db
+             * so we only need to map imported products
+             */
+            ops.push(() => {
+                const importedStripePrices = importers.stripe_prices.importedData;
+                const importedProducts = importers.products.importedData;
+                const productOps = [];
+
+                _.forEach(importedProducts, (importedProduct) => {
+                    return _.forEach(['monthly_price_id', 'yearly_price_id'], (field) => {
+                        const mappedPrice = _.find(importedStripePrices, {originalId: importedProduct[field]});
+                        if (mappedPrice) {
+                            productOps.push(() => {
+                                return models.Product.edit({[field]: mappedPrice.id}, {id: importedProduct.id, transacting});
+                            });
+                        }
+                    });
+                });
+
+                return sequence(productOps);
+            });
+
             return sequence(ops)
                 .then(function () {
                     results.forEach(function (promise) {

diff --git a/core/server/data/importer/importers/data/newsletters.js b/core/server/data/importer/importers/data/newsletters.js
@@ -1,6 +1,7 @@
 const debug = require('@tryghost/debug')('importer:newsletters');
 const _ = require('lodash');
 const BaseImporter = require('./base');
+const models = require('../../../../models');
 
 const ignoredColumns = ['sender_email'];
 
@@ -23,6 +24,13 @@ class NewslettersImporter extends BaseImporter {
         });
     }
 
+    fetchExisting(modelOptions) {
+        return models.Newsletter.findAll(_.merge({columns: ['id']}, modelOptions))
+            .then((existingData) => {
+                this.existingData = existingData.toJSON();
+            });
+    }
+
     beforeImport() {
         debug('beforeImport');
         this.sanitizeValues();

diff --git a/core/server/data/importer/importers/data/posts.js b/core/server/data/importer/importers/data/posts.js
@@ -6,16 +6,23 @@ const mobiledocLib = require('../../../../lib/mobiledoc');
 const validator = require('@tryghost/validator');
 const postsMetaSchema = require('../../../schema').tables.posts_meta;
 const metaAttrs = _.keys(_.omit(postsMetaSchema, ['id']));
-const ignoredColumns = ['newsletter_id'];
 
 class PostsImporter extends BaseImporter {
     constructor(allDataFromFile) {
         super(allDataFromFile, {
             modelName: 'Post',
             dataKeyToImport: 'posts',
-            requiredFromFile: ['posts', 'tags', 'posts_tags', 'posts_authors', 'posts_meta'],
-            requiredImportedData: ['tags'],
-            requiredExistingData: ['tags']
+            requiredFromFile: [
+                'posts',
+                'tags',
+                'posts_tags',
+                'posts_authors',
+                'posts_meta',
+                'products',
+                'posts_products'
+            ],
+            requiredImportedData: ['tags', 'products', 'newsletters'],
+            requiredExistingData: ['tags', 'products', 'newsletters']
         });
     }
 
@@ -42,10 +49,6 @@ class PostsImporter extends BaseImporter {
                 }
                 delete obj.send_email_when_published;
             }
-
-            ignoredColumns.forEach((column) => {
-                delete obj[column];
-            });
         });
     }
 
@@ -64,15 +67,17 @@ class PostsImporter extends BaseImporter {
     }
 
     /**
-     * Naive function to attach related tags and authors.
+     * Naive function to attach related tags, authors, and products.
      */
     addNestedRelations() {
         this.requiredFromFile.posts_tags = _.orderBy(this.requiredFromFile.posts_tags, ['post_id', 'sort_order'], ['asc', 'asc']);
         this.requiredFromFile.posts_authors = _.orderBy(this.requiredFromFile.posts_authors, ['post_id', 'sort_order'], ['asc', 'asc']);
+        this.requiredFromFile.posts_products = _.orderBy(this.requiredFromFile.posts_products, ['post_id', 'sort_order'], ['asc', 'asc']);
 
         /**
          * from {post_id: 1, tag_id: 2} to post.tags=[{id:id}]
          * from {post_id: 1, author_id: 2} post.authors=[{id:id}]
+         * from {post_id: 1, product_id: 2} post.products=[{id:id}]
          */
         const run = (relations, target, fk) => {
             _.each(relations, (relation) => {
@@ -102,6 +107,7 @@ class PostsImporter extends BaseImporter {
 
         run(this.requiredFromFile.posts_tags, 'tags', 'tag_id');
         run(this.requiredFromFile.posts_authors, 'authors', 'author_id');
+        run(this.requiredFromFile.posts_products, 'tiers', 'product_id');
     }
 
     /**
@@ -179,6 +185,29 @@ class PostsImporter extends BaseImporter {
         _.each(this.dataToImport, (postToImport, postIndex) => {
             run(postToImport, postIndex, 'tags', 'tags');
             run(postToImport, postIndex, 'authors', 'users');
+            run(postToImport, postIndex, 'tiers', 'products');
+        });
+
+        // map newsletter_id -> newsletters.id
+        _.each(this.dataToImport, (objectInFile) => {
+            if (!objectInFile.newsletter_id) {
+                return;
+            }
+            const importedObject = _.find(this.requiredImportedData.newsletters, {originalId: objectInFile.newsletter_id});
+            // CASE: we've imported the newsletter
+            if (importedObject) {
+                debug(`replaced newsletter_id ${objectInFile.newsletter_id} with ${importedObject.id}`);
+                objectInFile.newsletter_id = importedObject.id;
+                return;
+            }
+            const existingObject = _.find(this.requiredExistingData.newsletters, {id: objectInFile.newsletter_id});
+            // CASE: newsletter already exists in the db
+            if (existingObject) {
+                return;
+            }
+            // CASE: newsletter doesn't exist; ignore it
+            debug(`newsletter ${objectInFile.newsletter_id} not found; ignoring`);
+            delete objectInFile.newsletter_id;
         });
 
         return super.replaceIdentifiers();

diff --git a/core/server/data/importer/importers/data/products.js b/core/server/data/importer/importers/data/products.js
@@ -0,0 +1,68 @@
+const _ = require('lodash');
+const BaseImporter = require('./base');
+const models = require('../../../../models');
+
+class ProductsImporter extends BaseImporter {
+    constructor(allDataFromFile) {
+        super(allDataFromFile, {
+            modelName: 'Product',
+            dataKeyToImport: 'products',
+            requiredFromFile: ['stripe_prices'],
+            requiredExistingData: ['stripe_prices']
+        });
+    }
+
+    fetchExisting(modelOptions) {
+        return models.Product.findAll(_.merge({columns: ['products.id as id']}, modelOptions))
+            .then((existingData) => {
+                this.existingData = existingData.toJSON();
+            });
+    }
+
+    mapImportedData(originalObject, importedObject) {
+        return {
+            id: importedObject.id,
+            originalId: this.originalIdMap[importedObject.id],
+            monthly_price_id: originalObject.monthly_price_id,
+            yearly_price_id: originalObject.yearly_price_id
+        };
+    }
+
+    validateStripePrice() {
+        // the stripe price either needs to exist in the current db,
+        // or be imported as part of the same import
+        let invalidProducts = [];
+        _.each(['monthly_price_id', 'yearly_price_id'], (field) => {
+            _.each(this.dataToImport, (objectInFile) => {
+                const importedObject = _.find(
+                    this.requiredFromFile.stripe_prices,
+                    {id: objectInFile[field]}
+                );
+                // CASE: we'll import the stripe price later
+                if (importedObject) {
+                    return;
+                }
+                const existingObject = _.find(
+                    this.requiredExistingData.stripe_prices,
+                    {id: objectInFile[field]}
+                );
+                // CASE: stripe price already exists in the DB
+                if (existingObject) {
+                    return;
+                }
+                // CASE: we don't know what stripe price this is for
+                invalidProducts.push(objectInFile.id);
+            });
+        });
+        // ignore prices with invalid products
+        this.dataToImport = this.dataToImport.filter(item => !invalidProducts.includes(item.id));
+    }
+
+    replaceIdentifiers() {
+        // this has to be in replaceIdentifiers because it's after required* fields are set
+        this.validateStripePrice();
+        return super.replaceIdentifiers();
+    }
+}
+
+module.exports = ProductsImporter;
diff --git a/core/server/data/importer/importers/data/settings.js b/core/server/data/importer/importers/data/settings.js
@@ -10,7 +10,7 @@ const keyTypeMapper = require('../../../../api/shared/serializers/input/utils/se
 const {WRITABLE_KEYS_ALLOWLIST} = require('../../../../../shared/labs');
 
 const labsDefaults = JSON.parse(defaultSettings.labs.labs.defaultValue);
-const ignoredSettings = ['slack_url', 'members_from_address', 'members_support_address'];
+const ignoredSettings = ['slack_url', 'members_from_address', 'members_support_address', 'portal_products'];
 
 // Importer maintains as much backwards compatibility as possible
 const renamedSettingsMap = {

diff --git a/core/server/data/importer/importers/data/stripe-prices.js b/core/server/data/importer/importers/data/stripe-prices.js
@@ -0,0 +1,59 @@
+const _ = require('lodash');
+const debug = require('@tryghost/debug')('importer:stripeprices');
+const BaseImporter = require('./base');
+const models = require('../../../../models');
+
+class StripePricesImporter extends BaseImporter {
+    constructor(allDataFromFile) {
+        super(allDataFromFile, {
+            modelName: 'StripePrice',
+            dataKeyToImport: 'stripe_prices',
+            requiredImportedData: ['stripe_products'],
+            requiredExistingData: ['stripe_products']
+        });
+    }
+
+    fetchExisting(modelOptions) {
+        return models.StripePrice.findAll(_.merge({columns: ['id', 'stripe_product_id']}, modelOptions))
+            .then((existingData) => {
+                this.existingData = existingData.toJSON();
+            });
+    }
+
+    validateStripeProduct() {
+        // ensure we have a valid stripe_product_id in the stripe_products table
+        let invalidPrices = [];
+        _.each(this.dataToImport, (objectInFile) => {
+            const importedObject = _.find(
+                this.requiredImportedData.stripe_products,
+                {stripe_product_id: objectInFile.stripe_product_id}
+            );
+            // CASE: we've imported the stripe_product
+            if (importedObject) {
+                return;
+            }
+            const existingObject = _.find(
+                this.requiredExistingData.stripe_products,
+                {stripe_product_id: objectInFile.stripe_product_id}
+            );
+            // CASE: stripe product already exists in the DB
+            if (existingObject) {
+                return;
+            }
+            // CASE: we don't know what stripe product this is for
+            debug(`ignoring invalid product ${objectInFile.stripe_product_id}`);
+            invalidPrices.push(objectInFile.id);
+        });
+        // ignore prices with invalid products
+        debug(`ignoring ${invalidPrices.length} products`);
+        this.dataToImport = this.dataToImport.filter(item => !invalidPrices.includes(item.id));
+    }
+
+    replaceIdentifiers() {
+        // this has to be in replaceIdentifiers because it's after required* fields are set
+        this.validateStripeProduct();
+        return super.replaceIdentifiers();
+    }
+}
+
+module.exports = StripePricesImporter;