diff --git a/.eslintrc.json b/.eslintrc.json index c3d5de9..fe7c18d 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,3 +1,6 @@ { - "extends": [ "apostrophe", "plugin:vue/vue3-recommended" ] + "extends": [ "apostrophe", "plugin:vue/vue3-recommended" ], + "globals": { + "apos": true + } } diff --git a/i18n/en.json b/i18n/en.json index 5088b5f..aedcfc3 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1,5 +1,7 @@ { "export": "Download {{ type }}", + "exportAttachmentError": "Some attachments could not be added to the {{ extension }} file", + "exportFileGenerationError": "The {{ extension }} file generation failed", "exported": "Downloaded {{ count }} {{ type }}", "exporting": "Downloading {{ type }}...", "exportModalDescription": "You've selected {{ count }} {{ type }} for download", diff --git a/lib/formats/archiver.js b/lib/formats/archiver.js index d9b4b83..ffda2ba 100644 --- a/lib/formats/archiver.js +++ b/lib/formats/archiver.js @@ -4,9 +4,10 @@ module.exports = { compress }; -function compress(filepath, data, archive) { +function compress(apos, filepath, data, archive) { return new Promise((resolve, reject) => { const output = createWriteStream(filepath); + let response; archive.on('warning', function(err) { if (err.code === 'ENOENT') { @@ -16,21 +17,59 @@ function compress(filepath, data, archive) { } }); archive.on('error', reject); - archive.on('finish', resolve); - + archive.on('finish', () => { + resolve(response); + }); archive.pipe(output); - for (const filename in data) { - const content = data[filename]; + for (const [ filename, content ] of Object.entries(data.json || {})) { + archive.append(content, { name: filename }); + } - if (content.endsWith('/')) { - archive.directory(content, filename); - continue; - } + compressAttachments(apos, archive, data.attachments || {}) + .then((res) => { + response = res; + archive.finalize(); + }); + }); +} - archive.append(content, { name: filename }); +async function compressAttachments(apos, archive, attachments = {}) { + const promises = Object.entries(attachments).map(([ name, url ]) => { + return new Promise((resolve, reject) => { + apos.http.get(url, { originalResponse: true }) + .then((res) => { + res.body.on('error', reject); + res.body.on('end', resolve); + + archive.append(res.body, { + name: `attachments/${name}` + }); + }) + .catch(reject); + }); + }); + + const chunkedPromises = chunkPromises(promises, 5); + let attachmentError = false; + + for (const chunk of chunkedPromises) { + const results = await Promise.allSettled(chunk); + if (results.some(({ status }) => status === 'rejected')) { + attachmentError = true; } + } + + return { attachmentError }; +} - archive.finalize(); +function chunkPromises(attachments, max) { + const length = Math.ceil(attachments.length / max); + return Array(length).fill([]).map((chunk, i) => { + const position = i * max; + return [ + ...chunk, + ...attachments.slice(position, position + max) + ]; }); } diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index 962bbf5..1c0b675 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -4,9 +4,9 @@ const { compress } = require('./archiver'); module.exports = { label: 'gzip', extension: 'tar.gz', - output(filepath, data) { + output(apos, filepath, data) { const archive = archiver('tar', { gzip: true }); - return compress(filepath, data, archive); + return compress(apos, filepath, data, archive); } }; diff --git a/lib/formats/zip.js b/lib/formats/zip.js index 0d56da2..192ead9 100644 --- a/lib/formats/zip.js +++ b/lib/formats/zip.js @@ -3,9 +3,9 @@ const { compress } = require('./archiver'); module.exports = { label: 'Zip', - output(filepath, data) { + output(apos, filepath, data) { const archive = archiver('zip'); - return compress(filepath, data, archive); + return compress(apos, filepath, data, archive); } }; diff --git a/lib/methods/archive.js b/lib/methods/archive.js index 62cbb1a..b24a581 100644 --- a/lib/methods/archive.js +++ b/lib/methods/archive.js @@ -4,7 +4,7 @@ const util = require('util'); module.exports = self => { return { - async createArchive(req, reporting, docs, attachments, options) { + async createArchive(req, reporting, data, options) { const { extension, format, @@ -16,16 +16,25 @@ module.exports = self => { const filename = `${self.apos.util.generateId()}-export.${specificExtension || extension}`; const filepath = path.join(self.apos.attachment.uploadfs.getTempPath(), filename); - const docsData = JSON.stringify(docs, undefined, 2); - const attachmentsData = JSON.stringify(attachments, undefined, 2); - - const data = { - 'aposDocs.json': docsData, - 'aposAttachments.json': attachmentsData - // attachments: 'attachments/' // TODO: add attachment into an "/attachments" folder - }; - - await format.output(filepath, data); + try { + const { attachmentError } = await format.output(self.apos, filepath, data); + if (attachmentError) { + await self.apos.notification.trigger(req, 'aposImportExport:exportAttachmentError', { + interpolate: { extension }, + dismiss: true, + icon: 'alert-circle-icon', + type: 'warning' + }); + } + } catch ({ message }) { + self.apos.error('error', message); + await self.apos.notification.trigger(req, 'aposImportExport:exportFileGenerationError', { + interpolate: { extension }, + dismiss: true, + icon: 'alert-circle-icon', + type: 'error' + }); + } // Must copy it to uploadfs, the server that created it // and the server that delivers it might be different diff --git a/lib/methods/export.js b/lib/methods/export.js index e50c4c1..cd5ac78 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -1,5 +1,4 @@ -// TODO: remove: -const attachmentsMock = [ { foo: 'bar' } ]; +const MAX_RECURSION = 10; module.exports = self => { return { @@ -12,7 +11,6 @@ module.exports = self => { reporting.setTotal(req.body._ids.length); } - // TODO: add batchSize? const ids = self.apos.launder.ids(req.body._ids); const relatedTypes = self.apos.launder.strings(req.body.relatedTypes); const extension = self.apos.launder.string(req.body.extension, 'zip'); @@ -30,7 +28,7 @@ module.exports = self => { if (!relatedTypes.length) { const docs = await self.fetchActualDocs(req, allIds, reporting); - return self.createArchive(req, reporting, docs, attachmentsMock, { + return self.createArchive(req, reporting, this.formatArchiveData(docs), { extension, expiration, format @@ -44,17 +42,52 @@ module.exports = self => { // since they might have different related documents. const relatedIds = draftDocs .concat(publishedDocs) - .flatMap(doc => self.getRelatedDocsIds(manager, doc, relatedTypes)); + .flatMap(doc => { + return self.getRelatedIdsBySchema({ + doc, + schema: self.apos.modules[doc.type].schema, + relatedTypes + }); + }); const allRelatedIds = self.getAllModesIds(relatedIds); const docs = await self.fetchActualDocs(req, [ ...allIds, ...allRelatedIds ], reporting); - return self.createArchive(req, reporting, docs, attachmentsMock, { - extension, - expiration, - format + const attachmentsIds = docs.flatMap(doc => { + return self.getRelatedIdsBySchema({ + doc, + schema: self.apos.modules[doc.type].schema, + type: 'attachment' + }); }); + const attachments = await self.fetchActualDocs(req, attachmentsIds, reporting, 'attachment'); + const attachmentUrls = Object.fromEntries( + attachments.map((attachment) => { + const name = `${attachment._id}-${attachment.name}.${attachment.extension}`; + return [ name, self.apos.attachment.url(attachment, { size: 'original' }) ]; + }) + ); + + return self.createArchive(req, + reporting, + self.formatArchiveData(docs, attachments, attachmentUrls), + { + extension, + expiration, + format + } + ); + }, + + formatArchiveData(docs, attachments = [], urls = {}) { + return { + json: { + 'aposDocs.json': JSON.stringify(docs, undefined, 2), + 'aposAttachments.json': JSON.stringify(attachments, undefined, 2) + }, + attachments: urls + }; }, // Add the published version ID next to each draft ID, @@ -72,10 +105,13 @@ module.exports = self => { // without altering the fields or populating them, as the managers would. // It is ok if docs corresponding to published IDs do not exist in the database, // as they simply will not be fetched. - async fetchActualDocs(req, docsIds, reporting) { - const docsIdsUniq = [ ...new Set(docsIds) ]; + async fetchActualDocs(req, docsIds, reporting, collection = 'doc') { + if (!docsIds.length) { + return []; + } - const docs = await self.apos.doc.db + const docsIdsUniq = [ ...new Set(docsIds) ]; + const docs = await self.apos[collection].db .find({ _id: { $in: docsIdsUniq @@ -115,56 +151,64 @@ module.exports = self => { .toArray(); }, - getRelatedDocsIds(manager, doc, relatedTypes) { - // Use `doc.type` for pages to get the actual schema of the corresponding page type. - const schema = manager.schema || self.apos.modules[doc.type].schema; + getRelatedIdsBySchema({ + doc, schema, type = 'relationship', relatedTypes, recursion = 0 + }) { + return schema.flatMap(field => { + const fieldValue = doc[field.name]; + const shouldRecurse = recursion <= MAX_RECURSION; + + if ( + !fieldValue || + (relatedTypes && field.withType && !relatedTypes.includes(field.withType)) + // TODO: handle 'exportDoc: false' option + /* ( */ + /* type === 'relationship' && */ + /* field.withType && */ + /* self.apos.modules[field.withType].options.relatedDocument === false */ + /* ) */ + ) { + return []; + } - return self - .getRelatedBySchema(doc, schema) - .filter(relatedDoc => relatedTypes.includes(relatedDoc.type)) - .map(relatedDoc => relatedDoc._id); - }, + if (shouldRecurse && field.type === 'array') { + return fieldValue.flatMap((subField) => self.getRelatedIdsBySchema({ + doc: subField, + schema: field.schema, + type, + relatedTypes, + recursion: recursion + 1 + })); + } - // TODO: factorize with the one from AposI18nLocalize.vue - // TODO: limit recursion to 10 as we do when retrieving related types? - getRelatedBySchema(object, schema) { - let related = []; - for (const field of schema) { - if (field.type === 'array') { - for (const value of (object[field.name] || [])) { - related = [ - ...related, - ...self.getRelatedBySchema(value, field.schema) - ]; - } - } else if (field.type === 'object') { - if (object[field.name]) { - related = [ - ...related, - ...self.getRelatedBySchema(object[field.name], field.schema) - ]; - } - } else if (field.type === 'area') { - for (const widget of (object[field.name]?.items || [])) { - related = [ - ...related, - ...self.getRelatedBySchema(widget, self.apos.modules[`${widget?.type}-widget`]?.schema || []) - ]; - } - } else if (field.type === 'relationship') { - related = [ - ...related, - ...(object[field.name] || []) - ]; - // Stop here, don't recurse through relationships or we're soon - // related to the entire site + if (shouldRecurse && field.type === 'object') { + return self.getRelatedIdsBySchema({ + doc: fieldValue, + schema: field.schema, + type, + relatedTypes, + recursion: recursion + 1 + }); } - } - // Filter out doc types that opt out completely (pages should - // never be considered "related" to other pages simply because - // of navigation links, the feature is meant for pieces that feel more like - // part of the document being localized) - return related.filter(doc => self.apos.modules[doc.type].relatedDocument !== false); + + if (shouldRecurse && field.type === 'area') { + return (fieldValue.items || []).flatMap((widget) => self.getRelatedIdsBySchema({ + doc: widget, + schema: self.apos.modules[`${widget?.type}-widget`]?.schema || [], + type, + relatedTypes, + recursion: recursion + 1 + })); + } + + if (field.type === type) { + return Array.isArray(fieldValue) + ? fieldValue.map(({ _id }) => _id) + : [ fieldValue._id ]; + } + + return []; + }); } }; }; diff --git a/test/example.js b/test/example.js deleted file mode 100644 index c48856c..0000000 --- a/test/example.js +++ /dev/null @@ -1,44 +0,0 @@ -const assert = require('assert').strict; -const t = require('apostrophe/test-lib/util.js'); - -const getAppConfig = () => { - return { - '@apostrophecms/express': { - options: { - session: { secret: 'supersecret' } - } - }, - '@apostrophecms/module': { - options: { - } - } - }; -}; - -describe('@apostrophecms/module', function () { - let apos; - - this.timeout(t.timeout); - - after(function () { - return t.destroy(apos); - }); - - before(async function() { - apos = await t.create({ - root: module, - baseUrl: 'http://localhost:3000', - testModule: true, - modules: getAppConfig() - }); - }); - - describe('init', function() { - it('should have module enabled', function () { - const actual = Object.keys(apos.modules).includes('@apostrophecms/module'); - const expected = true; - - assert.equal(actual, expected); - }); - }); -}); diff --git a/test/index.js b/test/index.js new file mode 100644 index 0000000..aba9e78 --- /dev/null +++ b/test/index.js @@ -0,0 +1,246 @@ +const assert = require('assert').strict; +const t = require('apostrophe/test-lib/util.js'); +const fs = require('fs'); +const path = require('path'); +const FormData = require('form-data'); + +describe('@apostrophecms/import-export', function () { + let apos; + + this.timeout(t.timeout); + + this.afterEach(function () { + return t.destroy(apos); + }); + + after(function() { + const attachmentPath = path.join(apos.rootDir, 'public/uploads/attachments'); + fs.readdirSync(attachmentPath).forEach((name) => { + fs.unlinkSync(path.join(attachmentPath, name)); + }); + }); + + beforeEach(async function() { + apos = await t.create({ + root: module, + baseUrl: 'http://localhost:3000', + testModule: true, + modules: getAppConfig() + }); + await insertPieces(apos); + }); + + it('should have module enabled', function () { + const actual = Object.keys(apos.modules).includes('@apostrophecms/module'); + const expected = true; + + assert.equal(actual, expected); + }); +}); + +async function insertPieces(apos) { + const req = apos.task.getReq(); + + const topic3 = await apos.topic.insert(req, { + ...apos.topic.newInstance(), + title: 'topic3' + }); + + const topic2 = await apos.topic.insert(req, { + ...apos.topic.newInstance(), + title: 'topic2' + }); + + const topic1 = await apos.topic.insert(req, { + ...apos.topic.newInstance(), + title: 'topic1', + _topics: [ topic3 ] + }); + + await apos.article.insert(req, { + ...apos.article.newInstance(), + title: 'article1', + _topics: [ topic2 ] + }); + + await apos.article.insert(req, { + ...apos.article.newInstance(), + title: 'article1', + _topics: [ topic1 ] + }); + + await insertAdminUser(apos); + + const formData = new FormData(); + formData.append( + 'file', + fs.createReadStream(path.join(apos.rootDir, '/public/test-image.jpg')) + ); + + const jar = await login(apos.http); + + const attachment = await apos.http.post('/api/v1/@apostrophecms/attachment/upload', { + body: formData, + jar + }); + + const image1 = await apos.image.insert(req, { + ...apos.image.newInstance(), + title: 'image1', + attachment + }); + + const pageInstance = await apos.http.post('/api/v1/@apostrophecms/page', { + body: { + _newInstance: true + }, + jar + }); + + await apos.page.insert(req, '_home', 'lastChild', { + ...pageInstance, + title: 'page1', + type: 'default-page', + main: { + _id: 'areaId', + items: [ + { + _id: 'cllp5ubqk001320613gaz2vmv', + metaType: 'widget', + type: '@apostrophecms/image', + aposPlaceholder: false, + imageIds: [ image1.aposDocId ], + imageFields: { + [image1.aposDocId]: { + top: null, + left: null, + width: null, + height: null, + x: null, + y: null + } + } + } + ], + metaType: 'area' + } + }); +} + +async function login(http, username = 'admin') { + const jar = http.jar(); + + await http.post('/api/v1/@apostrophecms/login/login', { + body: { + username, + password: username, + session: true + }, + jar + }); + + const loggedPage = await http.get('/', { + jar + }); + + assert(loggedPage.match(/logged in/)); + + return jar; +} + +async function insertAdminUser(apos) { + const user = { + ...apos.user.newInstance(), + title: 'admin', + username: 'admin', + password: 'admin', + role: 'admin' + }; + + await apos.user.insert(apos.task.getReq(), user); +} + +function getAppConfig() { + return { + '@apostrophecms/express': { + options: { + session: { secret: 'supersecret' } + } + }, + '@apostrophecms/import-export': {}, + 'home-page': { + extend: '@apostrophecms/page-type' + }, + 'default-page': { + extend: '@apostrophecms/page-type', + fields: { + add: { + main: { + label: 'Main', + type: 'area', + options: { + widgets: { + '@apostrophecms/rich-text': {}, + '@apostrophecms/image': {}, + '@apostrophecms/video': {} + } + } + }, + _articles: { + label: 'Articles', + type: 'relationship', + withType: 'article' + } + } + } + }, + + article: { + extend: '@apostrophecms/piece-type', + options: { + alias: 'article' + }, + fields: { + add: { + main: { + label: 'Main', + type: 'area', + options: { + widgets: { + '@apostrophecms/rich-text': {}, + '@apostrophecms/image': {}, + '@apostrophecms/video': {} + } + } + }, + _topics: { + label: 'Topics', + type: 'relationship', + withType: 'topic' + } + } + } + }, + + topic: { + extend: '@apostrophecms/piece-type', + options: { + alias: 'topic' + }, + fields: { + add: { + description: { + label: 'Description', + type: 'string' + }, + _topics: { + label: 'Related Topics', + type: 'relationship', + withType: 'topic', + max: 2 + } + } + } + } + }; +} diff --git a/test/modules/@apostrophecms/home-page/views/page.html b/test/modules/@apostrophecms/home-page/views/page.html new file mode 100644 index 0000000..fb48504 --- /dev/null +++ b/test/modules/@apostrophecms/home-page/views/page.html @@ -0,0 +1,16 @@ +{{ data.page.title }} +

Home Page Template

+{# This is necessary to the login.js tests. -Tom #} +{% if data.user %} + logged in as {{ data.user.title }} +{% else %} + logged out +{% endif %} +{# Necessary to the @apostrophecms/global tests. #} +counts: {{ data.global.counts }} +

Translations

+ diff --git a/test/package.json b/test/package.json index a627685..0b27276 100644 --- a/test/package.json +++ b/test/package.json @@ -4,6 +4,6 @@ " */": "exist in package.json at project level, which for a test is here", "dependencies": { "apostrophe": "git+https://github.com/apostrophecms/apostrophe.git", - "@apostrophecms/module": "git+https://github.com/apostrophecms/module.git" + "@apostrophecms/import-export": "git+https://github.com/apostrophecms/import-export.git" } } diff --git a/test/public/test-image.jpg b/test/public/test-image.jpg new file mode 100644 index 0000000..236c9c4 Binary files /dev/null and b/test/public/test-image.jpg differ diff --git a/ui/apos/components/AposExportModal.vue b/ui/apos/components/AposExportModal.vue index eebfe6e..c2317b2 100644 --- a/ui/apos/components/AposExportModal.vue +++ b/ui/apos/components/AposExportModal.vue @@ -17,7 +17,12 @@

- {{ $t('aposImportExport:exportModalDescription', { count, type: moduleLabel }) }} + {{ + $t('aposImportExport:exportModalDescription', { + count: selectedDocIds.length, + type: moduleLabel + }) + }}

@@ -143,6 +148,10 @@ export default { type: Array, default: () => [] }, + doc: { + type: Object, + default: null + }, action: { type: String, required: true @@ -169,14 +178,18 @@ export default { relatedTypes: null, checkedRelatedTypes: [], type: this.moduleName, - extension: 'zip' + extension: 'zip', + selectedDocIds: [] }; }, computed: { moduleLabel() { - const moduleOptions = window.apos.modules[this.moduleName]; - const label = this.checked.length > 1 ? moduleOptions.pluralLabel : moduleOptions.label; + const moduleOptions = apos.modules[this.moduleName]; + const label = this.count > 1 + ? moduleOptions.pluralLabel + : moduleOptions.label; + return this.$t(label).toLowerCase(); }, @@ -190,7 +203,7 @@ export default { }, count() { - return this.checked.length || 1; + return this.selectedDocIds.length; }, extensions() { @@ -200,9 +213,13 @@ export default { async mounted() { this.modal.active = true; + this.selectedDocIds = [ + ...this.checked, + ...this.doc ? [ this.doc._id ] : [] + ]; if (this.type === '@apostrophecms/page') { - this.type = this.$attrs.doc?.type; + this.type = this.doc?.type; } }, @@ -211,10 +228,6 @@ export default { this.$refs.exportDocs.$el.querySelector('button').focus(); }, async exportDocs() { - const docsId = this.checked.length - ? this.checked - : [ this.$attrs.doc?._id ]; - const relatedTypes = this.relatedDocumentsDisabled ? [] : this.checkedRelatedTypes; @@ -223,7 +236,7 @@ export default { const result = await window.apos.http.post(`${action}/${this.action}`, { busy: true, body: { - _ids: docsId, + _ids: this.selectedDocIds, relatedTypes, messages: this.messages, extension: this.extension @@ -241,14 +254,16 @@ export default { this.relatedDocumentsDisabled = !this.relatedDocumentsDisabled; if (!this.relatedDocumentsDisabled && this.relatedTypes === null) { - this.relatedTypes = await window.apos.http.get('/api/v1/@apostrophecms/import-export/related', { + this.relatedTypes = await apos.http.get('/api/v1/@apostrophecms/import-export/related', { busy: true, qs: { type: this.type } }); this.checkedRelatedTypes = this.relatedTypes; - const height = this.checkedRelatedTypes.length ? this.checkedRelatedTypes.length * CONTAINER_ITEM_HEIGHT + CONTAINER_DESCRIPTION_HEIGHT : CONTAINER_MINIMUM_HEIGHT; + const height = this.checkedRelatedTypes.length + ? this.checkedRelatedTypes.length * CONTAINER_ITEM_HEIGHT + CONTAINER_DESCRIPTION_HEIGHT + : CONTAINER_MINIMUM_HEIGHT; this.$refs.container.style.setProperty('--container-height', `${height}px`); } }, @@ -259,11 +274,12 @@ export default { if (evt.target.checked) { this.checkedRelatedTypes.push(evt.target.value); } else { - this.checkedRelatedTypes = this.checkedRelatedTypes.filter(relatedType => relatedType !== evt.target.value); + this.checkedRelatedTypes = this.checkedRelatedTypes + .filter(relatedType => relatedType !== evt.target.value); } }, getRelatedTypeLabel(moduleName) { - const moduleOptions = window.apos.modules[moduleName]; + const moduleOptions = apos.modules[moduleName]; return this.$t(moduleOptions.label); }, onExtensionChange(value) {