Skip to content

Commit

Permalink
Merge pull request #2545 from botpress/ya-cms-export
Browse files Browse the repository at this point in the history
feat(cms): import/export of content
  • Loading branch information
allardy committed Nov 13, 2019
2 parents 22ef2de + bfaf697 commit 4ae2514
Show file tree
Hide file tree
Showing 6 changed files with 351 additions and 7 deletions.
74 changes: 72 additions & 2 deletions src/bp/core/routers/bots/content.ts
@@ -1,9 +1,14 @@
import { ContentElement, Logger } from 'botpress/sdk'
import AuthService, { TOKEN_AUDIENCE } from 'core/services/auth/auth-service'
import { DefaultSearchParams } from 'core/services/cms'
import { InvalidOperationError } from 'core/services/auth/errors'
import { CMSService } from 'core/services/cms'
import { CmsImportSchema, DefaultSearchParams } from 'core/services/cms'
import { WorkspaceService } from 'core/services/workspace-service'
import { RequestHandler, Router } from 'express'
import { validate } from 'joi'
import _ from 'lodash'
import moment from 'moment'
import multer from 'multer'

import { CustomRouter } from '../customRouter'
import { checkTokenHeader, needPermissions } from '../util'
Expand All @@ -13,7 +18,7 @@ export class ContentRouter extends CustomRouter {
private _needPermissions: (operation: string, resource: string) => RequestHandler

constructor(
logger: Logger,
private logger: Logger,
private authService: AuthService,
private cms: CMSService,
private workspaceService: WorkspaceService
Expand Down Expand Up @@ -129,6 +134,71 @@ export class ContentRouter extends CustomRouter {
res.sendStatus(200)
})
)

this.router.get(
'/export',
this._checkTokenHeader,
this._needPermissions('read', 'bot.content'),
async (req, res) => {
// TODO: chunk elements if there are too many of them
const elements = await this.cms.getAllElements(req.params.botId)
const filtered = elements.map(x => _.omit(x, ['createdBy', 'createdOn', 'modifiedOn']))

res.setHeader('Content-Type', 'application/json')
res.setHeader('Content-disposition', `attachment; filename=content_${moment().format('DD-MM-YYYY')}.json`)
res.end(JSON.stringify(filtered, undefined, 2))
}
)

const upload = multer()
this.router.post(
'/analyzeImport',
this._checkTokenHeader,
this._needPermissions('write', 'bot.content'),
upload.single('file'),
this.asyncMiddleware(async (req: any, res) => {
try {
const existingElements = await this.cms.getAllElements(req.params.botId)
const contentTypes = (await this.cms.getAllContentTypes(req.params.botId)).map(x => x.id)

const importData = (await validate(JSON.parse(req.file.buffer), CmsImportSchema)) as ContentElement[]
const importedContentTypes = _.uniq(importData.map(x => x.contentType))

res.send({
cmsCount: (existingElements && existingElements.length) || 0,
fileCmsCount: (importData && importData.length) || 0,
missingContentTypes: _.difference(importedContentTypes, contentTypes)
})
} catch (err) {
throw new InvalidOperationError(`Error importing your file: ${err}`)
}
})
)

this.router.post(
'/import',
this._checkTokenHeader,
this._needPermissions('write', 'bot.content'),
upload.single('file'),
async (req: any, res) => {
if (req.body.action === 'clear_insert') {
await this.cms.deleteAllElements(req.params.botId)
}

try {
const importData: ContentElement[] = await JSON.parse(req.file.buffer)

for (const { contentType, formData, id } of importData) {
await this.cms.createOrUpdateContentElement(req.params.botId, contentType, formData, id)
}

await this.cms.loadElementsForBot(req.params.botId)
res.sendStatus(200)
} catch (e) {
this.logger.attachError(e).error('JSON Import Failure')
}
}
)
}

private _augmentElement = async (element: ContentElement) => {
Expand Down
23 changes: 22 additions & 1 deletion src/bp/core/services/cms.ts
Expand Up @@ -4,6 +4,7 @@ import { KnexExtension } from 'common/knex'
import { renderRecursive, renderTemplate } from 'core/misc/templating'
import { ModuleLoader } from 'core/module-loader'
import { inject, injectable, tagged } from 'inversify'
import Joi from 'joi'
import Knex from 'knex'
import _ from 'lodash'
import nanoid from 'nanoid'
Expand All @@ -26,6 +27,14 @@ export const DefaultSearchParams: SearchParams = {
count: 50
}

export const CmsImportSchema = Joi.array().items(
Joi.object().keys({
id: Joi.string().required(),
contentType: Joi.string().required(),
formData: Joi.object().required()
})
)

@injectable()
export class CMSService implements IDisposeOnExit {
broadcastAddElement: Function = this.local__addElementToCache
Expand Down Expand Up @@ -83,7 +92,7 @@ export class CMSService implements IDisposeOnExit {
})
}

async loadElementsForBot(botId: string): Promise<any[]> {
async getAllElements(botId: string): Promise<ContentElement[]> {
const fileNames = await this.ghost.forBot(botId).directoryListing(this.elementsDir, '*.json')
let contentElements: ContentElement[] = []

Expand All @@ -97,6 +106,12 @@ export class CMSService implements IDisposeOnExit {
contentElements = _.concat(contentElements, fileContentElements)
}

return contentElements
}

async loadElementsForBot(botId: string): Promise<any[]> {
const contentElements = await this.getAllElements(botId)

const elements = await Promise.map(contentElements, element => {
return this.memDb(this.contentTable)
.insert(this.transformItemApiToDb(botId, element))
Expand All @@ -111,6 +126,12 @@ export class CMSService implements IDisposeOnExit {
return elements
}

async deleteAllElements(botId: string): Promise<void> {
const files = await this.ghost.forBot(botId).directoryListing(this.elementsDir, '*.json')
await Promise.map(files, file => this.ghost.forBot(botId).deleteFile(this.elementsDir, file))
await this.clearElementsFromCache(botId)
}

async clearElementsFromCache(botId: string) {
await this.memDb(this.contentTable)
.where({ botId })
Expand Down
226 changes: 226 additions & 0 deletions src/bp/ui-studio/src/web/views/Content/ImportModal.tsx
@@ -0,0 +1,226 @@
import { Button, Callout, Classes, Dialog, FileInput, FormGroup, Intent, Radio, RadioGroup } from '@blueprintjs/core'
import axios from 'axios'
import 'bluebird-global'
import _ from 'lodash'
import React, { FC, Fragment, useState } from 'react'
import { AccessControl, toastFailure } from '~/components/Shared/Utils'

const axiosConfig = { headers: { 'Content-Type': 'multipart/form-data' } }

interface Props {
onImportCompleted: () => void
}

interface Analysis {
cmsCount: number
fileCmsCount: number
}

export const ImportModal: FC<Props> = props => {
const [file, setFile] = useState<any>()
const [filePath, setFilePath] = useState<string>()
const [isLoading, setIsLoading] = useState(false)
const [isDialogOpen, setDialogOpen] = useState(false)
const [importAction, setImportAction] = useState('insert')
const [analysis, setAnalysis] = useState<Analysis>()
const [uploadStatus, setUploadStatus] = useState<string>()
const [hasError, setHasError] = useState(false)

const analyzeImport = async () => {
setIsLoading(true)
try {
const form = new FormData()
form.append('file', file)

const { data } = await axios.post(`${window.BOT_API_PATH}/content/analyzeImport`, form, axiosConfig)

if (!data.fileCmsCount) {
setUploadStatus(`We were not able to extract any data from your file.
Either the file is empty, or it doesn't match any known format.`)
setHasError(true)
}

if (data.missingContentTypes.length) {
setUploadStatus(`Your bot is missing these content types: ${data.missingContentTypes.join(', ')}.`)
setHasError(true)
}

setAnalysis(data)
} catch (err) {
toastFailure(_.get(err, 'response.data.message', err.message))
} finally {
setIsLoading(false)
}
}

const submitChanges = async () => {
setIsLoading(true)

try {
const form = new FormData()
form.append('file', file)
form.append('action', importAction)

await axios.post(`${window.BOT_API_PATH}/content/import`, form, axiosConfig)
closeDialog()
props.onImportCompleted()
} catch (err) {
clearStatus()
setHasError(true)
toastFailure(err.message)
} finally {
setIsLoading(false)
}
}

const readFile = (files: FileList | null) => {
if (files) {
setFile(files[0])
setFilePath(files[0].name)
}
}

const clearStatus = () => {
setIsLoading(false)
}

const closeDialog = () => {
clearState()
setDialogOpen(false)
}

const clearState = () => {
setFilePath(undefined)
setFile(undefined)
setUploadStatus(undefined)

setAnalysis(undefined)
setHasError(false)
}

const renderUpload = () => {
return (
<div
onDragOver={e => e.preventDefault()}
onDrop={e => {
e.preventDefault()
readFile(e.dataTransfer.files)
}}
>
<div className={Classes.DIALOG_BODY}>
<FormGroup
label={<span>Select your JSON file</span>}
labelFor="input-json"
helperText={
<span>
Select a JSON file. It must be exported from the Content page. You will see a summary of modifications
when clicking on Next
</span>
}
>
<FileInput
text={filePath || 'Choose file...'}
onChange={e => readFile((e.target as HTMLInputElement).files)}
inputProps={{ accept: '.json' }}
fill={true}
/>
</FormGroup>
</div>
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
<Button
id="btn-next"
text={isLoading ? 'Please wait...' : 'Next'}
disabled={!filePath || !file || isLoading}
onClick={analyzeImport}
intent={Intent.PRIMARY}
/>
</div>
</div>
</div>
)
}

const renderAnalysis = () => {
const { cmsCount, fileCmsCount } = analysis

return (
<Fragment>
<div className={Classes.DIALOG_BODY}>
<div>
<p>
Your file contains <strong>{fileCmsCount}</strong> content elements, while this bot contains{' '}
<strong>{cmsCount}</strong> elements.
</p>

<div style={{ marginTop: 30 }}>
<RadioGroup
label=" What would you like to do? "
onChange={e => setImportAction(e.target['value'])}
selectedValue={importAction}
>
<Radio id="radio-insert" label="Update or create missing elements present in my file" value="insert" />
<Radio
id="radio-clearInsert"
label="Clear all existing elements, them import those from my file"
value="clear_insert"
/>
</RadioGroup>
</div>
</div>
</div>
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
<Button id="btn-back" text={'Back'} disabled={isLoading} onClick={clearState} />
<Button
id="btn-submit"
text={isLoading ? 'Please wait...' : 'Submit'}
disabled={isLoading || hasError}
onClick={submitChanges}
intent={Intent.PRIMARY}
/>
</div>
</div>
</Fragment>
)
}

const renderStatus = () => {
return (
<Fragment>
<div className={Classes.DIALOG_BODY}>
<Callout title={hasError ? 'Error' : 'Upload status'} intent={hasError ? Intent.DANGER : Intent.PRIMARY}>
{uploadStatus}
</Callout>
</div>
<div className={Classes.DIALOG_FOOTER}>
<div className={Classes.DIALOG_FOOTER_ACTIONS}>
{hasError && <Button id="btn-back" text={'Back'} disabled={isLoading} onClick={clearState} />}
</div>
</div>
</Fragment>
)
}

const showStatus = uploadStatus || hasError

return (
<Fragment>
<AccessControl resource="content" operation="write">
<Button icon="download" id="btn-importJson" text="Import JSON" onClick={() => setDialogOpen(true)} />
</AccessControl>

<Dialog
title={analysis ? 'Analysis' : 'Upload File'}
icon="import"
isOpen={isDialogOpen}
onClose={closeDialog}
transitionDuration={0}
canOutsideClickClose={false}
>
{showStatus && renderStatus()}
{!showStatus && (analysis ? renderAnalysis() : renderUpload())}
</Dialog>
</Fragment>
)
}

0 comments on commit 4ae2514

Please sign in to comment.