Skip to content

Commit

Permalink
Merge pull request #1940 from botpress/fl_qna_import_export_broken
Browse files Browse the repository at this point in the history 
feat(qna): QNA import and export use JSON format for QNA items
  • Loading branch information
@franklevasseur
franklevasseur committed Jun 20, 2019
2 parents d660594 + a3dda95 commit f11ca7c
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 148 deletions.
59 changes: 16 additions & 43 deletions modules/qna/src/backend/api.ts
View file
@@ -1,20 +1,18 @@
import * as sdk from 'botpress/sdk'
import iconv from 'iconv-lite'
import { validate } from 'joi'
import { Parser as Json2csvParser } from 'json2csv'
import _ from 'lodash'
import moment from 'moment'
import multer from 'multer'
import nanoid from 'nanoid'
import yn from 'yn'

import { QnaEntry } from './qna'
import { QnaEntry, QnaItem } from './qna'
import Storage from './storage'
import { importQuestions, prepareExport } from './transfer'
import { QnaDefSchema } from './validation'
import { QnaDefSchema, QnaItemArraySchema } from './validation'

export default async (bp: typeof sdk, botScopedStorage: Map<string, Storage>) => {
const csvUploadStatuses = {}
const jsonUploadStatuses = {}
const router = bp.http.createRouterForBot('qna')

router.get('/questions', async (req, res) => {
Expand Down Expand Up @@ -93,36 +91,17 @@ export default async (bp: typeof sdk, botScopedStorage: Map<string, Storage>) =>
res.send({ categories })
})

// TODO make sure that this works properly
router.get('/export/:format', async (req, res) => {
router.get('/export', async (req, res) => {
const storage = botScopedStorage.get(req.params.botId)
const config = await bp.config.getModuleConfigForBot('qna', req.params.botId)
const data = await prepareExport(storage, { flat: true })

if (req.params.format === 'csv') {
res.setHeader('Content-Type', 'text/csv')
res.setHeader('Content-disposition', `attachment; filename=qna_${moment().format('DD-MM-YYYY')}.csv`)

const categoryWrapper = storage.hasCategories() ? ['category'] : []
const parseOptions = {
fields: ['question', 'action', 'answer', 'answer2', ...categoryWrapper],
header: true
}
const json2csvParser = new Json2csvParser(parseOptions)

res.end(iconv.encode(json2csvParser.parse(data), config.exportCsvEncoding))
} else {
res.setHeader('Content-Type', 'application/json')
res.setHeader('Content-disposition', `attachment; filename=qna_${moment().format('DD-MM-YYYY')}.json`)
res.end(JSON.stringify(data))
}
const data: string = await prepareExport(storage)
res.setHeader('Content-Type', 'application/json')
res.setHeader('Content-disposition', `attachment; filename=qna_${moment().format('DD-MM-YYYY')}.json`)
res.end(data)
})

// TODO make sure that this works properly
const upload = multer()
router.post('/import/csv', upload.single('csv'), async (req, res) => {
router.post('/import', upload.single('json'), async (req, res) => {
const storage = botScopedStorage.get(req.params.botId)
const config = await bp.config.getModuleConfigForBot('qna', req.params.botId)

const uploadStatusId = nanoid()
res.end(uploadStatusId)
Expand All @@ -136,25 +115,19 @@ export default async (bp: typeof sdk, botScopedStorage: Map<string, Storage>) =>
}

try {
const questions = iconv.decode(req.file.buffer, config.exportCsvEncoding)
const params = {
storage,
config,
format: 'csv',
statusCallback: updateUploadStatus,
uploadStatusId
}
await importQuestions(questions, params)
const parsedJson: any = JSON.parse(req.file.buffer)
const questions = (await validate(parsedJson, QnaItemArraySchema)) as QnaItem[]

await importQuestions(questions, storage, updateUploadStatus, uploadStatusId)
updateUploadStatus(uploadStatusId, 'Completed')
} catch (e) {
bp.logger.attachError(e).error('CSV Import Failure')
bp.logger.attachError(e).error('JSON Import Failure')
updateUploadStatus(uploadStatusId, `Error: ${e.message}`)
}
})

router.get('/csv-upload-status/:uploadStatusId', async (req, res) => {
res.end(csvUploadStatuses[req.params.uploadStatusId])
router.get('/json-upload-status/:uploadStatusId', async (req, res) => {
res.end(jsonUploadStatuses[req.params.uploadStatusId])
})

const sendToastError = (action, error) => {
Expand All @@ -167,6 +140,6 @@ export default async (bp: typeof sdk, botScopedStorage: Map<string, Storage>) =>
if (!uploadStatusId) {
return
}
csvUploadStatuses[uploadStatusId] = status
jsonUploadStatuses[uploadStatusId] = status
}
}
68 changes: 12 additions & 56 deletions modules/qna/src/backend/transfer.ts
View file
@@ -1,66 +1,22 @@
import _ from 'lodash'

import * as parsers from './parsers.js'

const ANSWERS_SPLIT_CHAR = '†'

export const importQuestions = async (questions, params) => {
const { storage, config, format = 'json', statusCallback, uploadStatusId } = params
import { QnaItem, QnaEntry } from './qna'
import Storage from './storage'

export const importQuestions = async (questions: QnaItem[], storage, statusCallback, uploadStatusId) => {
statusCallback(uploadStatusId, 'Calculating diff with existing questions')

const existingQuestions = (await storage.fetchAllQuestions()).map(item =>
JSON.stringify(_.omit(item.data, 'enabled'))
)

const hasCategory = storage.hasCategories()
const parsedQuestions =
typeof questions === 'string' ? parsers[`${format}Parse`](questions, { hasCategory }) : questions
const questionsToSave = parsedQuestions.filter(item => !existingQuestions.includes(JSON.stringify(item)))

if (config.qnaMakerApiKey) {
return storage.insert(questionsToSave.map(question => ({ ...question, enabled: true })))
}
const existingQuestionItems = (await (storage as Storage).fetchQNAs()).map(item => item.id)
const itemsToSave = questions.filter(item => !existingQuestionItems.includes(item.id))
const entriesToSave = itemsToSave.map(q => q.data)

let questionsSavedCount = 0
return Promise.each(questionsToSave, async question => {
const answers = question['answer'].split(ANSWERS_SPLIT_CHAR)
await storage.insert({ ...question, answers, enabled: true })
return Promise.each(entriesToSave, async (question: QnaEntry) => {
await (storage as Storage).insert({ ...question, enabled: true })
questionsSavedCount += 1
statusCallback(uploadStatusId, `Saved ${questionsSavedCount}/${questionsToSave.length} questions`)
statusCallback(uploadStatusId, `Saved ${questionsSavedCount}/${entriesToSave.length} questions`)
})
}

export const prepareExport = async (storage, { flat = false } = {}) => {
const qnas = await storage.fetchAllQuestions()

return _.flatMap(qnas, question => {
const { data } = question
const { questions, action, redirectNode, redirectFlow, category, answers, answer: textAnswer } = data

// FIXME: Remove v11.2 answer support
let answer = answers.join(ANSWERS_SPLIT_CHAR) || textAnswer // textAnswer allow to support v11.2 answer format
let answer2 = undefined

// FIXME: Refactor these answer, answer2 fieds for something more meaningful like a 'redirect' field.
// redirect dont need text so answer is overriden with the redirect flow
if (action === 'redirect') {
answer = redirectFlow
if (redirectNode) {
answer += '#' + redirectNode
}
// text_redirect will display a text before redirecting to the desired flow
} else if (action === 'text_redirect') {
answer2 = redirectFlow
if (redirectNode) {
answer2 += '#' + redirectNode
}
}
const categoryWrapper = storage.getCategories() ? { category } : {}

if (!flat) {
return { questions, action, answer, answer2, ...categoryWrapper }
}
return questions.map(question => ({ question, action, answer, answer2, ...categoryWrapper }))
})
export const prepareExport = async (storage: Storage) => {
const qnas = await storage.fetchQNAs()
return JSON.stringify(qnas, undefined, 2)
}
7 changes: 7 additions & 0 deletions modules/qna/src/backend/validation.ts
View file
Expand Up @@ -17,3 +17,10 @@ export const QnaDefSchema = Joi.object().keys({
.pattern(/.*/, Joi.array().items(Joi.string()))
.default({})
})

const QnaItemSchema = Joi.object().keys({
id: Joi.string().required(),
data: QnaDefSchema
})

export const QnaItemArraySchema = Joi.array().items(QnaItemSchema)
5 changes: 0 additions & 5 deletions modules/qna/src/config.ts
View file
Expand Up @@ -7,11 +7,6 @@ export interface Config {
* @default #builtin_text
*/
textRenderer: string
/**
* @default utf8
*/
exportCsvEncoding: string
qnaMakerApiKey?: string
/**
* @default botpress
*/
Expand Down
88 changes: 44 additions & 44 deletions modules/qna/src/views/full/index.jsx
View file
Expand Up @@ -29,12 +29,12 @@ import 'react-select/dist/react-select.css'
import './button.css'

const ITEMS_PER_PAGE = 50
const CSV_STATUS_POLL_INTERVAL = 1000
const JSON_STATUS_POLL_INTERVAL = 1000

export default class QnaAdmin extends Component {
constructor(props) {
super(props)
this.csvDownloadableLink = React.createRef()
this.jsonDownloadableLink = React.createRef()
}

state = {
Expand Down Expand Up @@ -128,45 +128,45 @@ export default class QnaAdmin extends Component {
.then(({ data: { items, count } }) => this.setState({ items, overallItemsCount: count, page }))
}

uploadCsv = async () => {
uploadJson = async () => {
const formData = new FormData()
formData.set('isReplace', this.state.isCsvUploadReplace)
formData.append('csv', this.state.csvToUpload)
formData.set('isReplace', this.state.isJsonUploadReplace)
formData.append('json', this.state.jsonToUpload)

const headers = { 'Content-Type': 'multipart/form-data' }
const { data: csvStatusId } = await this.props.bp.axios.post('/mod/qna/import/csv', formData, { headers })
const { data: jsonStatusId } = await this.props.bp.axios.post('/mod/qna/import', formData, { headers })

this.setState({ csvStatusId })
this.setState({ jsonStatusId })

while (this.state.csvStatusId) {
while (this.state.jsonStatusId) {
try {
const { data: status } = await this.props.bp.axios.get(`/mod/qna/csv-upload-status/${csvStatusId}`)
const { data: status } = await this.props.bp.axios.get(`/mod/qna/json-upload-status/${jsonStatusId}`)

this.setState({ csvUploadStatus: status })
this.setState({ jsonUploadStatus: status })

if (status === 'Completed') {
this.setState({ csvStatusId: null, importCsvModalShow: false })
this.setState({ jsonStatusId: null, importJsonModalShow: false })
this.fetchData()
} else if (status.startsWith('Error')) {
this.setState({ csvStatusId: null })
this.setState({ jsonStatusId: null })
}

await Promise.delay(CSV_STATUS_POLL_INTERVAL)
await Promise.delay(JSON_STATUS_POLL_INTERVAL)
} catch (e) {
return this.setState({ csvUploadStatus: 'Server Error', csvStatusId: null })
return this.setState({ jsonUploadStatus: 'Server Error', jsonStatusId: null })
}
}
}

downloadCsv = () =>
downloadJson = () =>
// We can't just download file directly due to security restrictions
this.props.bp.axios({ method: 'get', url: '/mod/qna/export/csv', responseType: 'blob' }).then(response => {
this.props.bp.axios({ method: 'get', url: '/mod/qna/export', responseType: 'blob' }).then(response => {
this.setState(
{
csvDownloadableLinkHref: window.URL.createObjectURL(new Blob([response.data])),
csvDownloadableFileName: /filename=(.*\.csv)/.exec(response.headers['content-disposition'])[1]
jsonDownloadableLinkHref: window.URL.createObjectURL(new Blob([response.data])),
jsonDownloadableFileName: /filename=(.*\.json)/.exec(response.headers['content-disposition'])[1]
},
() => this.csvDownloadableLink.current.click()
() => this.jsonDownloadableLink.current.click()
)
})

Expand Down Expand Up @@ -216,40 +216,40 @@ export default class QnaAdmin extends Component {
}

renderImportModal() {
const { csvUploadStatus } = this.state
const { jsonUploadStatus } = this.state

return (
<Modal
show={this.state.importCsvModalShow}
onHide={() => this.setState({ importCsvModalShow: false })}
show={this.state.importJsonModalShow}
onHide={() => this.setState({ importJsonModalShow: false })}
backdrop={'static'}
>
<Modal.Header closeButton>
<Modal.Title>Import CSV</Modal.Title>
<Modal.Title>Import JSON</Modal.Title>
</Modal.Header>
<Modal.Body>
{csvUploadStatus && (
{jsonUploadStatus && (
<Alert
bsStyle={csvUploadStatus.startsWith('Error') ? 'danger' : 'info'}
onDismiss={() => this.setState({ csvUploadStatus: null })}
bsStyle={jsonUploadStatus.startsWith('Error') ? 'danger' : 'info'}
onDismiss={() => this.setState({ jsonUploadStatus: null })}
>
<p>{this.state.csvUploadStatus}</p>
<p>{this.state.jsonUploadStatus}</p>
</Alert>
)}
<form>
<FormGroup>
<ControlLabel>CSV file</ControlLabel>
<ControlLabel>JSON file</ControlLabel>
<FormControl
type="file"
accept=".csv"
onChange={e => this.setState({ csvToUpload: e.target.files[0] })}
accept=".json"
onChange={e => this.setState({ jsonToUpload: e.target.files[0] })}
/>
<HelpBlock>CSV should be formatted &quot;question,answer_type,answer,answer2,category&quot;</HelpBlock>
<HelpBlock>JSON should be formatted &quot;question,answer_type,answer,answer2,category&quot;</HelpBlock>
</FormGroup>
<FormGroup>
<Checkbox
checked={this.state.isCsvUploadReplace}
onChange={e => this.setState({ isCsvUploadReplace: e.target.checked })}
checked={this.state.isJsonUploadReplace}
onChange={e => this.setState({ isJsonUploadReplace: e.target.checked })}
>
Replace existing FAQs
</Checkbox>
Expand All @@ -258,7 +258,7 @@ export default class QnaAdmin extends Component {
</form>
</Modal.Body>
<Modal.Footer>
<Button bsStyle="primary" onClick={this.uploadCsv} disabled={!Boolean(this.state.csvToUpload)}>
<Button bsStyle="primary" onClick={this.uploadJson} disabled={!Boolean(this.state.jsonToUpload)}>
Upload
</Button>
</Modal.Footer>
Expand All @@ -278,18 +278,18 @@ export default class QnaAdmin extends Component {
bsStyle="default"
onClick={() =>
this.setState({
importCsvModalShow: true,
csvToUpload: null,
csvUploadStatus: null,
isCsvUploadReplace: false
importJsonModalShow: true,
jsonToUpload: null,
jsonUploadStatus: null,
isJsonUploadReplace: false
})
}
type="button"
>
Import from CSV
Import from JSON
</Button>
<Button bsStyle="default" onClick={this.downloadCsv} type="button">
Export to CSV
<Button bsStyle="default" onClick={this.downloadJson} type="button">
Export to JSON
</Button>
</ButtonGroup>
</ButtonToolbar>
Expand Down Expand Up @@ -510,9 +510,9 @@ export default class QnaAdmin extends Component {
return (
<Panel className={classnames(style.qnaContainer, 'qnaContainer')}>
<a
ref={this.csvDownloadableLink}
href={this.state.csvDownloadableLinkHref}
download={this.state.csvDownloadableFileName}
ref={this.jsonDownloadableLink}
href={this.state.jsonDownloadableLinkHref}
download={this.state.jsonDownloadableFileName}
/>
<Panel.Body>
{this.renderQnAHeader()}
Expand Down

0 comments on commit f11ca7c

Please sign in to comment.