From a399ee8dc96359ca7ab97d6c4e92f8ed87f4d233 Mon Sep 17 00:00:00 2001 From: Nut He <18328704+hetao92@users.noreply.github.com> Date: Thu, 20 Jan 2022 20:56:35 +0800 Subject: [PATCH] fix: fix import status check bug mod: update rate --- app/modules/Import/Tasks/Import.tsx | 65 +++++++++++++----------- app/modules/Schema/CreateSpace/index.tsx | 6 +-- app/utils/import.ts | 33 +++++------- 3 files changed, 49 insertions(+), 55 deletions(-) diff --git a/app/modules/Import/Tasks/Import.tsx b/app/modules/Import/Tasks/Import.tsx index 3061e2f0..1068ae4c 100644 --- a/app/modules/Import/Tasks/Import.tsx +++ b/app/modules/Import/Tasks/Import.tsx @@ -130,8 +130,7 @@ class Import extends React.Component { activeStep, }); if (errCode === 0) { - this.logTimer = setTimeout(this.readlog, 3000); - this.checkTimer = setTimeout(this.checkIsFinished, 2000); + this.checkTimer = setTimeout(this.checkTaskStatus, 3000); } trackEvent('import', 'import_data', 'start'); }; @@ -148,12 +147,14 @@ class Import extends React.Component { } }; - checkIsFinished = async () => { + checkTaskStatus = async () => { const { taskId } = this.props; const { code, data, message: errMsg } = await this.props.checkImportStatus({ taskID: taskId, taskAction: 'actionQuery', }); + await this.readlog(); + if (code !== 0) { message.warning(errMsg); return; @@ -165,50 +166,56 @@ class Import extends React.Component { message.warning(result.taskMessage); } if (result?.taskStatus === ITaskStatus.statusFinished) { - message.success(intl.get('import.importFinished')); + trackEvent('import', 'import_data', 'finish'); } + this.logTimer = setTimeout(this.checkLogFinished, 3000); service.finishImport({ taskId }); clearTimeout(this.checkTimer); } else { - this.checkTimer = setTimeout(this.checkIsFinished, 2000); + this.checkTimer = setTimeout(this.checkTaskStatus, 3000); + } + }; + + checkLogFinished = async () => { + const { update } = this.props; + const empty = await this.readlog(); + if (empty) { + message.success(intl.get('import.importFinished')); + this.setState({ + startByte: 0, + endByte: 1000000, + }); + update({ + isImporting: false, + }); + clearTimeout(this.logTimer); + } else { + this.logTimer = setTimeout(this.checkLogFinished, 3000); } }; readlog = async () => { const { startByte, endByte } = this.state; - const { taskDir, taskId, update } = this.props; - const result: any = await service.getLog({ + const { taskDir, taskId } = this.props; + const { code, data } = await service.getLog({ dir: taskDir, startByte, endByte, taskId, }); - const byteLength = result.data ? getStringByteLength(result.data) : 0; - if (result.data && result.code === 0) { - this.setState( - { + const byteLength = data ? getStringByteLength(data) : 0; + if (code === 0) { + if (data) { + this.setState({ startByte: startByte + byteLength, endByte: startByte + byteLength + 1000000, - }, - () => { - this.logTimer = setTimeout(this.readlog, 2000); - }, - ); - this.ref.innerHTML += result.data; - } else { - if (result.code === 0) { - this.logTimer = setTimeout(this.readlog, 2000); - } else { - this.setState({ - startByte: 0, - endByte: 1000000, - }); - update({ - isImporting: false, }); - clearTimeout(this.logTimer); - trackEvent('import', 'import_data', 'finish'); + this.ref.innerHTML += data; + } else { + return true; } + } else { + return true; } this.ref.scrollTop = this.ref.scrollHeight; }; diff --git a/app/modules/Schema/CreateSpace/index.tsx b/app/modules/Schema/CreateSpace/index.tsx index ad234afe..6c27ed4b 100644 --- a/app/modules/Schema/CreateSpace/index.tsx +++ b/app/modules/Schema/CreateSpace/index.tsx @@ -17,11 +17,7 @@ import { RouteComponentProps, withRouter } from 'react-router-dom'; import { Instruction } from '#app/components'; import GQLCodeMirror from '#app/components/GQLCodeMirror'; -import { - nameRulesFn, - numberRulesFn, - replicaRulesFn, -} from '#app/config/rules'; +import { nameRulesFn, numberRulesFn, replicaRulesFn } from '#app/config/rules'; import { IDispatch, IRootState } from '#app/store'; import { getSpaceCreateGQL } from '#app/utils/gql'; import { trackEvent, trackPageView } from '#app/utils/stat'; diff --git a/app/utils/import.ts b/app/utils/import.ts index 934a9b2c..08e9f72a 100644 --- a/app/utils/import.ts +++ b/app/utils/import.ts @@ -3,6 +3,7 @@ import _ from 'lodash'; import intl from 'react-intl-universal'; import { handleVidStringName } from './function'; + export function configToJson(payload) { const { currentSpace, @@ -42,7 +43,7 @@ export function configToJson(payload) { address: host, }, }, - logPath: taskDir + '/import.log', + logPath: `${taskDir}/import.log`, files, }; return configJson; @@ -57,9 +58,7 @@ export function edgeDataToJSON( const limit = activeStep === 2 || activeStep === 3 ? 10 : undefined; const files = config.map(edge => { const edgePorps: any[] = []; - _.sortBy(edge.props, t => { - return t.mapping; - }).forEach(prop => { + _.sortBy(edge.props, t => t.mapping).forEach(prop => { switch (prop.name) { case 'rank': if (prop.mapping !== null) { @@ -95,7 +94,7 @@ export function edgeDataToJSON( const edgeConfig = { path: edge.file.path, failDataPath: `${taskDir}/err/${edge.name}Fail.csv`, - batchSize: 10, + batchSize: 60, limit, type: 'csv', csv: { @@ -109,7 +108,7 @@ export function edgeDataToJSON( srcVID: edge.srcVID, dstVID: edge.dstVID, rank: edge.rank, - withRanking: edge.rank?.index !== undefined ? true : false, + withRanking: edge.rank?.index !== undefined, props: edgePorps, }, }, @@ -129,9 +128,7 @@ export function vertexDataToJSON( const files = config.map(vertex => { const tags = vertex.tags.map(tag => { const props = tag.props - .sort((p1, p2) => { - return p1.mapping - p2.mapping; - }) + .sort((p1, p2) => p1.mapping - p2.mapping) .map(prop => { if (prop.mapping === null && prop.isDefault) { return null; @@ -151,7 +148,7 @@ export function vertexDataToJSON( const vertexConfig: any = { path: vertex.file.path, failDataPath: `${taskDir}/err/${vertex.name}Fail.csv`, - batchSize: 10, + batchSize: 60, limit, type: 'csv', csv: { @@ -187,7 +184,7 @@ export function getStringByteLength(str: string) { const len = str.length; for (let i = 0, n = len; i < n; i++) { const c = str.charCodeAt(i); - if ((c >= 0x0001 && c <= 0x007e) || (0xff60 <= c && c <= 0xff9f)) { + if ((c >= 0x0001 && c <= 0x007e) || (c >= 0xff60 && c <= 0xff9f)) { bytesCount += 1; } else { bytesCount += 2; @@ -224,7 +221,7 @@ export function getGQLByConfig(payload) { } if (prop.mapping !== null) { // HACK: Processing keyword - tagField.push('`' + prop.name + '`'); + tagField.push(`\`${prop.name}\``); const value = prop.type === 'string' ? `"${columns[prop.mapping]}"` @@ -233,10 +230,7 @@ export function getGQLByConfig(payload) { } }); NGQL.push( - 'INSERT VERTEX ' + - '`' + - tag.name + - '`' + + `${'INSERT VERTEX ' + '`'}${tag.name}\`` + `(${tagField}) VALUES ${handleVidStringName( columns[vertexConfig.idMapping], spaceVidType, @@ -266,7 +260,7 @@ export function getGQLByConfig(payload) { prop.mapping !== null ) { // HACK: Processing keyword - edgeField.push('`' + prop.name + '`'); + edgeField.push(`\`${prop.name}\``); const value = prop.type === 'string' ? `"${columns[prop.mapping]}"` @@ -279,10 +273,7 @@ export function getGQLByConfig(payload) { ? '' : `@${columns[edgeConfig.props[2].mapping]}`; NGQL.push( - 'INSERT EDGE ' + - '`' + - edgeConfig.type + - '`' + + `${'INSERT EDGE ' + '`'}${edgeConfig.type}\`` + `(${edgeField.join(',')}) VALUES ${handleVidStringName( columns[edgeConfig.props[0].mapping], spaceVidType,