Skip to content

Commit

Permalink
feat(MySQL): option to export from results SQL INSERTS in chunks, closes
Browse files Browse the repository at this point in the history 
 #501
  • Loading branch information
@Fabio286
Fabio286 committed Apr 8, 2023
1 parent afa61a9 commit 0f24c80
Show file tree
Hide file tree
Showing 6 changed files with 117 additions and 44 deletions.
46 changes: 38 additions & 8 deletions src/common/libs/sqlUtils.ts
View file
Expand Up @@ -94,7 +94,7 @@ export const valueToSqlString = (args: {
? escapeAndQuote(moment(val).format(`YYYY-MM-DD HH:mm:ss${datePrecision}`), client)
: escapeAndQuote(val, client);
}
else if ('isArray' in field) {
else if ('isArray' in field && field.isArray) {
let localVal;
if (Array.isArray(val))
localVal = JSON.stringify(val).replaceAll('[', '{').replaceAll(']', '}');
Expand Down Expand Up @@ -152,17 +152,47 @@ export const valueToSqlString = (args: {
};

export const jsonToSqlInsert = (args: {
json: { [key: string]: any};
json: { [key: string]: any}[];
client: ClientCode;
fields: { [key: string]: {type: string; datePrecision: number}};
table: string;
options?: {sqlInsertAfter: number; sqlInsertDivider: 'bytes' | 'rows'};
}) => {
const { client, json, fields, table } = args;
const { client, json, fields, table, options } = args;
const sqlInsertAfter = options && options.sqlInsertAfter ? options.sqlInsertAfter : 1;
const sqlInsertDivider = options && options.sqlInsertDivider ? options.sqlInsertDivider : 'rows';
const { elementsWrapper: ew } = customizations[client];
const fieldNames = Object.keys(json).map(key => `${ew}${key}${ew}`);
const values = Object.keys(json).map(key => (
valueToSqlString({ val: json[key], client, field: fields[key] })
));
const fieldNames = Object.keys(json[0]).map(key => `${ew}${key}${ew}`);
let insertStmt = `INSERT INTO ${ew}${table}${ew} (${fieldNames.join(', ')}) VALUES `;
let insertsString = '';
let queryLength = 0;
let rowsWritten = 0;

for (const row of json) {
const values = [];

values.push(Object.keys(row).map(key => (
valueToSqlString({ val: row[key], client, field: fields[key] })
)));

if (
(sqlInsertDivider === 'bytes' && queryLength >= sqlInsertAfter * 1024) ||
(sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter)
) {
insertsString += insertStmt+';';
insertStmt = `\nINSERT INTO ${ew}${table}${ew} (${fieldNames.join(', ')}) VALUES `;
rowsWritten = 0;
}
rowsWritten++;

if (rowsWritten > 1) insertStmt += ',\n';

insertStmt += `(${values.join(',')})`;
queryLength = insertStmt.length;
}

if (rowsWritten > 0)
insertsString += insertStmt+';';

return `INSERT INTO ${ew}${table}${ew} (${fieldNames.join(', ')}) VALUES (${values.join(', ')});`;
return insertsString;
};
6 changes: 3 additions & 3 deletions src/renderer/components/ModalExportSchema.vue
View file
Expand Up @@ -6,7 +6,7 @@
<div class="modal-header pl-2">
<div class="modal-title h6">
<div class="d-flex">
<i class="mdi mdi-24px mdi-database-arrow-down mr-1" />
<i class="mdi mdi-24px mdi-database-export mr-1" />
<span class="cut-text">{{ t('message.exportSchema') }}</span>
</div>
</div>
Expand Down Expand Up @@ -78,7 +78,7 @@
<div ref="table" class="table table-hover">
<div class="thead">
<div class="tr text-center">
<div class="th no-border" style="width: 50%;" />
<div class="th no-border" :style="'width: 50%;'" />
<div class="th no-border">
<label
class="form-checkbox m-0 px-2 form-inline"
Expand Down Expand Up @@ -120,7 +120,7 @@
</div>
</div>
<div class="tr">
<div class="th" style="width: 50%;">
<div class="th" :style="'width: 50%;'">
<div class="table-column-title">
<span>{{ t('word.table') }}</span>
</div>
Expand Down
2 changes: 1 addition & 1 deletion src/renderer/components/ModalImportSchema.vue
View file
Expand Up @@ -6,7 +6,7 @@
<div class="modal-header pl-2">
<div class="modal-title h6">
<div class="d-flex">
<i class="mdi mdi-24px mdi-database-arrow-up mr-1" />
<i class="mdi mdi-24px mdi-database-import mr-1" />
<span class="cut-text">{{ t('message.importSchema') }}</span>
</div>
</div>
Expand Down
4 changes: 2 additions & 2 deletions src/renderer/components/WorkspaceExploreBarSchemaContext.vue
View file
Expand Up @@ -63,14 +63,14 @@
class="context-element"
@click="showExportSchemaModal"
>
<span class="d-flex"><i class="mdi mdi-18px mdi-database-arrow-down text-light pr-1" /> {{ t('word.export') }}</span>
<span class="d-flex"><i class="mdi mdi-18px mdi-database-export text-light pr-1" /> {{ t('word.export') }}</span>
</div>
<div
v-if="workspace.customizations.schemaImport"
class="context-element"
@click="initImport"
>
<span class="d-flex"><i class="mdi mdi-18px mdi-database-arrow-up text-light pr-1" /> {{ t('word.import') }}</span>
<span class="d-flex"><i class="mdi mdi-18px mdi-database-import text-light pr-1" /> {{ t('word.import') }}</span>
</div>
<div
v-if="workspace.customizations.schemaEdit"
Expand Down
81 changes: 63 additions & 18 deletions src/renderer/components/WorkspaceTabQueryTable.vue
View file
Expand Up @@ -31,7 +31,7 @@
:class="{'active': resultsetIndex === index}"
@click="selectResultset(index)"
>
<a>{{ result.fields ? result.fields[0].table : '' }} ({{ result.rows.length }})</a>
<a>{{ result.fields ? result.fields[0]?.table : '' }} ({{ result.rows.length }})</a>
</li>
</ul>
<div ref="table" class="table table-hover">
Expand Down Expand Up @@ -111,6 +111,37 @@
</div>
</template>
</ConfirmModal>

<ConfirmModal
v-if="chunkModalRequest"
@confirm="downloadTable('sql', chunkModalRequest as string, true)"
@hide="chunkModalRequest = false"
>
<template #header>
<div class="d-flex">
<i class="mdi mdi-24px mdi-file-export mr-1" />
<span class="cut-text">{{ t('message.newInserStmtEvery') }}:</span>
</div>
</template>
<template #body>
<div class="columns">
<div class="column col-6">
<input
v-model.number="sqlExportOptions.sqlInsertAfter"
type="number"
class="form-input"
>
</div>
<div class="column col-6">
<BaseSelect
v-model="sqlExportOptions.sqlInsertDivider"
class="form-select"
:options="[{value: 'bytes', label: 'KiB'}, {value: 'rows', label: t('word.row', 2)}]"
/>
</div>
</div>
</template>
</ConfirmModal>
</div>
</template>

Expand All @@ -128,6 +159,7 @@ import BaseVirtualScroll from '@/components/BaseVirtualScroll.vue';
import WorkspaceTabQueryTableRow from '@/components/WorkspaceTabQueryTableRow.vue';
import TableContext from '@/components/WorkspaceTabQueryTableContext.vue';
import ConfirmModal from '@/components/BaseConfirmModal.vue';
import BaseSelect from '@/components/BaseSelect.vue';
import * as moment from 'moment';
import { useI18n } from 'vue-i18n';
import { TableField, QueryResult } from 'common/interfaces/antares';
Expand Down Expand Up @@ -179,9 +211,15 @@ const scrollElement = ref(null);
const rowHeight = ref(23);
const selectedField = ref(null);
const isEditingRow = ref(false);
const chunkModalRequest: Ref<false | string> = ref(false);
const sqlExportOptions = ref({
sqlInsertAfter: 250,
sqlInsertDivider: 'bytes' as 'bytes' | 'rows'
});

const workspaceSchema = computed(() => getWorkspace(props.connUid).breadcrumbs.schema);
const workspaceClient = computed(() => getWorkspace(props.connUid).client);
const customizations = computed(() => getWorkspace(props.connUid).customizations);

const primaryField = computed(() => {
const primaryFields = fields.value.filter(field => field.key === 'pri');
Expand Down Expand Up @@ -219,7 +257,7 @@ const sortedResults = computed(() => {
return localResults.value;
});

const resultsWithRows = computed(() => props.results.filter(result => result.rows));
const resultsWithRows = computed(() => props.results.filter(result => result.rows.length));
const fields = computed(() => resultsWithRows.value.length ? resultsWithRows.value[resultsetIndex.value].fields : []);
const keyUsage = computed(() => resultsWithRows.value.length ? resultsWithRows.value[resultsetIndex.value].keys : []);

Expand Down Expand Up @@ -438,20 +476,17 @@ const copyRow = (format: string) => {
if (format === 'json')
navigator.clipboard.writeText(JSON.stringify(contentToCopy));
else if (format === 'sql') {
const sqlInserts = [];
if (!Array.isArray(contentToCopy)) contentToCopy = [contentToCopy];

for (const row of contentToCopy) {
sqlInserts.push(jsonToSqlInsert({
json: row,
client: workspaceClient.value,
fields: fieldsObj.value as {
[key: string]: {type: string; datePrecision: number};
},
table: getTable(resultsetIndex.value)
}));
}
navigator.clipboard.writeText(sqlInserts.join('\n'));
const sqlInserts = jsonToSqlInsert({
json: contentToCopy,
client: workspaceClient.value,
fields: fieldsObj.value as {
[key: string]: {type: string; datePrecision: number};
},
table: getTable(resultsetIndex.value)
});
navigator.clipboard.writeText(sqlInserts);
}
else if (format === 'csv') {
const csv = [];
Expand Down Expand Up @@ -665,21 +700,31 @@ const selectResultset = (index: number) => {
resultsetIndex.value = index;
};

const downloadTable = (format: 'csv' | 'json' | 'sql', table: string) => {
const downloadTable = (format: 'csv' | 'json' | 'sql', table: string, chunks = false) => {
if (!sortedResults.value) return;

if (format === 'sql' && !chunks && customizations.value.exportByChunks) {
chunkModalRequest.value = table;
return;
}
else
chunkModalRequest.value = false;

const rows = sortedResults.value.map((row: any) => {
delete row._antares_id;
return row;
const clonedRow = { ...row };
delete clonedRow._antares_id;
return clonedRow;
});

exportRows({
type: format,
content: rows,
fields: JSON.parse(JSON.stringify(fieldsObj.value)) as {
[key: string]: {type: string; datePrecision: number};
},
client: workspaceClient.value,
table
table,
sqlOptions: chunks ? { ...sqlExportOptions.value }: null
});
};

Expand Down
22 changes: 10 additions & 12 deletions src/renderer/libs/exportRows.ts
View file
Expand Up @@ -9,6 +9,7 @@ export const exportRows = (args: {
fields?: {
[key: string]: {type: string; datePrecision: number};
};
sqlOptions?: {sqlInsertAfter: number; sqlInsertDivider: 'bytes' | 'rows'};
}) => {
let mime;
let content;
Expand All @@ -35,19 +36,16 @@ export const exportRows = (args: {
}
case 'sql': {
mime = 'text/sql';
const sql = [];
const sql = jsonToSqlInsert({
json: args.content,
client:
args.client,
fields: args.fields,
table: args.table,
options: args.sqlOptions
});

for (const row of args.content) {
sql.push(jsonToSqlInsert({
json: row,
client:
args.client,
fields: args.fields,
table: args.table
}));
}

content = sql.join('\n');
content = sql;
break;
}
case 'json':
Expand Down

0 comments on commit 0f24c80

Please sign in to comment.