diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index b81ef5c9e..e12decda0 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -1,8 +1,10 @@ -import { createAsyncWriteStream, runCommandOnDriver, runQueryOnDriver } from 'dbgate-tools'; +import { createAsyncWriteStream, getLogger, runCommandOnDriver, runQueryOnDriver } from 'dbgate-tools'; import { DatabaseInfo, EngineDriver, ForeignKeyInfo, TableInfo } from 'dbgate-types'; import _pick from 'lodash/pick'; import _omit from 'lodash/omit'; +const logger = getLogger('dataDuplicator'); + export interface DataDuplicatorItem { openStream: () => Promise; name: string; @@ -86,6 +88,10 @@ class DuplicatorItemHolder { const readStream = await this.item.openStream(); const driver = this.duplicator.driver; const pool = this.duplicator.pool; + let inserted = 0; + let mapped = 0; + let missing = 0; + const writeStream = createAsyncWriteStream(this.duplicator.stream, { processItem: async chunk => { if (chunk.__isStreamHeader) { @@ -102,6 +108,7 @@ class DuplicatorItemHolder { Object.values(insertedObj) ) ); + inserted += 1; if (this.autoColumn && this.isReferenced) { const res = await runQueryOnDriver(pool, driver, dmp => dmp.selectScopeIdentity(this.table)); const resId = Object.entries(res?.rows?.[0])?.[0]?.[1]; @@ -129,9 +136,12 @@ class DuplicatorItemHolder { ); const resId = Object.entries(res?.rows?.[0])?.[0]?.[1]; if (resId != null) { + mapped += 1; this.idMap[chunk[this.autoColumn]] = resId; } else if (this.item.operation == 'insertMissing') { await doCopy(); + } else { + missing += 1; } break; } @@ -147,6 +157,8 @@ class DuplicatorItemHolder { // this.idMap[oldId] = newId; // }, // }); + + return { inserted, mapped, missing }; } } @@ -196,7 +208,10 @@ export class DataDuplicator { this.createPlan(); for (const item of this.itemPlan) { - await item.runImport(); + const stats = await item.runImport(); + logger.info( + `Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows` + ); } } } diff --git a/packages/web/src/tabs/DataDuplicatorTab.svelte b/packages/web/src/tabs/DataDuplicatorTab.svelte index b2d3e127d..ab51d1957 100644 --- a/packages/web/src/tabs/DataDuplicatorTab.svelte +++ b/packages/web/src/tabs/DataDuplicatorTab.svelte @@ -23,11 +23,13 @@ import invalidateCommands from '../commands/invalidateCommands'; import registerCommand from '../commands/registerCommand'; import TableControl from '../elements/TableControl.svelte'; + import VerticalSplitter from '../elements/VerticalSplitter.svelte'; import CheckboxField from '../forms/CheckboxField.svelte'; import SelectField from '../forms/SelectField.svelte'; import { extractShellConnection } from '../impexp/createImpExpScript'; + import SocketMessageView from '../query/SocketMessageView.svelte'; import useEditorData from '../query/useEditorData'; - import { currentArchive, getCurrentConfig } from '../stores'; + import { getCurrentConfig } from '../stores'; import { apiCall, apiOff, apiOn } from '../utility/api'; import { changeTab } from '../utility/common'; import createActivator, { getActiveComponent } from '../utility/createActivator'; @@ -40,6 +42,7 @@ let busy = false; let runnerId = null; + let executeNumber = 0; export const activator = createActivator('DataDuplicatorTab', true); @@ -106,6 +109,7 @@ export async function run() { if (busy) return; + executeNumber += 1; busy = true; const script = await createScript(); let runid = runnerId; @@ -156,77 +160,85 @@ -
-
Source archive
- { - setEditorData(old => ({ - ...old, - archiveFolder: e.detail, - })); - }} - options={$archiveFolders?.map(x => ({ - label: x.name, - value: x.name, - })) || []} - /> - -
Imported files
- - Table', fieldName: 'name' }, - { header: 'Operation', fieldName: 'operation', slot: 2 }, - { header: 'Match column', fieldName: 'matchColumn1', slot: 3 }, - ]} - > - - { - changeTable({ ...row, isChecked: e.target.checked }); - }} - /> - - + + +
+
Source archive
{ - changeTable({ ...row, operation: e.detail }); + setEditorData(old => ({ + ...old, + archiveFolder: e.detail, + })); }} - disabled={!row.isChecked} - options={[ - { label: 'Copy row', value: 'copy' }, - { label: 'Lookup (find matching row)', value: 'lookup' }, - { label: 'Insert if not exists', value: 'insertMissing' }, - ]} + options={$archiveFolders?.map(x => ({ + label: x.name, + value: x.name, + })) || []} /> - - - {#if row.operation != 'copy'} - { - changeTable({ ...row, matchColumn1: e.detail }); - }} - disabled={!row.isChecked} - options={$dbinfo?.tables - ?.find(x => x.pureName?.toUpperCase() == row.name.toUpperCase()) - ?.columns?.map(col => ({ - label: col.columnName, - value: col.columnName, - })) || []} - /> - {/if} - - -
+ +
Imported files
+ + Table', fieldName: 'name' }, + { header: 'Operation', fieldName: 'operation', slot: 2 }, + { header: 'Match column', fieldName: 'matchColumn1', slot: 3 }, + ]} + > + + { + changeTable({ ...row, isChecked: e.target.checked }); + }} + /> + + + { + changeTable({ ...row, operation: e.detail }); + }} + disabled={!row.isChecked} + options={[ + { label: 'Copy row', value: 'copy' }, + { label: 'Lookup (find matching row)', value: 'lookup' }, + { label: 'Insert if not exists', value: 'insertMissing' }, + ]} + /> + + + {#if row.operation != 'copy'} + { + changeTable({ ...row, matchColumn1: e.detail }); + }} + disabled={!row.isChecked} + options={$dbinfo?.tables + ?.find(x => x.pureName?.toUpperCase() == row.name.toUpperCase()) + ?.columns?.map(col => ({ + label: col.columnName, + value: col.columnName, + })) || []} + /> + {/if} + + +
+ + + + + +