diff --git a/packages/api/src/shell/dataDuplicator.js b/packages/api/src/shell/dataDuplicator.js index 15f308c69..dedc37d01 100644 --- a/packages/api/src/shell/dataDuplicator.js +++ b/packages/api/src/shell/dataDuplicator.js @@ -9,7 +9,7 @@ const copyStream = require('./copyStream'); const jsonLinesReader = require('./jsonLinesReader'); const { resolveArchiveFolder } = require('../utility/directories'); -async function dataDuplicator({ connection, archive, items, analysedStructure = null }) { +async function dataDuplicator({ connection, archive, items, options, analysedStructure = null }) { const driver = requireEngineDriver(connection); const pool = await connectUtility(driver, connection, 'write'); logger.info(`Connected.`); @@ -29,7 +29,8 @@ async function dataDuplicator({ connection, archive, items, analysedStructure = openStream: () => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) }), })), stream, - copyStream + copyStream, + options ); await dupl.run(); diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index e15bb0dc2..7a13e73ea 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -12,6 +12,11 @@ export interface DataDuplicatorItem { matchColumns: string[]; } +export interface DataDuplicatorOptions { + rollbackAfterFinish: boolean; + skipRowsWithUnresolvedRefs: boolean; +} + class DuplicatorReference { constructor( public base: DuplicatorItemHolder, @@ -78,6 +83,13 @@ class DuplicatorItemHolder { if (ref) { // remap id res[key] = ref.ref.idMap[res[key]]; + if (ref.isMandatory && res[key] == null) { + // mandatory refertence not matched + if (this.duplicator.options.skipRowsWithUnresolvedRefs) { + return null; + } + throw new Error(`Unresolved reference, base=${ref.base.name}, ref=${ref.ref.name}, ${key}=${chunk[key]}`); + } } } @@ -91,6 +103,7 @@ class DuplicatorItemHolder { let inserted = 0; let mapped = 0; let missing = 0; + let skipped = 0; let lastLogged = new Date(); const writeStream = createAsyncWriteStream(this.duplicator.stream, { @@ -101,6 +114,10 @@ class DuplicatorItemHolder { const doCopy = async () => { const insertedObj = this.createInsertObject(chunk); + if (insertedObj == null) { + skipped += 1; + return; + } await runCommandOnDriver(pool, driver, dmp => dmp.putCmd( '^insert ^into %f (%,i) ^values (%,v)', @@ -150,7 +167,7 @@ class DuplicatorItemHolder { if (new Date().getTime() - lastLogged.getTime() > 5000) { logger.info( - `Duplicating ${this.item.name} in progress, inserted ${inserted} rows, mapped ${mapped} rows, missing ${missing} rows` + `Duplicating ${this.item.name} in progress, inserted ${inserted} rows, mapped ${mapped} rows, missing ${missing} rows, skipped ${skipped} rows` ); lastLogged = new Date(); } @@ -166,7 +183,7 @@ class DuplicatorItemHolder { // }, // }); - return { inserted, mapped, missing }; + return { inserted, mapped, missing, skipped }; } } @@ -180,7 +197,8 @@ export class DataDuplicator { public db: DatabaseInfo, public items: DataDuplicatorItem[], public stream, - public copyStream: (input, output) => Promise + public copyStream: (input, output) => Promise, + public options: DataDuplicatorOptions ) { this.itemHolders = items.map(x => new DuplicatorItemHolder(x, this)); this.itemHolders.forEach(x => x.initializeReferences()); @@ -220,13 +238,19 @@ export class DataDuplicator { for (const item of this.itemPlan) { const stats = await item.runImport(); logger.info( - `Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows` + `Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows, skipped ${stats.skipped} rows` ); } } catch (err) { logger.error({ err }, 'Failed duplicator job, rollbacking'); await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction()); } - await runCommandOnDriver(this.pool, this.driver, dmp => dmp.commitTransaction()); + if (this.options.rollbackAfterFinish) { + logger.info('Rollbacking transaction, nothing was changed'); + await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction()); + } else { + logger.info('Committing duplicator transaction'); + await runCommandOnDriver(this.pool, this.driver, dmp => dmp.commitTransaction()); + } } } diff --git a/packages/web/src/tabs/DataDuplicatorTab.svelte b/packages/web/src/tabs/DataDuplicatorTab.svelte index 3a1ab4d95..c97e9d344 100644 --- a/packages/web/src/tabs/DataDuplicatorTab.svelte +++ b/packages/web/src/tabs/DataDuplicatorTab.svelte @@ -37,6 +37,7 @@ import TableControl from '../elements/TableControl.svelte'; import VerticalSplitter from '../elements/VerticalSplitter.svelte'; import CheckboxField from '../forms/CheckboxField.svelte'; + import FormFieldTemplateLarge from '../forms/FormFieldTemplateLarge.svelte'; import SelectField from '../forms/SelectField.svelte'; import { extractShellConnection } from '../impexp/createImpExpScript'; import SocketMessageView from '../query/SocketMessageView.svelte'; @@ -116,6 +117,10 @@ operation: row.operation, matchColumns: _.compact([row.matchColumn1]), })), + options: { + rollbackAfterFinish: !!$editorState.value?.rollbackAfterFinish, + skipRowsWithUnresolvedRefs: !!$editorState.value?.skipRowsWithUnresolvedRefs, + }, }); return script.getScript(); } @@ -211,21 +216,68 @@
-
Source archive
- { - setEditorData(old => ({ - ...old, - archiveFolder: e.detail, - })); + + { + setEditorData(old => ({ + ...old, + archiveFolder: e.detail, + })); + }} + options={$archiveFolders?.map(x => ({ + label: x.name, + value: x.name, + })) || []} + /> + + + { + setEditorData(old => ({ + ...old, + rollbackAfterFinish: !$editorState.value?.rollbackAfterFinish, + })); + }, }} - options={$archiveFolders?.map(x => ({ - label: x.name, - value: x.name, - })) || []} - /> + > + { + setEditorData(old => ({ + ...old, + rollbackAfterFinish: e.target.checked, + })); + }} + /> + + + { + setEditorData(old => ({ + ...old, + skipRowsWithUnresolvedRefs: !$editorState.value?.skipRowsWithUnresolvedRefs, + })); + }, + }} + > + { + setEditorData(old => ({ + ...old, + skipRowsWithUnresolvedRefs: e.target.checked, + })); + }} + /> +
diff --git a/packages/web/src/widgets/StatusBar.svelte b/packages/web/src/widgets/StatusBar.svelte index b448245d4..e020fa5ff 100644 --- a/packages/web/src/widgets/StatusBar.svelte +++ b/packages/web/src/widgets/StatusBar.svelte @@ -6,7 +6,14 @@ import FontIcon from '../icons/FontIcon.svelte'; - import { activeTabId, currentDatabase, currentThemeDefinition, visibleCommandPalette } from '../stores'; + import { + activeTabId, + currentArchive, + currentDatabase, + currentThemeDefinition, + selectedWidget, + visibleCommandPalette, + } from '../stores'; import getConnectionLabel from '../utility/getConnectionLabel'; import { useConnectionList, useDatabaseServerVersion, useDatabaseStatus } from '../utility/metadataLoaders'; import { findCommand } from '../commands/runCommand'; @@ -140,6 +147,18 @@
{/if} + {#if $currentArchive} +
{ + $selectedWidget = 'archive'; + }} + > + + {$currentArchive} +
+ {/if}
{#each contextItems || [] as item}