From cd1267b4644099c722625a53eda12a42db8ff9ca Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 16 Feb 2023 11:47:17 +0100 Subject: [PATCH 01/40] schema editing in dataset --- packages/datalib/src/ChangeSet.ts | 20 ++- packages/datalib/src/GridDisplay.ts | 1 + packages/datalib/src/JslGridDisplay.ts | 1 + .../web/src/datagrid/ColumnManager.svelte | 63 +++++++++- .../web/src/datagrid/ColumnManagerRow.svelte | 114 ++++++++++++++---- packages/web/src/datagrid/JslDataGrid.svelte | 6 +- .../src/tableeditor/ColumnEditorModal.svelte | 11 +- packages/web/src/tabs/ArchiveFileTab.svelte | 1 + 8 files changed, 184 insertions(+), 33 deletions(-) diff --git a/packages/datalib/src/ChangeSet.ts b/packages/datalib/src/ChangeSet.ts index 707db5db1..4e564af30 100644 --- a/packages/datalib/src/ChangeSet.ts +++ b/packages/datalib/src/ChangeSet.ts @@ -9,7 +9,7 @@ import { AllowIdentityInsert, Expression, } from 'dbgate-sqltree'; -import type { NamedObjectInfo, DatabaseInfo } from 'dbgate-types'; +import type { NamedObjectInfo, DatabaseInfo, TableInfo } from 'dbgate-types'; export interface ChangeSetItem { pureName: string; @@ -21,7 +21,16 @@ export interface ChangeSetItem { fields?: { [column: string]: string }; } +export interface ChangeSetDataUpdateCommand { + type: 'renameField' | 'deleteField' | 'setField'; + field: string; + value?: any; +} + export interface ChangeSet { + structure?: TableInfo; + dataUpdateCommands?: ChangeSetDataUpdateCommand[]; + setColumnMode?: 'fixed' | 'variable'; inserts: ChangeSetItem[]; updates: ChangeSetItem[]; deletes: ChangeSetItem[]; @@ -456,5 +465,12 @@ export function changeSetInsertDocuments(changeSet: ChangeSet, documents: any[], export function changeSetContainsChanges(changeSet: ChangeSet) { if (!changeSet) return false; - return changeSet.deletes.length > 0 || changeSet.updates.length > 0 || changeSet.inserts.length > 0; + return ( + changeSet.deletes.length > 0 || + changeSet.updates.length > 0 || + changeSet.inserts.length > 0 || + !!changeSet.structure || + !!changeSet.setColumnMode || + changeSet.dataUpdateCommands?.length > 0 + ); } diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 4f505ebca..9a7fc7707 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -84,6 +84,7 @@ export abstract class GridDisplay { return this.baseTable || this.baseView; } changeSetKeyFields: string[] = null; + editableStructure: TableInfo = null; sortable = false; groupable = false; filterable = false; diff --git a/packages/datalib/src/JslGridDisplay.ts b/packages/datalib/src/JslGridDisplay.ts index 9a34f490d..030471bea 100644 --- a/packages/datalib/src/JslGridDisplay.ts +++ b/packages/datalib/src/JslGridDisplay.ts @@ -24,6 +24,7 @@ export class JslGridDisplay extends GridDisplay { this.isDynamicStructure = isDynamicStructure; this.filterTypeOverride = 'eval'; this.editable = editable; + this.editableStructure = editable ? structure : null; if (structure?.columns) { this.columns = _.uniqBy( diff --git a/packages/web/src/datagrid/ColumnManager.svelte b/packages/web/src/datagrid/ColumnManager.svelte index 896295da2..1957f6e9f 100644 --- a/packages/web/src/datagrid/ColumnManager.svelte +++ b/packages/web/src/datagrid/ColumnManager.svelte @@ -1,6 +1,6 @@ +{#if allowChangeChangeSetStructure} +
+ +
+{/if} @@ -122,6 +164,9 @@ }}>Add {/if} + {#if allowChangeChangeSetStructure} + Add + {/if} display.hideAllColumns()}>Hide display.showAllColumns()}>Show @@ -139,12 +184,18 @@ /> {#each items as column (column.uniqueName)} + {@const columnIndex = items.indexOf(column)} { if (domFocusField) domFocusField.focus(); @@ -198,4 +249,14 @@ left: -1000px; top: -1000px; } + + .selectwrap :global(select) { + flex: 1; + padding: 3px 0px; + border: none; + } + + .selectwrap { + border-bottom: 1px solid var(--theme-border); + } diff --git a/packages/web/src/datagrid/ColumnManagerRow.svelte b/packages/web/src/datagrid/ColumnManagerRow.svelte index 06ee30bcf..ded11ca42 100644 --- a/packages/web/src/datagrid/ColumnManagerRow.svelte +++ b/packages/web/src/datagrid/ColumnManagerRow.svelte @@ -4,6 +4,8 @@ import FontIcon from '../icons/FontIcon.svelte'; import ColumnLabel from '../elements/ColumnLabel.svelte'; import { createEventDispatcher } from 'svelte'; + import { showModal } from '../modals/modalTools'; + import ColumnEditorModal from '../tableeditor/ColumnEditorModal.svelte'; export let column; export let display; @@ -12,6 +14,26 @@ export let conid; export let database; + export let tableInfo; + export let setTableInfo; + + export let columnInfo = null; + export let columnIndex = -1; + + export let allowChangeChangeSetStructure = false; + + function handleEditColumn() { + showModal(ColumnEditorModal, { columnInfo, tableInfo, setTableInfo }); + } + + function exchange(array, i1, i2) { + const i1r = (i1 + array.length) % array.length; + const i2r = (i2 + array.length) % array.length; + const res = [...array]; + [res[i1r], res[i2r]] = [res[i2r], res[i1r]]; + return res; + } + const dispatch = createEventDispatcher(); @@ -29,32 +51,63 @@ on:mousemove on:mouseup > - - display.toggleExpandedColumn(column.uniqueName)} - /> - - {#if isJsonView} - - {:else} - { - e.stopPropagation(); - }} - on:mousedown={e => { - e.stopPropagation(); - }} - on:change={() => { - const newValue = !column.isChecked; - display.setColumnVisibility(column.uniquePath, newValue); - dispatch('setvisibility', newValue); - }} - /> +
+ + display.toggleExpandedColumn(column.uniqueName)} + /> + + {#if isJsonView} + + {:else} + { + e.stopPropagation(); + }} + on:mousedown={e => { + e.stopPropagation(); + }} + on:change={() => { + const newValue = !column.isChecked; + display.setColumnVisibility(column.uniquePath, newValue); + dispatch('setvisibility', newValue); + }} + /> + {/if} + +
+ + {#if allowChangeChangeSetStructure} +
+ + + + + setTableInfo(info => ({ ...info, columns: info.columns.filter(x => x.pairingId != columnInfo?.pairingId) }))} + > + + + + setTableInfo(info => ({ ...info, columns: exchange(info.columns, columnIndex, columnIndex - 1) }))} + > + + + + setTableInfo(info => ({ ...info, columns: exchange(info.columns, columnIndex, columnIndex + 1) }))} + > + + +
{/if} - diff --git a/packages/web/src/datagrid/JslDataGrid.svelte b/packages/web/src/datagrid/JslDataGrid.svelte index 6ae1a2a31..1a52767ed 100644 --- a/packages/web/src/datagrid/JslDataGrid.svelte +++ b/packages/web/src/datagrid/JslDataGrid.svelte @@ -1,5 +1,6 @@ @@ -31,7 +31,10 @@ - + {#if driver?.dialect?.columnProperties?.isUnsigned} diff --git a/packages/web/src/tabs/ArchiveFileTab.svelte b/packages/web/src/tabs/ArchiveFileTab.svelte index 868404c26..0f6441919 100644 --- a/packages/web/src/tabs/ArchiveFileTab.svelte +++ b/packages/web/src/tabs/ArchiveFileTab.svelte @@ -87,6 +87,7 @@ Date: Thu, 16 Feb 2023 13:14:56 +0100 Subject: [PATCH 02/40] change structure generates data commands --- packages/datalib/src/ChangeSet.ts | 9 +- packages/tools/src/schemaEditorTools.ts | 84 ++++++++++++++++++- packages/tools/src/stringTools.ts | 12 +++ .../web/src/datagrid/ColumnManager.svelte | 12 ++- .../web/src/datagrid/ColumnManagerRow.svelte | 10 +-- packages/web/src/datagrid/JslDataGrid.svelte | 9 +- .../src/datagrid/LoadingDataGridCore.svelte | 5 +- .../src/tableeditor/ColumnEditorModal.svelte | 10 ++- 8 files changed, 127 insertions(+), 24 deletions(-) diff --git a/packages/datalib/src/ChangeSet.ts b/packages/datalib/src/ChangeSet.ts index 4e564af30..0238061a3 100644 --- a/packages/datalib/src/ChangeSet.ts +++ b/packages/datalib/src/ChangeSet.ts @@ -10,6 +10,7 @@ import { Expression, } from 'dbgate-sqltree'; import type { NamedObjectInfo, DatabaseInfo, TableInfo } from 'dbgate-types'; +import { JsonDataObjectUpdateCommand } from 'dbgate-tools'; export interface ChangeSetItem { pureName: string; @@ -21,15 +22,9 @@ export interface ChangeSetItem { fields?: { [column: string]: string }; } -export interface ChangeSetDataUpdateCommand { - type: 'renameField' | 'deleteField' | 'setField'; - field: string; - value?: any; -} - export interface ChangeSet { structure?: TableInfo; - dataUpdateCommands?: ChangeSetDataUpdateCommand[]; + dataUpdateCommands?: JsonDataObjectUpdateCommand[]; setColumnMode?: 'fixed' | 'variable'; inserts: ChangeSetItem[]; updates: ChangeSetItem[]; diff --git a/packages/tools/src/schemaEditorTools.ts b/packages/tools/src/schemaEditorTools.ts index 5c5c92657..b3a4b8f80 100644 --- a/packages/tools/src/schemaEditorTools.ts +++ b/packages/tools/src/schemaEditorTools.ts @@ -10,6 +10,14 @@ import type { UniqueInfo, } from 'dbgate-types'; import _ from 'lodash'; +import { parseSqlDefaultValue } from './stringTools'; + +export interface JsonDataObjectUpdateCommand { + type: 'renameField' | 'deleteField' | 'setField' | 'setFieldIfNull'; + oldField?: string; + newField?: string; + value?: any; +} export interface EditorColumnInfo extends ColumnInfo { isPrimaryKey?: boolean; @@ -23,6 +31,41 @@ export function fillEditorColumnInfo(column: ColumnInfo, table: TableInfo): Edit }; } +export function processJsonDataUpdateCommands(obj: any, commands: JsonDataObjectUpdateCommand[] = []) { + for (const cmd of commands) { + switch (cmd.type) { + case 'deleteField': + obj = { + ...obj, + }; + delete obj[cmd.oldField]; + break; + case 'renameField': + obj = { + ...obj, + }; + obj[cmd.newField] = obj[cmd.oldField]; + delete obj[cmd.oldField]; + break; + case 'setField': + obj = { + ...obj, + }; + obj[cmd.newField] = cmd.value; + break; + case 'setFieldIfNull': + obj = { + ...obj, + }; + if (obj[cmd.newField] == null) { + obj[cmd.newField] = cmd.value; + } + break; + } + } + return obj; +} + function processPrimaryKey(table: TableInfo, oldColumn: EditorColumnInfo, newColumn: EditorColumnInfo): TableInfo { if (!oldColumn?.isPrimaryKey && newColumn?.isPrimaryKey) { let primaryKey = table?.primaryKey; @@ -71,7 +114,11 @@ function processPrimaryKey(table: TableInfo, oldColumn: EditorColumnInfo, newCol return table; } -export function editorAddColumn(table: TableInfo, column: EditorColumnInfo): TableInfo { +function defineDataCommand(table: TableInfo, cmd: () => JsonDataObjectUpdateCommand) { + table['__addDataCommands'] = [...(table['__addDataCommands'] || []), cmd()]; +} + +export function editorAddColumn(table: TableInfo, column: EditorColumnInfo, addDataCommand?: boolean): TableInfo { let res = { ...table, columns: [...(table?.columns || []), { ...column, pairingId: uuidv1() }], @@ -79,10 +126,18 @@ export function editorAddColumn(table: TableInfo, column: EditorColumnInfo): Tab res = processPrimaryKey(res, null, column); + if (addDataCommand && column.defaultValue) { + defineDataCommand(res, () => ({ + type: 'setField', + field: column.columnName, + value: parseSqlDefaultValue(column.defaultValue), + })); + } + return res; } -export function editorModifyColumn(table: TableInfo, column: EditorColumnInfo): TableInfo { +export function editorModifyColumn(table: TableInfo, column: EditorColumnInfo, addDataCommand?: boolean): TableInfo { const oldColumn = table?.columns?.find(x => x.pairingId == column.pairingId); let res = { @@ -91,10 +146,26 @@ export function editorModifyColumn(table: TableInfo, column: EditorColumnInfo): }; res = processPrimaryKey(res, fillEditorColumnInfo(oldColumn, table), column); + if (addDataCommand && oldColumn.columnName != column.columnName) { + defineDataCommand(res, () => ({ + type: 'renameField', + oldField: oldColumn.columnName, + newField: column.columnName, + })); + } + + if (addDataCommand && !oldColumn.defaultValue && column.defaultValue) { + defineDataCommand(res, () => ({ + type: 'setFieldIfNull', + newField: column.columnName, + value: parseSqlDefaultValue(column.defaultValue), + })); + } + return res; } -export function editorDeleteColumn(table: TableInfo, column: EditorColumnInfo): TableInfo { +export function editorDeleteColumn(table: TableInfo, column: EditorColumnInfo, addDataCommand?: boolean): TableInfo { let res = { ...table, columns: table.columns.filter(col => col.pairingId != column.pairingId), @@ -102,6 +173,13 @@ export function editorDeleteColumn(table: TableInfo, column: EditorColumnInfo): res = processPrimaryKey(res, column, null); + if (addDataCommand) { + defineDataCommand(res, () => ({ + type: 'deleteField', + oldField: column.columnName, + })); + } + return res; } diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index 8705ac370..21be36385 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -115,3 +115,15 @@ export function getAsImageSrc(obj) { return null; } + +export function parseSqlDefaultValue(value: string) { + if (!value) return undefined; + if (!_isString(value)) return undefined; + if (value.startsWith("'") && value.endsWith("'")) { + return value.slice(1, -1); + } + if (!isNaN(value as any) && !isNaN(parseFloat(value))) { + return parseFloat(value); + } + return undefined; +} diff --git a/packages/web/src/datagrid/ColumnManager.svelte b/packages/web/src/datagrid/ColumnManager.svelte index 1957f6e9f..b484472a7 100644 --- a/packages/web/src/datagrid/ColumnManager.svelte +++ b/packages/web/src/datagrid/ColumnManager.svelte @@ -1,5 +1,5 @@ @@ -55,9 +57,9 @@ on:click={e => { closeCurrentModal(); if (columnInfo) { - setTableInfo(tbl => editorModifyColumn(tbl, e.detail)); + setTableInfo(tbl => editorModifyColumn(tbl, e.detail, addDataCommand)); } else { - setTableInfo(tbl => editorAddColumn(tbl, e.detail)); + setTableInfo(tbl => editorAddColumn(tbl, e.detail, addDataCommand)); if (onAddNext) onAddNext(); } }} @@ -68,7 +70,7 @@ value="Save" on:click={e => { closeCurrentModal(); - setTableInfo(tbl => editorAddColumn(tbl, e.detail)); + setTableInfo(tbl => editorAddColumn(tbl, e.detail, addDataCommand)); }} /> {/if} @@ -80,7 +82,7 @@ value="Remove" on:click={() => { closeCurrentModal(); - setTableInfo(tbl => editorDeleteColumn(tbl, columnInfo)); + setTableInfo(tbl => editorDeleteColumn(tbl, columnInfo, addDataCommand)); }} /> {/if} From 3114a05c3b156f5c6f287d3c1f6a3791ea9386e5 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 16 Feb 2023 13:33:28 +0100 Subject: [PATCH 03/40] save structure changes to jsonl file --- .../api/src/shell/modifyJsonLinesReader.js | 24 +++++++++++-- packages/tools/src/diffTools.ts | 36 ++++++++++++++++++- packages/web/src/datagrid/JslDataGrid.svelte | 3 +- packages/web/src/tabs/ArchiveFileTab.svelte | 6 ++++ 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/packages/api/src/shell/modifyJsonLinesReader.js b/packages/api/src/shell/modifyJsonLinesReader.js index d76ed8c8a..bb5c6b1ed 100644 --- a/packages/api/src/shell/modifyJsonLinesReader.js +++ b/packages/api/src/shell/modifyJsonLinesReader.js @@ -2,7 +2,7 @@ const fs = require('fs'); const _ = require('lodash'); const stream = require('stream'); const byline = require('byline'); -const { getLogger } = require('dbgate-tools'); +const { getLogger, processJsonDataUpdateCommands, removeTablePairingId } = require('dbgate-tools'); const logger = getLogger('modifyJsonLinesReader'); const stableStringify = require('json-stable-stringify'); @@ -11,6 +11,7 @@ class ParseStream extends stream.Transform { super({ objectMode: true }); this.limitRows = limitRows; this.changeSet = changeSet; + this.wasHeader = false; this.currentRowIndex = 0; if (mergeMode == 'merge') { if (mergedRows && mergeKey) { @@ -28,12 +29,28 @@ class ParseStream extends stream.Transform { _transform(chunk, encoding, done) { let obj = JSON.parse(chunk); if (obj.__isStreamHeader) { - this.push(obj); + if (this.changeSet && this.changeSet.structure) { + this.push({ + ...removeTablePairingId(this.changeSet.structure), + __isStreamHeader: true, + }); + } else { + this.push(obj); + } + this.wasHeader = true; done(); return; } if (this.changeSet) { + if (!this.wasHeader && this.changeSet.structure) { + this.push({ + ...removeTablePairingId(this.changeSet.structure), + __isStreamHeader: true, + }); + this.wasHeader = true; + } + if (!this.limitRows || this.currentRowIndex < this.limitRows) { if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) { obj = null; @@ -48,6 +65,9 @@ class ParseStream extends stream.Transform { } if (obj) { + if (this.changeSet.dataUpdateCommands) { + obj = processJsonDataUpdateCommands(obj, this.changeSet.dataUpdateCommands); + } this.push(obj); } this.currentRowIndex += 1; diff --git a/packages/tools/src/diffTools.ts b/packages/tools/src/diffTools.ts index ecda30151..01081f663 100644 --- a/packages/tools/src/diffTools.ts +++ b/packages/tools/src/diffTools.ts @@ -72,6 +72,34 @@ export function generateTablePairingId(table: TableInfo): TableInfo { return table; } +export function removeTablePairingId(table: TableInfo): TableInfo { + if (!table) return table; + return { + ...table, + columns: table.columns?.map(col => ({ + ...col, + pairingId: undefined, + })), + foreignKeys: table.foreignKeys?.map(cnt => ({ + ...cnt, + pairingId: undefined, + })), + checks: table.checks?.map(cnt => ({ + ...cnt, + pairingId: undefined, + })), + indexes: table.indexes?.map(cnt => ({ + ...cnt, + pairingId: undefined, + })), + uniques: table.uniques?.map(cnt => ({ + ...cnt, + pairingId: undefined, + })), + pairingId: undefined, + }; +} + function generateObjectPairingId(obj) { if (obj.objectTypeField) return { @@ -346,7 +374,13 @@ function createPairs(oldList, newList, additionalCondition = null) { function planTablePreload(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo) { const key = newTable.preloadedRowsKey || newTable.primaryKey?.columns?.map(x => x.columnName); if (newTable.preloadedRows?.length > 0 && key?.length > 0) { - plan.fillPreloadedRows(newTable, oldTable?.preloadedRows, newTable.preloadedRows, key, newTable.preloadedRowsInsertOnly); + plan.fillPreloadedRows( + newTable, + oldTable?.preloadedRows, + newTable.preloadedRows, + key, + newTable.preloadedRowsInsertOnly + ); } } diff --git a/packages/web/src/datagrid/JslDataGrid.svelte b/packages/web/src/datagrid/JslDataGrid.svelte index bdc4ddbcb..b430151e8 100644 --- a/packages/web/src/datagrid/JslDataGrid.svelte +++ b/packages/web/src/datagrid/JslDataGrid.svelte @@ -18,11 +18,12 @@ export let dispatchChangeSet = null; export let allowChangeChangeSetStructure = false; + export let infoLoadCounter = 0; let loadedRows; let infoCounter = 0; - $: info = useApiCall('jsldata/get-info', { jslid, infoCounter }, {}); + $: info = useApiCall('jsldata/get-info', { jslid, infoCounter, infoLoadCounter }, {}); // $: columns = ($info && $info.columns) || []; const config = writable(createGridConfig()); diff --git a/packages/web/src/tabs/ArchiveFileTab.svelte b/packages/web/src/tabs/ArchiveFileTab.svelte index 0f6441919..7ce9a347d 100644 --- a/packages/web/src/tabs/ArchiveFileTab.svelte +++ b/packages/web/src/tabs/ArchiveFileTab.svelte @@ -42,6 +42,7 @@ export let jslid = undefined; export let tabid; + let infoLoadCounter = 0; const quickExportHandlerRef = createQuickExportHandlerRef(); @@ -73,7 +74,11 @@ file: archiveFile, changeSet: $changeSetStore.value, }); + const structureChanged = !!$changeSetStore.value?.structure; dispatchChangeSet({ type: 'reset', value: createChangeSet() }); + if (structureChanged) { + infoLoadCounter += 1; + } await tick(); runCommand('dataGrid.refresh'); } @@ -92,6 +97,7 @@ focusOnVisible {changeSetStore} {dispatchChangeSet} + {infoLoadCounter} /> From b514f8ae35d307e08c60c6de82959d2ea19a1771 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 16 Feb 2023 15:11:33 +0100 Subject: [PATCH 04/40] using readline instead of line-reader-fixes freeze --- .../api/src/utility/JsonLinesDatastore.js | 35 ++------ packages/api/src/utility/LineReader.js | 88 +++++++++++++++++++ 2 files changed, 96 insertions(+), 27 deletions(-) create mode 100644 packages/api/src/utility/LineReader.js diff --git a/packages/api/src/utility/JsonLinesDatastore.js b/packages/api/src/utility/JsonLinesDatastore.js index b7f9c109d..c34cb3300 100644 --- a/packages/api/src/utility/JsonLinesDatastore.js +++ b/packages/api/src/utility/JsonLinesDatastore.js @@ -2,7 +2,6 @@ const fs = require('fs'); const os = require('os'); const rimraf = require('rimraf'); const path = require('path'); -const lineReader = require('line-reader'); const AsyncLock = require('async-lock'); const lock = new AsyncLock(); const stableStringify = require('json-stable-stringify'); @@ -11,23 +10,7 @@ const requirePluginFunction = require('./requirePluginFunction'); const esort = require('external-sorting'); const uuidv1 = require('uuid/v1'); const { jsldir } = require('./directories'); - -function fetchNextLineFromReader(reader) { - return new Promise((resolve, reject) => { - if (!reader.hasNextLine()) { - resolve(null); - return; - } - - reader.nextLine((err, line) => { - if (err) { - reject(err); - } else { - resolve(line); - } - }); - }); -} +const LineReader = require('./LineReader'); class JsonLinesDatastore { constructor(file, formatterFunction) { @@ -74,7 +57,7 @@ class JsonLinesDatastore { await new Promise(resolve => rimraf(tempDir, resolve)); } - _closeReader() { + async _closeReader() { // console.log('CLOSING READER', this.reader); if (!this.reader) return; const reader = this.reader; @@ -84,7 +67,7 @@ class JsonLinesDatastore { // this.firstRowToBeReturned = null; this.currentFilter = null; this.currentSort = null; - return new Promise(resolve => reader.close(resolve)); + await reader.close(); } async notifyChanged(callback) { @@ -100,12 +83,9 @@ class JsonLinesDatastore { async _openReader(fileName) { // console.log('OPENING READER', fileName); // console.log(fs.readFileSync(fileName, 'utf-8')); - return new Promise((resolve, reject) => - lineReader.open(fileName, (err, reader) => { - if (err) reject(err); - resolve(reader); - }) - ); + + const fileStream = fs.createReadStream(fileName); + return new LineReader(fileStream); } parseLine(line) { @@ -120,7 +100,7 @@ class JsonLinesDatastore { // return res; // } for (;;) { - const line = await fetchNextLineFromReader(this.reader); + const line = await this.reader.readLine(); if (!line) { // EOF return null; @@ -240,6 +220,7 @@ class JsonLinesDatastore { // console.log(JSON.stringify(this.currentFilter, undefined, 2)); for (let i = 0; i < limit; i += 1) { const line = await this._readLine(true); + // console.log('READED LINE', i); if (line == null) break; res.push(line); } diff --git a/packages/api/src/utility/LineReader.js b/packages/api/src/utility/LineReader.js new file mode 100644 index 000000000..7fbed5b48 --- /dev/null +++ b/packages/api/src/utility/LineReader.js @@ -0,0 +1,88 @@ +const readline = require('readline'); + +class Queue { + constructor() { + this.elements = {}; + this.head = 0; + this.tail = 0; + } + enqueue(element) { + this.elements[this.tail] = element; + this.tail++; + } + dequeue() { + const item = this.elements[this.head]; + delete this.elements[this.head]; + this.head++; + return item; + } + peek() { + return this.elements[this.head]; + } + getLength() { + return this.tail - this.head; + } + isEmpty() { + return this.getLength() === 0; + } +} + +class LineReader { + constructor(input) { + this.input = input; + this.queue = new Queue(); + this.resolve = null; + this.isEnded = false; + this.rl = readline.createInterface({ + input, + }); + this.input.pause(); + + this.rl.on('line', line => { + this.input.pause(); + if (this.resolve) { + const resolve = this.resolve; + this.resolve = null; + resolve(line); + return; + } + this.queue.enqueue(line); + }); + + this.rl.on('close', () => { + if (this.resolve) { + const resolve = this.resolve; + this.resolve = null; + this.isEnded = true; + resolve(null); + return; + } + this.queue.enqueue(null); + }); + } + + readLine() { + if (this.isEnded) { + return Promise.resolve(null); + } + + if (!this.queue.isEmpty()) { + const res = this.queue.dequeue(); + if (res == null) this.isEnded = true; + return Promise.resolve(res); + } + + this.input.resume(); + + return new Promise(resolve => { + this.resolve = resolve; + }); + } + + close() { + this.isEnded = true; + return new Promise(resolve => this.input.close(resolve)); + } +} + +module.exports = LineReader; From fb1c2c61fb0cfadbea9ffc9ea8b4c1dcbd916907 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 16 Feb 2023 17:25:54 +0100 Subject: [PATCH 05/40] duplicator improvements --- packages/datalib/src/DataDuplicator.ts | 8 +++ packages/tools/src/schemaEditorTools.ts | 2 +- .../web/src/tabs/DataDuplicatorTab.svelte | 61 +++++++++++++++++-- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index faa475708..e15bb0dc2 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -91,6 +91,7 @@ class DuplicatorItemHolder { let inserted = 0; let mapped = 0; let missing = 0; + let lastLogged = new Date(); const writeStream = createAsyncWriteStream(this.duplicator.stream, { processItem: async chunk => { @@ -146,6 +147,13 @@ class DuplicatorItemHolder { break; } } + + if (new Date().getTime() - lastLogged.getTime() > 5000) { + logger.info( + `Duplicating ${this.item.name} in progress, inserted ${inserted} rows, mapped ${mapped} rows, missing ${missing} rows` + ); + lastLogged = new Date(); + } // this.idMap[oldId] = newId; }, }); diff --git a/packages/tools/src/schemaEditorTools.ts b/packages/tools/src/schemaEditorTools.ts index b3a4b8f80..18e095198 100644 --- a/packages/tools/src/schemaEditorTools.ts +++ b/packages/tools/src/schemaEditorTools.ts @@ -129,7 +129,7 @@ export function editorAddColumn(table: TableInfo, column: EditorColumnInfo, addD if (addDataCommand && column.defaultValue) { defineDataCommand(res, () => ({ type: 'setField', - field: column.columnName, + newField: column.columnName, value: parseSqlDefaultValue(column.defaultValue), })); } diff --git a/packages/web/src/tabs/DataDuplicatorTab.svelte b/packages/web/src/tabs/DataDuplicatorTab.svelte index 9e7390c29..3a1ab4d95 100644 --- a/packages/web/src/tabs/DataDuplicatorTab.svelte +++ b/packages/web/src/tabs/DataDuplicatorTab.svelte @@ -12,6 +12,16 @@ testEnabled: () => getCurrentEditor()?.canRun(), onClick: () => getCurrentEditor().run(), }); + registerCommand({ + id: 'dataDuplicator.kill', + category: 'Data duplicator', + icon: 'icon close', + name: 'Kill', + toolbar: true, + isRelatedToTab: true, + testEnabled: () => getCurrentEditor()?.canKill(), + onClick: () => getCurrentEditor().kill(), + }); - +
@@ -185,6 +229,12 @@ +
+ Check all + | + Uncheck all +
+ + From 8109dd862eb49aa454758d6eaf19e78669507544 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 16 Feb 2023 18:00:04 +0100 Subject: [PATCH 06/40] change theme fix --- packages/web/src/commands/stdCommands.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/src/commands/stdCommands.ts b/packages/web/src/commands/stdCommands.ts index 62302dfc2..3fef3009c 100644 --- a/packages/web/src/commands/stdCommands.ts +++ b/packages/web/src/commands/stdCommands.ts @@ -58,7 +58,7 @@ registerCommand({ category: 'Theme', name: 'Change', toolbarName: 'Change theme', - onClick: () => showModal(SettingsModal, { selectedTab: 1 }), + onClick: () => showModal(SettingsModal, { selectedTab: 2 }), // getSubCommands: () => get(extensions).themes.map(themeCommand), }); From 1365f2b47c3a9f326f5d8620daf98279b27335d9 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 16 Feb 2023 18:27:05 +0100 Subject: [PATCH 07/40] duplicator options --- packages/api/src/shell/dataDuplicator.js | 5 +- packages/datalib/src/DataDuplicator.ts | 34 ++++++-- .../web/src/tabs/DataDuplicatorTab.svelte | 80 +++++++++++++++---- packages/web/src/widgets/StatusBar.svelte | 21 ++++- 4 files changed, 118 insertions(+), 22 deletions(-) diff --git a/packages/api/src/shell/dataDuplicator.js b/packages/api/src/shell/dataDuplicator.js index 15f308c69..dedc37d01 100644 --- a/packages/api/src/shell/dataDuplicator.js +++ b/packages/api/src/shell/dataDuplicator.js @@ -9,7 +9,7 @@ const copyStream = require('./copyStream'); const jsonLinesReader = require('./jsonLinesReader'); const { resolveArchiveFolder } = require('../utility/directories'); -async function dataDuplicator({ connection, archive, items, analysedStructure = null }) { +async function dataDuplicator({ connection, archive, items, options, analysedStructure = null }) { const driver = requireEngineDriver(connection); const pool = await connectUtility(driver, connection, 'write'); logger.info(`Connected.`); @@ -29,7 +29,8 @@ async function dataDuplicator({ connection, archive, items, analysedStructure = openStream: () => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) }), })), stream, - copyStream + copyStream, + options ); await dupl.run(); diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index e15bb0dc2..7a13e73ea 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -12,6 +12,11 @@ export interface DataDuplicatorItem { matchColumns: string[]; } +export interface DataDuplicatorOptions { + rollbackAfterFinish: boolean; + skipRowsWithUnresolvedRefs: boolean; +} + class DuplicatorReference { constructor( public base: DuplicatorItemHolder, @@ -78,6 +83,13 @@ class DuplicatorItemHolder { if (ref) { // remap id res[key] = ref.ref.idMap[res[key]]; + if (ref.isMandatory && res[key] == null) { + // mandatory refertence not matched + if (this.duplicator.options.skipRowsWithUnresolvedRefs) { + return null; + } + throw new Error(`Unresolved reference, base=${ref.base.name}, ref=${ref.ref.name}, ${key}=${chunk[key]}`); + } } } @@ -91,6 +103,7 @@ class DuplicatorItemHolder { let inserted = 0; let mapped = 0; let missing = 0; + let skipped = 0; let lastLogged = new Date(); const writeStream = createAsyncWriteStream(this.duplicator.stream, { @@ -101,6 +114,10 @@ class DuplicatorItemHolder { const doCopy = async () => { const insertedObj = this.createInsertObject(chunk); + if (insertedObj == null) { + skipped += 1; + return; + } await runCommandOnDriver(pool, driver, dmp => dmp.putCmd( '^insert ^into %f (%,i) ^values (%,v)', @@ -150,7 +167,7 @@ class DuplicatorItemHolder { if (new Date().getTime() - lastLogged.getTime() > 5000) { logger.info( - `Duplicating ${this.item.name} in progress, inserted ${inserted} rows, mapped ${mapped} rows, missing ${missing} rows` + `Duplicating ${this.item.name} in progress, inserted ${inserted} rows, mapped ${mapped} rows, missing ${missing} rows, skipped ${skipped} rows` ); lastLogged = new Date(); } @@ -166,7 +183,7 @@ class DuplicatorItemHolder { // }, // }); - return { inserted, mapped, missing }; + return { inserted, mapped, missing, skipped }; } } @@ -180,7 +197,8 @@ export class DataDuplicator { public db: DatabaseInfo, public items: DataDuplicatorItem[], public stream, - public copyStream: (input, output) => Promise + public copyStream: (input, output) => Promise, + public options: DataDuplicatorOptions ) { this.itemHolders = items.map(x => new DuplicatorItemHolder(x, this)); this.itemHolders.forEach(x => x.initializeReferences()); @@ -220,13 +238,19 @@ export class DataDuplicator { for (const item of this.itemPlan) { const stats = await item.runImport(); logger.info( - `Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows` + `Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows, skipped ${stats.skipped} rows` ); } } catch (err) { logger.error({ err }, 'Failed duplicator job, rollbacking'); await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction()); } - await runCommandOnDriver(this.pool, this.driver, dmp => dmp.commitTransaction()); + if (this.options.rollbackAfterFinish) { + logger.info('Rollbacking transaction, nothing was changed'); + await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction()); + } else { + logger.info('Committing duplicator transaction'); + await runCommandOnDriver(this.pool, this.driver, dmp => dmp.commitTransaction()); + } } } diff --git a/packages/web/src/tabs/DataDuplicatorTab.svelte b/packages/web/src/tabs/DataDuplicatorTab.svelte index 3a1ab4d95..c97e9d344 100644 --- a/packages/web/src/tabs/DataDuplicatorTab.svelte +++ b/packages/web/src/tabs/DataDuplicatorTab.svelte @@ -37,6 +37,7 @@ import TableControl from '../elements/TableControl.svelte'; import VerticalSplitter from '../elements/VerticalSplitter.svelte'; import CheckboxField from '../forms/CheckboxField.svelte'; + import FormFieldTemplateLarge from '../forms/FormFieldTemplateLarge.svelte'; import SelectField from '../forms/SelectField.svelte'; import { extractShellConnection } from '../impexp/createImpExpScript'; import SocketMessageView from '../query/SocketMessageView.svelte'; @@ -116,6 +117,10 @@ operation: row.operation, matchColumns: _.compact([row.matchColumn1]), })), + options: { + rollbackAfterFinish: !!$editorState.value?.rollbackAfterFinish, + skipRowsWithUnresolvedRefs: !!$editorState.value?.skipRowsWithUnresolvedRefs, + }, }); return script.getScript(); } @@ -211,21 +216,68 @@
-
Source archive
- { - setEditorData(old => ({ - ...old, - archiveFolder: e.detail, - })); + + { + setEditorData(old => ({ + ...old, + archiveFolder: e.detail, + })); + }} + options={$archiveFolders?.map(x => ({ + label: x.name, + value: x.name, + })) || []} + /> + + + { + setEditorData(old => ({ + ...old, + rollbackAfterFinish: !$editorState.value?.rollbackAfterFinish, + })); + }, }} - options={$archiveFolders?.map(x => ({ - label: x.name, - value: x.name, - })) || []} - /> + > + { + setEditorData(old => ({ + ...old, + rollbackAfterFinish: e.target.checked, + })); + }} + /> + + + { + setEditorData(old => ({ + ...old, + skipRowsWithUnresolvedRefs: !$editorState.value?.skipRowsWithUnresolvedRefs, + })); + }, + }} + > + { + setEditorData(old => ({ + ...old, + skipRowsWithUnresolvedRefs: e.target.checked, + })); + }} + /> + diff --git a/packages/web/src/widgets/StatusBar.svelte b/packages/web/src/widgets/StatusBar.svelte index b448245d4..e020fa5ff 100644 --- a/packages/web/src/widgets/StatusBar.svelte +++ b/packages/web/src/widgets/StatusBar.svelte @@ -6,7 +6,14 @@ import FontIcon from '../icons/FontIcon.svelte'; - import { activeTabId, currentDatabase, currentThemeDefinition, visibleCommandPalette } from '../stores'; + import { + activeTabId, + currentArchive, + currentDatabase, + currentThemeDefinition, + selectedWidget, + visibleCommandPalette, + } from '../stores'; import getConnectionLabel from '../utility/getConnectionLabel'; import { useConnectionList, useDatabaseServerVersion, useDatabaseStatus } from '../utility/metadataLoaders'; import { findCommand } from '../commands/runCommand'; @@ -140,6 +147,18 @@
{/if} + {#if $currentArchive} +
{ + $selectedWidget = 'archive'; + }} + > + + {$currentArchive} +
+ {/if}
{#each contextItems || [] as item} From 6a0feb235a3486276f7ec0162ba3432f4fc94d26 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 17 Feb 2023 08:46:31 +0100 Subject: [PATCH 08/40] fixed compilation error --- packages/datalib/src/ChangeSet.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/datalib/src/ChangeSet.ts b/packages/datalib/src/ChangeSet.ts index 0238061a3..28396351a 100644 --- a/packages/datalib/src/ChangeSet.ts +++ b/packages/datalib/src/ChangeSet.ts @@ -22,15 +22,18 @@ export interface ChangeSetItem { fields?: { [column: string]: string }; } -export interface ChangeSet { - structure?: TableInfo; - dataUpdateCommands?: JsonDataObjectUpdateCommand[]; - setColumnMode?: 'fixed' | 'variable'; +export interface ChangeSetItemFields { inserts: ChangeSetItem[]; updates: ChangeSetItem[]; deletes: ChangeSetItem[]; } +export interface ChangeSet extends ChangeSetItemFields { + structure?: TableInfo; + dataUpdateCommands?: JsonDataObjectUpdateCommand[]; + setColumnMode?: 'fixed' | 'variable'; +} + export function createChangeSet(): ChangeSet { return { inserts: [], @@ -55,7 +58,7 @@ export interface ChangeSetFieldDefinition extends ChangeSetRowDefinition { export function findExistingChangeSetItem( changeSet: ChangeSet, definition: ChangeSetRowDefinition -): [keyof ChangeSet, ChangeSetItem] { +): [keyof ChangeSetItemFields, ChangeSetItem] { if (!changeSet || !definition) return ['updates', null]; if (definition.insertedRowIndex != null) { return [ From b6c5f26eb47a2706c7424f10d2dbcbb7bda8aabf Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 17 Feb 2023 09:15:13 +0100 Subject: [PATCH 09/40] data duplicator test --- .../__tests__/data-duplicator.spec.js | 73 +++++++++++++++++++ integration-tests/engines.js | 4 +- packages/api/src/shell/dataDuplicator.js | 15 +++- packages/api/src/shell/fakeObjectReader.js | 26 ++++--- packages/datalib/src/DataDuplicator.ts | 6 +- 5 files changed, 107 insertions(+), 17 deletions(-) create mode 100644 integration-tests/__tests__/data-duplicator.spec.js diff --git a/integration-tests/__tests__/data-duplicator.spec.js b/integration-tests/__tests__/data-duplicator.spec.js new file mode 100644 index 000000000..08992645a --- /dev/null +++ b/integration-tests/__tests__/data-duplicator.spec.js @@ -0,0 +1,73 @@ +const engines = require('../engines'); +const { testWrapper } = require('../tools'); +const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator'); +const fakeObjectReader = require('dbgate-api/src/shell/fakeObjectReader'); + +const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val varchar(50) null)'; +const t2Sql = + 'CREATE TABLE t2 (id int not null primary key, val varchar(50) null, valfk int, foreign key (valfk) references t2(id))'; + +describe('Data duplicator', () => { + test.each(engines.map(engine => [engine.label, engine]))( + 'Insert simple data - %s', + testWrapper(async (conn, driver, engine) => { + await driver.query(conn, t1Sql); + await driver.query(conn, t2Sql); + + const t1 = await fakeObjectReader({ + dynamicData: [ + { id: 1, val: 'v1' }, + { id: 2, val: 'v2' }, + { id: 3, val: 'v3' }, + ], + }); + const t2 = await fakeObjectReader({ + dynamicData: [ + { id: 1, val: 'v1', valfk: 1 }, + { id: 2, val: 'v2', valfk: 2 }, + { id: 3, val: 'v3', valfk: 3 }, + ], + }); + + await dataDuplicator({ + systemConnection: conn, + driver, + items: [ + { + name: 't1', + operation: 'copy', + openStream: () => t1, + }, + { + name: 't2', + operation: 'copy', + openStream: () => t2, + }, + ], + }); + + await dataDuplicator({ + systemConnection: conn, + driver, + items: [ + { + name: 't1', + operation: 'copy', + openStream: () => t1, + }, + { + name: 't2', + operation: 'copy', + openStream: () => t2, + }, + ], + }); + + const res1 = await driver.query(conn, `select count(*) as cnt from t1`); + expect(res1.rows[0].cnt.toString()).toEqual('6'); + + const res2 = await driver.query(conn, `select count(*) as cnt from t2`); + expect(res2.rows[0].cnt.toString()).toEqual('6'); + }) + ); +}); diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 8f10950d5..41e0345aa 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -134,9 +134,9 @@ const engines = [ const filterLocal = [ // filter local testing '-MySQL', - '-MariaDB', + 'MariaDB', '-PostgreSQL', - 'SQL Server', + '-SQL Server', '-SQLite', '-CockroachDB', ]; diff --git a/packages/api/src/shell/dataDuplicator.js b/packages/api/src/shell/dataDuplicator.js index dedc37d01..702d3a7b8 100644 --- a/packages/api/src/shell/dataDuplicator.js +++ b/packages/api/src/shell/dataDuplicator.js @@ -9,9 +9,18 @@ const copyStream = require('./copyStream'); const jsonLinesReader = require('./jsonLinesReader'); const { resolveArchiveFolder } = require('../utility/directories'); -async function dataDuplicator({ connection, archive, items, options, analysedStructure = null }) { - const driver = requireEngineDriver(connection); - const pool = await connectUtility(driver, connection, 'write'); +async function dataDuplicator({ + connection, + archive, + items, + options, + analysedStructure = null, + driver, + systemConnection, +}) { + if (!driver) driver = requireEngineDriver(connection); + const pool = systemConnection || (await connectUtility(driver, connection, 'write')); + logger.info(`Connected.`); if (!analysedStructure) { diff --git a/packages/api/src/shell/fakeObjectReader.js b/packages/api/src/shell/fakeObjectReader.js index 18e2d70a4..d1bdb593e 100644 --- a/packages/api/src/shell/fakeObjectReader.js +++ b/packages/api/src/shell/fakeObjectReader.js @@ -1,18 +1,26 @@ const stream = require('stream'); -async function fakeObjectReader({ delay = 0 } = {}) { +async function fakeObjectReader({ delay = 0, dynamicData = null } = {}) { const pass = new stream.PassThrough({ objectMode: true, }); function doWrite() { - pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true }); - pass.write({ id: 1, country: 'Czechia' }); - pass.write({ id: 2, country: 'Austria' }); - pass.write({ country: 'Germany', id: 3 }); - pass.write({ country: 'Romania', id: 4 }); - pass.write({ country: 'Great Britain', id: 5 }); - pass.write({ country: 'Bosna, Hecegovina', id: 6 }); - pass.end(); + if (dynamicData) { + pass.write({ __isStreamHeader: true, __isDynamicStructure: true }); + for (const item of dynamicData) { + pass.write(item); + } + pass.end(); + } else { + pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true }); + pass.write({ id: 1, country: 'Czechia' }); + pass.write({ id: 2, country: 'Austria' }); + pass.write({ country: 'Germany', id: 3 }); + pass.write({ country: 'Romania', id: 4 }); + pass.write({ country: 'Great Britain', id: 5 }); + pass.write({ country: 'Bosna, Hecegovina', id: 6 }); + pass.end(); + } } if (delay) { diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index 7a13e73ea..49a51b6de 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -13,8 +13,8 @@ export interface DataDuplicatorItem { } export interface DataDuplicatorOptions { - rollbackAfterFinish: boolean; - skipRowsWithUnresolvedRefs: boolean; + rollbackAfterFinish?: boolean; + skipRowsWithUnresolvedRefs?: boolean; } class DuplicatorReference { @@ -198,7 +198,7 @@ export class DataDuplicator { public items: DataDuplicatorItem[], public stream, public copyStream: (input, output) => Promise, - public options: DataDuplicatorOptions + public options: DataDuplicatorOptions = {} ) { this.itemHolders = items.map(x => new DuplicatorItemHolder(x, this)); this.itemHolders.forEach(x => x.initializeReferences()); From 1ab58a491a5594b63eeaff02bd771285d17ba24b Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 17 Feb 2023 09:27:16 +0100 Subject: [PATCH 10/40] data duplicator test --- .../__tests__/data-duplicator.spec.js | 59 +++++++++++++------ packages/api/src/shell/dataDuplicator.js | 4 +- packages/datalib/src/DataDuplicator.ts | 1 + 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/integration-tests/__tests__/data-duplicator.spec.js b/integration-tests/__tests__/data-duplicator.spec.js index 08992645a..3227cd984 100644 --- a/integration-tests/__tests__/data-duplicator.spec.js +++ b/integration-tests/__tests__/data-duplicator.spec.js @@ -1,33 +1,54 @@ const engines = require('../engines'); +const stream = require('stream'); const { testWrapper } = require('../tools'); const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator'); -const fakeObjectReader = require('dbgate-api/src/shell/fakeObjectReader'); - -const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val varchar(50) null)'; -const t2Sql = - 'CREATE TABLE t2 (id int not null primary key, val varchar(50) null, valfk int, foreign key (valfk) references t2(id))'; +const { runCommandOnDriver } = require('dbgate-tools'); describe('Data duplicator', () => { test.each(engines.map(engine => [engine.label, engine]))( 'Insert simple data - %s', testWrapper(async (conn, driver, engine) => { - await driver.query(conn, t1Sql); - await driver.query(conn, t2Sql); + runCommandOnDriver(conn, driver, dmp => + dmp.createTable({ + pureName: 't1', + columns: [ + { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true }, + { columnName: 'val', dataType: 'varchar(50)' }, + ], + primaryKey: { + columns: [{ columnName: 'id' }], + }, + }) + ); + runCommandOnDriver(conn, driver, dmp => + dmp.createTable({ + pureName: 't2', + columns: [ + { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true }, + { columnName: 'val', dataType: 'varchar(50)' }, + { columnName: 'valfk', dataType: 'int', notNull: true }, + ], + primaryKey: { + columns: [{ columnName: 'id' }], + }, + foreignKeys: [{ refTableName: 't1', columns: [{ columnName: 'valfk', refColumnName: 'id' }] }], + }) + ); - const t1 = await fakeObjectReader({ - dynamicData: [ + const gett1 = () => + stream.Readable.from([ + { __isStreamHeader: true, __isDynamicStructure: true }, { id: 1, val: 'v1' }, { id: 2, val: 'v2' }, { id: 3, val: 'v3' }, - ], - }); - const t2 = await fakeObjectReader({ - dynamicData: [ + ]); + const gett2 = () => + stream.Readable.from([ + { __isStreamHeader: true, __isDynamicStructure: true }, { id: 1, val: 'v1', valfk: 1 }, { id: 2, val: 'v2', valfk: 2 }, { id: 3, val: 'v3', valfk: 3 }, - ], - }); + ]); await dataDuplicator({ systemConnection: conn, @@ -36,12 +57,12 @@ describe('Data duplicator', () => { { name: 't1', operation: 'copy', - openStream: () => t1, + openStream: gett1, }, { name: 't2', operation: 'copy', - openStream: () => t2, + openStream: gett2, }, ], }); @@ -53,12 +74,12 @@ describe('Data duplicator', () => { { name: 't1', operation: 'copy', - openStream: () => t1, + openStream: gett1, }, { name: 't2', operation: 'copy', - openStream: () => t2, + openStream: gett2, }, ], }); diff --git a/packages/api/src/shell/dataDuplicator.js b/packages/api/src/shell/dataDuplicator.js index 702d3a7b8..60b68daa4 100644 --- a/packages/api/src/shell/dataDuplicator.js +++ b/packages/api/src/shell/dataDuplicator.js @@ -35,7 +35,9 @@ async function dataDuplicator({ name: item.name, operation: item.operation, matchColumns: item.matchColumns, - openStream: () => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) }), + openStream: + item.openStream || + (() => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) })), })), stream, copyStream, diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index 49a51b6de..043cdc911 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -244,6 +244,7 @@ export class DataDuplicator { } catch (err) { logger.error({ err }, 'Failed duplicator job, rollbacking'); await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction()); + throw err; } if (this.options.rollbackAfterFinish) { logger.info('Rollbacking transaction, nothing was changed'); From 6b783027e50ee11292c92612b44b10507a808d4d Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 17 Feb 2023 10:00:21 +0100 Subject: [PATCH 11/40] data duplicator fix --- packages/datalib/src/DataDuplicator.ts | 21 ++++++++++++++----- packages/tools/src/driverBase.ts | 2 ++ packages/types/dialect.d.ts | 1 + .../src/frontend/drivers.js | 1 + 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index 043cdc911..938591e67 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -118,17 +118,28 @@ class DuplicatorItemHolder { skipped += 1; return; } - await runCommandOnDriver(pool, driver, dmp => - dmp.putCmd( + let res = await runQueryOnDriver(pool, driver, dmp => { + dmp.put( '^insert ^into %f (%,i) ^values (%,v)', this.table, Object.keys(insertedObj), Object.values(insertedObj) - ) - ); + ); + + if ( + this.autoColumn && + this.isReferenced && + !this.duplicator.driver.dialect.requireStandaloneSelectForScopeIdentity + ) { + dmp.selectScopeIdentity(this.table); + } + }); inserted += 1; if (this.autoColumn && this.isReferenced) { - const res = await runQueryOnDriver(pool, driver, dmp => dmp.selectScopeIdentity(this.table)); + if (this.duplicator.driver.dialect.requireStandaloneSelectForScopeIdentity) { + res = await runQueryOnDriver(pool, driver, dmp => dmp.selectScopeIdentity(this.table)); + } + // console.log('IDRES', JSON.stringify(res)); const resId = Object.entries(res?.rows?.[0])?.[0]?.[1]; if (resId != null) { this.idMap[chunk[this.autoColumn]] = resId; diff --git a/packages/tools/src/driverBase.ts b/packages/tools/src/driverBase.ts index ac7b312a6..626ed0b24 100644 --- a/packages/tools/src/driverBase.ts +++ b/packages/tools/src/driverBase.ts @@ -23,6 +23,7 @@ const dialect = { export async function runCommandOnDriver(pool, driver: EngineDriver, cmd: (dmp: SqlDumper) => void): Promise { const dmp = driver.createDumper(); cmd(dmp as any); + // console.log('CMD:', dmp.s); await driver.query(pool, dmp.s, { discardResult: true }); } @@ -33,6 +34,7 @@ export async function runQueryOnDriver( ): Promise { const dmp = driver.createDumper(); cmd(dmp as any); + // console.log('QUERY:', dmp.s); return await driver.query(pool, dmp.s); } diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index 8669b4f6a..166d8298f 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -11,6 +11,7 @@ export interface SqlDialect { anonymousPrimaryKey?: boolean; defaultSchemaName?: string; enableConstraintsPerTable?: boolean; + requireStandaloneSelectForScopeIdentity?: boolean; dropColumnDependencies?: string[]; changeColumnDependencies?: string[]; diff --git a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js index 3a98fefaa..20106c99c 100644 --- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js @@ -42,6 +42,7 @@ const dialect = { dropCheck: true, dropReferencesWhenDropTable: false, + requireStandaloneSelectForScopeIdentity: true, columnProperties: { columnComment: true, From 7802cde14d658aaa6ddb5431d013fdc33fe0d61b Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 17 Feb 2023 10:26:44 +0100 Subject: [PATCH 12/40] duplicator fixes --- packages/datalib/src/DataDuplicator.ts | 2 +- packages/web/src/tabs/DataDuplicatorTab.svelte | 1 + plugins/dbgate-plugin-postgres/src/frontend/drivers.js | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index 938591e67..92d083b9e 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -255,7 +255,7 @@ export class DataDuplicator { } catch (err) { logger.error({ err }, 'Failed duplicator job, rollbacking'); await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction()); - throw err; + return; } if (this.options.rollbackAfterFinish) { logger.info('Rollbacking transaction, nothing was changed'); diff --git a/packages/web/src/tabs/DataDuplicatorTab.svelte b/packages/web/src/tabs/DataDuplicatorTab.svelte index c97e9d344..e118f7137 100644 --- a/packages/web/src/tabs/DataDuplicatorTab.svelte +++ b/packages/web/src/tabs/DataDuplicatorTab.svelte @@ -158,6 +158,7 @@ const handleRunnerDone = () => { busy = false; + timerLabel.stop(); }; export function canKill() { diff --git a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js index 93aee6de7..fa2f2bb96 100644 --- a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js @@ -35,6 +35,7 @@ const dialect = { dropCheck: true, dropReferencesWhenDropTable: true, + requireStandaloneSelectForScopeIdentity: true, predefinedDataTypes: [ 'bigint', From 67e287cfdf56b20500bf1cfa42b70e066bda3d67 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 17 Feb 2023 10:41:01 +0100 Subject: [PATCH 13/40] added links from duplicator --- packages/datalib/src/DataDuplicator.ts | 2 + .../web/src/tabs/DataDuplicatorTab.svelte | 44 +++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index 92d083b9e..18ca1341d 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -113,7 +113,9 @@ class DuplicatorItemHolder { } const doCopy = async () => { + // console.log('chunk', this.name, JSON.stringify(chunk)); const insertedObj = this.createInsertObject(chunk); + // console.log('insertedObj', this.name, JSON.stringify(insertedObj)); if (insertedObj == null) { skipped += 1; return; diff --git a/packages/web/src/tabs/DataDuplicatorTab.svelte b/packages/web/src/tabs/DataDuplicatorTab.svelte index e118f7137..0cbe07517 100644 --- a/packages/web/src/tabs/DataDuplicatorTab.svelte +++ b/packages/web/src/tabs/DataDuplicatorTab.svelte @@ -39,6 +39,7 @@ import CheckboxField from '../forms/CheckboxField.svelte'; import FormFieldTemplateLarge from '../forms/FormFieldTemplateLarge.svelte'; import SelectField from '../forms/SelectField.svelte'; + import FontIcon from '../icons/FontIcon.svelte'; import { extractShellConnection } from '../impexp/createImpExpScript'; import SocketMessageView from '../query/SocketMessageView.svelte'; import useEditorData from '../query/useEditorData'; @@ -47,6 +48,7 @@ import { changeTab } from '../utility/common'; import createActivator, { getActiveComponent } from '../utility/createActivator'; import { useArchiveFiles, useArchiveFolders, useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders'; + import openNewTab from '../utility/openNewTab'; import useEffect from '../utility/useEffect'; import useTimerLabel from '../utility/useTimerLabel'; @@ -188,8 +190,10 @@ isChecked, operation, matchColumn1, - file: `${name}.jsonl`, + file: name, table: tableInfo?.schemaName ? `${tableInfo?.schemaName}.${tableInfo?.pureName}` : tableInfo?.pureName, + schemaName: tableInfo?.schemaName, + pureName: tableInfo?.pureName, }; }); @@ -292,8 +296,8 @@ rows={tableRows} columns={[ { header: '', fieldName: 'isChecked', slot: 1 }, - { header: 'Source file', fieldName: 'file' }, - { header: 'Target table', fieldName: 'table' }, + { header: 'Source file', fieldName: 'file', slot: 4 }, + { header: 'Target table', fieldName: 'table', slot: 5 }, { header: 'Operation', fieldName: 'operation', slot: 2 }, { header: 'Match column', fieldName: 'matchColumn1', slot: 3 }, ]} @@ -339,6 +343,40 @@ /> {/if} + + { + openNewTab({ + title: row.file, + icon: 'img archive', + tooltip: `${$editorState.value?.archiveFolder}\n${row.file}`, + tabComponent: 'ArchiveFileTab', + props: { + archiveFile: row.file, + archiveFolder: $editorState.value?.archiveFolder, + }, + }); + }}> {row.file} + + + { + openNewTab({ + title: row.pureName, + icon: 'img table', + tabComponent: 'TableDataTab', + props: { + schemaName: row.schemaName, + pureName: row.pureName, + conid, + database, + objectTypeField: 'tables', + }, + }); + }}> {row.table} +
From 7b6a1543de8f468a0770e71caf9b896397c11eac Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 17 Feb 2023 12:14:58 +0100 Subject: [PATCH 14/40] duplicator UX --- packages/web/src/elements/Link.svelte | 5 ++++- packages/web/src/tabs/DataDuplicatorTab.svelte | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/web/src/elements/Link.svelte b/packages/web/src/elements/Link.svelte index 84e9058a2..2f6de116b 100644 --- a/packages/web/src/elements/Link.svelte +++ b/packages/web/src/elements/Link.svelte @@ -1,15 +1,18 @@ { + on:click={e => { if (onClick) onClick(e); else openWebLink(href); }} + use:contextMenu={menu} > diff --git a/packages/web/src/tabs/DataDuplicatorTab.svelte b/packages/web/src/tabs/DataDuplicatorTab.svelte index 0cbe07517..f3b676224 100644 --- a/packages/web/src/tabs/DataDuplicatorTab.svelte +++ b/packages/web/src/tabs/DataDuplicatorTab.svelte @@ -51,6 +51,8 @@ import openNewTab from '../utility/openNewTab'; import useEffect from '../utility/useEffect'; import useTimerLabel from '../utility/useTimerLabel'; + import appObjectTypes from '../appobj'; + import RowHeaderCell from '../datagrid/RowHeaderCell.svelte'; export let conid; export let database; @@ -194,6 +196,7 @@ table: tableInfo?.schemaName ? `${tableInfo?.schemaName}.${tableInfo?.pureName}` : tableInfo?.pureName, schemaName: tableInfo?.schemaName, pureName: tableInfo?.pureName, + tableInfo, }; }); @@ -361,6 +364,7 @@ { openNewTab({ title: row.pureName, From c7aaf06506307479e4a7fbcb585e221414f3da57 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 17 Feb 2023 12:15:19 +0100 Subject: [PATCH 15/40] v5.2.3-beta.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 59f808cee..c9707c506 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "private": true, - "version": "5.2.3-beta.6", + "version": "5.2.3-beta.7", "name": "dbgate-all", "workspaces": [ "packages/*", From 36c792f44e2388720f1fa55cc217cbfd064c914f Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Fri, 17 Feb 2023 13:57:30 +0100 Subject: [PATCH 16/40] excel import fix --- .../web/src/impexp/ImportExportConfigurator.svelte | 3 ++- packages/web/src/plugins/PluginsProvider.svelte | 5 ----- plugins/dbgate-plugin-excel/src/frontend/index.js | 11 ++--------- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/packages/web/src/impexp/ImportExportConfigurator.svelte b/packages/web/src/impexp/ImportExportConfigurator.svelte index 982253d67..daa58a338 100644 --- a/packages/web/src/impexp/ImportExportConfigurator.svelte +++ b/packages/web/src/impexp/ImportExportConfigurator.svelte @@ -22,7 +22,7 @@ for (const file of getAsArray(files)) { const format = findFileFormat(extensions, storage); if (format) { - await (format.addFileToSourceList || addFileToSourceListDefault)(file, newSources, newValues); + await (format.addFileToSourceList || addFileToSourceListDefault)(file, newSources, newValues, apiCall); } } newValues['sourceList'] = [...(values.sourceList || []).filter(x => !newSources.includes(x)), ...newSources]; @@ -58,6 +58,7 @@ import { showModal } from '../modals/modalTools'; import { findFileFormat } from '../plugins/fileformats'; import { extensions } from '../stores'; + import { apiCall } from '../utility/api'; import getAsArray from '../utility/getAsArray'; import { useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders'; import { setUploadListener } from '../utility/uploadFiles'; diff --git a/packages/web/src/plugins/PluginsProvider.svelte b/packages/web/src/plugins/PluginsProvider.svelte index 28580b065..42a10bced 100644 --- a/packages/web/src/plugins/PluginsProvider.svelte +++ b/packages/web/src/plugins/PluginsProvider.svelte @@ -1,8 +1,4 @@ - - - - - {#each structure?.columns || [] as column, index} - {#if index == editingColumn} - { - dispatchChangeColumns( - $$props, - cols => cols.map((col, i) => (index == i ? { columnName } : col)), - row => _.mapKeys(row, (v, k) => (k == column.columnName ? columnName : k)) - ); - }} - onBlur={() => (editingColumn = null)} - focusOnCreate - blurOnEnter - existingNames={structure?.columns.map(x => x.columnName)} - /> - {:else} - (editingColumn = index)} - onRemove={() => { - dispatchChangeColumns($$props, cols => cols.filter((c, i) => i != index)); - }} - onUp={() => { - dispatchChangeColumns($$props, cols => exchange(cols, index, index - 1)); - }} - onDown={() => { - dispatchChangeColumns($$props, cols => exchange(cols, index, index + 1)); - }} - /> - {/if} - {/each} - { - dispatchChangeColumns($$props, cols => [...cols, { columnName }]); - }} - placeholder="New column" - existingNames={(structure?.columns || []).map(x => x.columnName)} - /> - diff --git a/packages/web/src/freetable/FreeTableGridCore.svelte b/packages/web/src/freetable/FreeTableGridCore.svelte deleted file mode 100644 index c791b16f6..000000000 --- a/packages/web/src/freetable/FreeTableGridCore.svelte +++ /dev/null @@ -1,84 +0,0 @@ - - - - - diff --git a/packages/web/src/freetable/FreeTableGrider.ts b/packages/web/src/freetable/FreeTableGrider.ts deleted file mode 100644 index 3d7e62b11..000000000 --- a/packages/web/src/freetable/FreeTableGrider.ts +++ /dev/null @@ -1,104 +0,0 @@ -import type { FreeTableModel } from 'dbgate-datalib'; -import Grider from '../datagrid/Grider'; - -export default class FreeTableGrider extends Grider { - public model: FreeTableModel; - private batchModel: FreeTableModel; - - constructor(public modelState, public dispatchModel) { - super(); - this.model = modelState && modelState.value; - } - getRowData(index: any) { - return this.model.rows?.[index]; - } - get rowCount() { - return this.model.rows?.length; - } - get currentModel(): FreeTableModel { - return this.batchModel || this.model; - } - set currentModel(value) { - if (this.batchModel) this.batchModel = value; - else this.dispatchModel({ type: 'set', value }); - } - setCellValue(index: number, uniqueName: string, value: any) { - const model = this.currentModel; - if (model.rows[index]) { - this.currentModel = { - ...model, - rows: model.rows.map((row, i) => (index == i ? { ...row, [uniqueName]: value } : row)), - }; - } - } - setRowData(index: number, document: any) { - const model = this.currentModel; - if (model.rows[index]) { - this.currentModel = { - ...model, - rows: model.rows.map((row, i) => (index == i ? document : row)), - }; - } - } - get editable() { - return true; - } - get canInsert() { - return true; - } - get allowSave() { - return true; - } - insertRow(): number { - const model = this.currentModel; - this.currentModel = { - ...model, - rows: [...model.rows, {}], - }; - return this.currentModel.rows.length - 1; - } - insertDocuments(documents: any[]): number { - const model = this.currentModel; - this.currentModel = { - ...model, - rows: [...model.rows, ...documents], - }; - return this.currentModel.rows.length - documents.length; - } - - deleteRow(index: number) { - const model = this.currentModel; - this.currentModel = { - ...model, - rows: model.rows.filter((row, i) => index != i), - }; - } - beginUpdate() { - this.batchModel = this.model; - } - endUpdate() { - if (this.model != this.batchModel) { - this.dispatchModel({ type: 'set', value: this.batchModel }); - this.batchModel = null; - } - } - - // static factory({ modelState, dispatchModel }): FreeTableGrider { - // return new FreeTableGrider(modelState, dispatchModel); - // } - // static factoryDeps({ modelState, dispatchModel }) { - // return [modelState, dispatchModel]; - // } - undo() { - this.dispatchModel({ type: 'undo' }); - } - redo() { - this.dispatchModel({ type: 'redo' }); - } - get canUndo() { - return this.modelState.canUndo; - } - get canRedo() { - return this.modelState.canRedo; - } -} diff --git a/packages/web/src/jsontree/JSONTree.svelte b/packages/web/src/jsontree/JSONTree.svelte index 1ffe37588..06b7a74c6 100644 --- a/packages/web/src/jsontree/JSONTree.svelte +++ b/packages/web/src/jsontree/JSONTree.svelte @@ -5,6 +5,7 @@ import openNewTab from '../utility/openNewTab'; import _ from 'lodash'; import { copyTextToClipboard } from '../utility/clipboard'; + import { openJsonLinesData } from '../utility/openJsonLinesData'; setContext('json-tree-context-key', {}); @@ -49,22 +50,9 @@ if (value && _.isArray(value)) { res.push({ - text: 'Open as data sheet', + text: 'Open as table', onClick: () => { - openNewTab( - { - title: 'Data #', - icon: 'img free-table', - tabComponent: 'FreeTableTab', - props: {}, - }, - { - editor: { - rows: value, - structure: { __isDynamicStructure: true, columns: [] }, - }, - } - ); + openJsonLinesData(value); }, }); } diff --git a/packages/web/src/tabs/FreeTableTab.svelte b/packages/web/src/tabs/FreeTableTab.svelte deleted file mode 100644 index 361181abe..000000000 --- a/packages/web/src/tabs/FreeTableTab.svelte +++ /dev/null @@ -1,167 +0,0 @@ - - - - -{#if isLoading} - -{:else if errorMessage} - -{:else} - - - - - - - -{/if} diff --git a/packages/web/src/tabs/index.js b/packages/web/src/tabs/index.js index 59243e4e4..fa73ab2ad 100644 --- a/packages/web/src/tabs/index.js +++ b/packages/web/src/tabs/index.js @@ -5,7 +5,6 @@ import * as TableStructureTab from './TableStructureTab.svelte'; import * as QueryTab from './QueryTab.svelte'; import * as ShellTab from './ShellTab.svelte'; import * as ArchiveFileTab from './ArchiveFileTab.svelte'; -import * as FreeTableTab from './FreeTableTab.svelte'; import * as PluginTab from './PluginTab.svelte'; import * as ChartTab from './ChartTab.svelte'; import * as MarkdownEditorTab from './MarkdownEditorTab.svelte'; @@ -38,7 +37,6 @@ export default { QueryTab, ShellTab, ArchiveFileTab, - FreeTableTab, PluginTab, ChartTab, MarkdownEditorTab, diff --git a/packages/web/src/utility/openJsonLinesData.ts b/packages/web/src/utility/openJsonLinesData.ts new file mode 100644 index 000000000..a84651b79 --- /dev/null +++ b/packages/web/src/utility/openJsonLinesData.ts @@ -0,0 +1,17 @@ +import uuidv1 from 'uuid/v1'; +import { apiCall } from './api'; +import openNewTab from './openNewTab'; + +export async function openJsonLinesData(rows) { + const jslid = uuidv1(); + + await apiCall('jsldata/save-rows', { jslid, rows }); + openNewTab({ + tabComponent: 'ArchiveFileTab', + icon: 'img archive', + title: 'Data #', + props: { + jslid, + }, + }); +} diff --git a/packages/web/src/widgets/ArchiveFilesList.svelte b/packages/web/src/widgets/ArchiveFilesList.svelte index b3ed4a6b1..8c39c8a25 100644 --- a/packages/web/src/widgets/ArchiveFilesList.svelte +++ b/packages/web/src/widgets/ArchiveFilesList.svelte @@ -42,30 +42,26 @@ apiCall('archive/refresh-files', { folder }); }; - function handleNewDataSheet() { + function handleNewJsonLines() { showModal(InputTextModal, { value: '', label: 'New file name', - header: 'Create new data sheet', + header: 'Create new JSON lines', onConfirm: async file => { - await apiCall('archive/save-free-table', { + await apiCall('archive/save-rows', { folder: $currentArchive, file, - data: createFreeTableModel(), + rows: [ + { id: 1, value: 'val1' }, + { id: 1, value: 'val2' }, + ], }); openNewTab({ title: file, - icon: 'img free-table', - tabComponent: 'FreeTableTab', + icon: 'img archive', + tabComponent: 'ArchiveFileTab', props: { - initialArgs: { - functionName: 'archiveReader', - props: { - fileName: file, - folderName: $currentArchive, - }, - }, archiveFile: file, archiveFolder: $currentArchive, }, @@ -75,7 +71,7 @@ } function createAddMenu() { - return [{ text: 'New data sheet', onClick: handleNewDataSheet }]; + return [{ text: 'New NDJSON file', onClick: handleNewJsonLines }]; } From 1c73920dd597f838cea0b0601ee3a71074192b5f Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 25 Feb 2023 11:34:19 +0100 Subject: [PATCH 25/40] save jsl data --- packages/api/src/controllers/archive.js | 15 ++- packages/api/src/controllers/jsldata.js | 6 ++ packages/web/src/tabs/ArchiveFileTab.svelte | 95 +++++++++++++++---- packages/web/src/utility/openJsonLinesData.ts | 21 ++-- packages/web/src/utility/openNewTab.ts | 2 +- 5 files changed, 108 insertions(+), 31 deletions(-) diff --git a/packages/api/src/controllers/archive.js b/packages/api/src/controllers/archive.js index dcf7eee31..4fd1d8bfd 100644 --- a/packages/api/src/controllers/archive.js +++ b/packages/api/src/controllers/archive.js @@ -169,11 +169,20 @@ module.exports = { }, saveJslData_meta: true, - async saveJslData({ folder, file, jslid }) { + async saveJslData({ folder, file, jslid, changeSet }) { const source = getJslFileName(jslid); const target = path.join(resolveArchiveFolder(folder), `${file}.jsonl`); - await fs.copyFile(source, target); - socket.emitChanged(`archive-files-changed`, { folder }); + if (changeSet) { + const reader = await dbgateApi.modifyJsonLinesReader({ + fileName: source, + changeSet, + }); + const writer = await dbgateApi.jsonLinesWriter({ fileName: target }); + await dbgateApi.copyStream(reader, writer); + } else { + await fs.copyFile(source, target); + socket.emitChanged(`archive-files-changed`, { folder }); + } return true; }, diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index 3da64d2e8..0cf04a978 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -147,6 +147,12 @@ module.exports = { return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters, _.isEmpty(sort) ? null : sort); }, + exists_meta: true, + async exists({ jslid }) { + const fileName = getJslFileName(jslid); + return fs.existsSync(fileName); + }, + getStats_meta: true, getStats({ jslid }) { const file = `${getJslFileName(jslid)}.stats`; diff --git a/packages/web/src/tabs/ArchiveFileTab.svelte b/packages/web/src/tabs/ArchiveFileTab.svelte index 7ce9a347d..66bd0a16f 100644 --- a/packages/web/src/tabs/ArchiveFileTab.svelte +++ b/packages/web/src/tabs/ArchiveFileTab.svelte @@ -18,7 +18,8 @@ - + {#if jslidChecked || !jslid} + + {/if} diff --git a/packages/web/src/utility/openJsonLinesData.ts b/packages/web/src/utility/openJsonLinesData.ts index a84651b79..27c0b53cb 100644 --- a/packages/web/src/utility/openJsonLinesData.ts +++ b/packages/web/src/utility/openJsonLinesData.ts @@ -5,13 +5,18 @@ import openNewTab from './openNewTab'; export async function openJsonLinesData(rows) { const jslid = uuidv1(); - await apiCall('jsldata/save-rows', { jslid, rows }); - openNewTab({ - tabComponent: 'ArchiveFileTab', - icon: 'img archive', - title: 'Data #', - props: { - jslid, + // await apiCall('jsldata/save-rows', { jslid, rows }); + openNewTab( + { + tabComponent: 'ArchiveFileTab', + icon: 'img archive', + title: 'Data #', + props: { + jslid, + }, }, - }); + { + rows, + } + ); } diff --git a/packages/web/src/utility/openNewTab.ts b/packages/web/src/utility/openNewTab.ts index 1fefd8798..080557fee 100644 --- a/packages/web/src/utility/openNewTab.ts +++ b/packages/web/src/utility/openNewTab.ts @@ -65,7 +65,7 @@ export default async function openNewTab(newTab, initialData = undefined, option const tabid = uuidv1(); if (initialData) { for (const key of _.keys(initialData)) { - if (key == 'editor') { + if (key == 'editor' || key == 'rows') { await localforage.setItem(`tabdata_${key}_${tabid}`, initialData[key]); } else { localStorage.setItem(`tabdata_${key}_${tabid}`, JSON.stringify(initialData[key])); From fa24d47c0358d1d612b9bc4142428230772571b9 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 25 Feb 2023 11:34:55 +0100 Subject: [PATCH 26/40] fixed tab component --- packages/web/src/TabRegister.svelte | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/web/src/TabRegister.svelte b/packages/web/src/TabRegister.svelte index d5bf9de5d..b70b1b184 100644 --- a/packages/web/src/TabRegister.svelte +++ b/packages/web/src/TabRegister.svelte @@ -4,7 +4,6 @@ if (tabComponent) { return { tabComponent, - props: selectedTab && selectedTab.props, }; } return null; @@ -50,12 +49,14 @@ } } } + + $: openedTabsByTabId = _.keyBy($openedTabs, x => x.tabid); {#each _.keys(mountedTabs) as tabid (tabid)} From 9fe689625e23780e379940f6f9933c278aeb3d7d Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 25 Feb 2023 11:36:16 +0100 Subject: [PATCH 27/40] simplified tab register --- packages/web/src/TabRegister.svelte | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/packages/web/src/TabRegister.svelte b/packages/web/src/TabRegister.svelte index b70b1b184..3f3f3966d 100644 --- a/packages/web/src/TabRegister.svelte +++ b/packages/web/src/TabRegister.svelte @@ -1,15 +1,3 @@ - - - -
-
{column.columnName}
-
- - - - - - - - - - - - -
-
- - diff --git a/packages/web/src/freetable/ColumnNameEditor.svelte b/packages/web/src/freetable/ColumnNameEditor.svelte deleted file mode 100644 index 0cb5975f7..000000000 --- a/packages/web/src/freetable/ColumnNameEditor.svelte +++ /dev/null @@ -1,55 +0,0 @@ - - - - - diff --git a/packages/web/src/freetable/MacroDetail.svelte b/packages/web/src/macro/MacroDetail.svelte similarity index 100% rename from packages/web/src/freetable/MacroDetail.svelte rename to packages/web/src/macro/MacroDetail.svelte diff --git a/packages/web/src/freetable/MacroHeader.svelte b/packages/web/src/macro/MacroHeader.svelte similarity index 100% rename from packages/web/src/freetable/MacroHeader.svelte rename to packages/web/src/macro/MacroHeader.svelte diff --git a/packages/web/src/freetable/MacroInfoTab.svelte b/packages/web/src/macro/MacroInfoTab.svelte similarity index 100% rename from packages/web/src/freetable/MacroInfoTab.svelte rename to packages/web/src/macro/MacroInfoTab.svelte diff --git a/packages/web/src/freetable/MacroManager.svelte b/packages/web/src/macro/MacroManager.svelte similarity index 100% rename from packages/web/src/freetable/MacroManager.svelte rename to packages/web/src/macro/MacroManager.svelte diff --git a/packages/web/src/freetable/MacroParameters.svelte b/packages/web/src/macro/MacroParameters.svelte similarity index 100% rename from packages/web/src/freetable/MacroParameters.svelte rename to packages/web/src/macro/MacroParameters.svelte diff --git a/packages/web/src/freetable/MacroPreviewGrider.ts b/packages/web/src/macro/MacroPreviewGrider.ts similarity index 100% rename from packages/web/src/freetable/MacroPreviewGrider.ts rename to packages/web/src/macro/MacroPreviewGrider.ts diff --git a/packages/web/src/freetable/macros.js b/packages/web/src/macro/macros.js similarity index 100% rename from packages/web/src/freetable/macros.js rename to packages/web/src/macro/macros.js From 490efb065a163e24d08c837dc718a10bac5fee0e Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 25 Feb 2023 13:31:24 +0100 Subject: [PATCH 30/40] fixes sqlite autoincrement column creation --- packages/tools/src/SqlDumper.ts | 24 +++++++++++-------- .../src/frontend/Dumper.js | 18 +++++++++++++- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index 590b62c2f..54cfd8d23 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -244,16 +244,7 @@ export class SqlDumper implements AlterProcessor { this.put('%i ', col.columnName); this.columnDefinition(col); }); - if (table.primaryKey) { - this.put(',&n'); - if (table.primaryKey.constraintName) { - this.put('^constraint %i', table.primaryKey.constraintName); - } - this.put( - ' ^primary ^key (%,i)', - table.primaryKey.columns.map(x => x.columnName) - ); - } + this.createTablePrimaryKeyCore(table); (table.foreignKeys || []).forEach(fk => { this.put(',&n'); @@ -275,6 +266,19 @@ export class SqlDumper implements AlterProcessor { }); } + createTablePrimaryKeyCore(table: TableInfo) { + if (table.primaryKey) { + this.put(',&n'); + if (table.primaryKey.constraintName) { + this.put('^constraint %i', table.primaryKey.constraintName); + } + this.put( + ' ^primary ^key (%,i)', + table.primaryKey.columns.map(x => x.columnName) + ); + } + } + createForeignKeyFore(fk: ForeignKeyInfo) { if (fk.constraintName != null) this.put('^constraint %i ', fk.constraintName); this.put( diff --git a/plugins/dbgate-plugin-sqlite/src/frontend/Dumper.js b/plugins/dbgate-plugin-sqlite/src/frontend/Dumper.js index e10b6ab7b..d2aa90e50 100644 --- a/plugins/dbgate-plugin-sqlite/src/frontend/Dumper.js +++ b/plugins/dbgate-plugin-sqlite/src/frontend/Dumper.js @@ -18,7 +18,23 @@ class Dumper extends SqlDumper { } selectScopeIdentity() { - this.put('^select last_insert_rowid()') + this.put('^select last_insert_rowid()'); + } + + columnDefinition(column, flags) { + if (column.dataType && column.dataType.toLowerCase().includes('int') && column.notNull && column.autoIncrement) { + this.put('^integer ^primary ^key ^autoincrement'); + return; + } + super.columnDefinition(column, flags); + } + + createTablePrimaryKeyCore(table) { + const column = table.columns.find((x) => x.autoIncrement); + if (column && column.dataType && column.dataType.toLowerCase().includes('int') && column.notNull) { + return; + } + super.createTablePrimaryKeyCore(table); } } From 2d74b831c5f437a20b89ed0d6e13f00597d3cb59 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 25 Feb 2023 13:33:33 +0100 Subject: [PATCH 31/40] fixed sqlite data duplicator --- integration-tests/engines.js | 4 ++-- integration-tests/package.json | 2 ++ plugins/dbgate-plugin-sqlite/src/frontend/driver.js | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 41e0345aa..196919223 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -134,10 +134,10 @@ const engines = [ const filterLocal = [ // filter local testing '-MySQL', - 'MariaDB', + '-MariaDB', '-PostgreSQL', '-SQL Server', - '-SQLite', + 'SQLite', '-CockroachDB', ]; diff --git a/integration-tests/package.json b/integration-tests/package.json index b647b30ab..81ed8d4a4 100644 --- a/integration-tests/package.json +++ b/integration-tests/package.json @@ -13,6 +13,8 @@ "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js", "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest", + "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js", + "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults", "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local" diff --git a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js index de6071d77..a24bacb5a 100644 --- a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js +++ b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js @@ -22,6 +22,7 @@ const dialect = { return `[${s}]`; }, anonymousPrimaryKey: true, + requireStandaloneSelectForScopeIdentity: true, createColumn: true, dropColumn: true, From c817bf59115d243a50fc5fdb0aa00f7f904ac025 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 25 Feb 2023 18:24:00 +0100 Subject: [PATCH 32/40] added import/export tab (not used) --- .../src/appobj/DatabaseObjectAppObject.svelte | 16 ++ packages/web/src/icons/FontIcon.svelte | 2 + packages/web/src/tabs/ImportExportTab.svelte | 197 ++++++++++++++++++ packages/web/src/tabs/index.js | 2 + 4 files changed, 217 insertions(+) create mode 100644 packages/web/src/tabs/ImportExportTab.svelte diff --git a/packages/web/src/appobj/DatabaseObjectAppObject.svelte b/packages/web/src/appobj/DatabaseObjectAppObject.svelte index 14acbd702..650308a9a 100644 --- a/packages/web/src/appobj/DatabaseObjectAppObject.svelte +++ b/packages/web/src/appobj/DatabaseObjectAppObject.svelte @@ -702,6 +702,22 @@ }, { onClick: () => { + // openNewTab( + // { + // tabComponent: 'ImportExportTab', + // title: 'Import/Export', + // icon: 'img export', + // }, + // { + // editor: { + // sourceStorageType: 'database', + // sourceConnectionId: data.conid, + // sourceDatabaseName: data.database, + // sourceSchemaName: data.schemaName, + // sourceList: [data.pureName], + // }, + // } + // ); showModal(ImportExportModal, { initialValues: { sourceStorageType: 'database', diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index 189098a5a..27eb71de3 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -228,6 +228,8 @@ 'img keydb': 'mdi mdi-key color-icon-blue', 'img duplicator': 'mdi mdi-content-duplicate color-icon-green', + 'img import': 'mdi mdi-database-import color-icon-green', + 'img export': 'mdi mdi-database-export color-icon-green', }; diff --git a/packages/web/src/tabs/ImportExportTab.svelte b/packages/web/src/tabs/ImportExportTab.svelte new file mode 100644 index 000000000..2e065d33f --- /dev/null +++ b/packages/web/src/tabs/ImportExportTab.svelte @@ -0,0 +1,197 @@ + + + + +
+ + + {#if busy} + + {/if} +
+ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/web/src/tabs/index.js b/packages/web/src/tabs/index.js index fa73ab2ad..b828fdb7e 100644 --- a/packages/web/src/tabs/index.js +++ b/packages/web/src/tabs/index.js @@ -28,6 +28,7 @@ import * as PerspectiveTab from './PerspectiveTab.svelte'; import * as ServerSummaryTab from './ServerSummaryTab.svelte'; import * as ProfilerTab from './ProfilerTab.svelte'; import * as DataDuplicatorTab from './DataDuplicatorTab.svelte'; +import * as ImportExportTab from './ImportExportTab.svelte'; export default { TableDataTab, @@ -60,4 +61,5 @@ export default { ServerSummaryTab, ProfilerTab, DataDuplicatorTab, + ImportExportTab, }; From 0c62349802ab7a396519f60a6464fd18e475e78d Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 25 Feb 2023 20:25:27 +0100 Subject: [PATCH 33/40] fixed error reporting problems --- packages/api/src/controllers/runners.js | 18 ++++++++++----- packages/api/src/shell/runScript.js | 2 +- packages/datalib/src/DataDuplicator.ts | 2 +- packages/tools/src/createAsyncWriteStream.ts | 23 +++++--------------- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index 393c64f84..d10eac39d 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -70,15 +70,20 @@ module.exports = { if (message) { const json = safeJsonParse(message.message); - if (json) logger.info(json); + if (json) logger.log(json); else logger.info(message.message); - socket.emit(`runner-info-${runid}`, { + const toEmit = { time: new Date(), - severity: 'info', ...message, message: json ? json.msg : message.message, - }); + }; + + if (json && json.level >= 50) { + toEmit.severity = 'error'; + } + + socket.emit(`runner-info-${runid}`, toEmit); } }, @@ -125,8 +130,9 @@ module.exports = { }, } ); - const pipeDispatcher = severity => data => - this.dispatchMessage(runid, { severity, message: data.toString().trim() }); + const pipeDispatcher = severity => data => { + return this.dispatchMessage(runid, { severity, message: data.toString().trim() }); + }; byline(subprocess.stdout).on('data', pipeDispatcher('info')); byline(subprocess.stderr).on('data', pipeDispatcher('error')); diff --git a/packages/api/src/shell/runScript.js b/packages/api/src/shell/runScript.js index 331e4e1c8..5ce4c1a80 100644 --- a/packages/api/src/shell/runScript.js +++ b/packages/api/src/shell/runScript.js @@ -11,7 +11,7 @@ async function runScript(func) { await func(); process.exit(0); } catch (err) { - logger.error('Error running script', err); + logger.error({ err }, `Error running script: ${err.message}`); process.exit(1); } } diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index 18ca1341d..f78c42149 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -255,7 +255,7 @@ export class DataDuplicator { ); } } catch (err) { - logger.error({ err }, 'Failed duplicator job, rollbacking'); + logger.error({ err }, `Failed duplicator job, rollbacking. ${err.message}`); await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction()); return; } diff --git a/packages/tools/src/createAsyncWriteStream.ts b/packages/tools/src/createAsyncWriteStream.ts index 81dd1155e..46a0ce95e 100644 --- a/packages/tools/src/createAsyncWriteStream.ts +++ b/packages/tools/src/createAsyncWriteStream.ts @@ -14,23 +14,12 @@ export function createAsyncWriteStream(stream, options: AsyncWriteStreamOptions) }); writable._write = async (chunk, encoding, callback) => { - await options.processItem(chunk); - - // const { sql, id, newIdSql } = chunk; - // if (_isArray(sql)) { - // for (const item of sql) await driver.query(pool, item, { discardResult: true }); - // } else { - // await driver.query(pool, sql, { discardResult: true }); - // } - // if (newIdSql) { - // const res = await driver.query(pool, newIdSql); - // const resId = Object.entries(res?.rows?.[0])?.[0]?.[1]; - - // if (options?.mapResultId) { - // options?.mapResultId(id, resId as string); - // } - // } - callback(); + try { + await options.processItem(chunk); + callback(null); + } catch (err) { + callback(err); + } }; // writable._final = async callback => { From 7ec23ecca4318a9cc8d94bd1bf0201db3911c83f Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 26 Feb 2023 08:41:30 +0100 Subject: [PATCH 34/40] fixed modify archive for windows --- packages/api/src/controllers/archive.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/api/src/controllers/archive.js b/packages/api/src/controllers/archive.js index 4fd1d8bfd..eb1a546e2 100644 --- a/packages/api/src/controllers/archive.js +++ b/packages/api/src/controllers/archive.js @@ -9,6 +9,7 @@ const { getLogger } = require('dbgate-tools'); const uuidv1 = require('uuid/v1'); const dbgateApi = require('../shell'); const jsldata = require('./jsldata'); +const platformInfo = require('../utility/platformInfo'); const logger = getLogger('archive'); @@ -136,8 +137,13 @@ module.exports = { }); const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath }); await dbgateApi.copyStream(reader, writer); - await fs.unlink(changedFilePath); - await fs.rename(path.join(tmpchangedFilePath), path.join(changedFilePath)); + if (platformInfo.isWindows) { + await fs.copyFile(tmpchangedFilePath, changedFilePath); + await fs.unlink(tmpchangedFilePath); + } else { + await fs.unlink(changedFilePath); + await fs.rename(tmpchangedFilePath, changedFilePath); + } return true; }, From a588d72b261a3fbe22dc1308f25688768e5c290a Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 26 Feb 2023 08:58:23 +0100 Subject: [PATCH 35/40] create default archive by default --- packages/api/src/utility/directories.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/api/src/utility/directories.js b/packages/api/src/utility/directories.js index 841ccb846..3d1f4f516 100644 --- a/packages/api/src/utility/directories.js +++ b/packages/api/src/utility/directories.js @@ -42,18 +42,23 @@ function datadir() { return dir; } -const dirFunc = (dirname, clean) => () => { - const dir = path.join(datadir(), dirname); - ensureDirectory(dir, clean); +const dirFunc = + (dirname, clean, subdirs = []) => + () => { + const dir = path.join(datadir(), dirname); + ensureDirectory(dir, clean); + for (const subdir of subdirs) { + ensureDirectory(path.join(dir, subdir), false); + } - return dir; -}; + return dir; + }; const jsldir = dirFunc('jsl', true); const rundir = dirFunc('run', true); const uploadsdir = dirFunc('uploads', true); const pluginsdir = dirFunc('plugins'); -const archivedir = dirFunc('archive'); +const archivedir = dirFunc('archive', false, ['default']); const appdir = dirFunc('apps'); const filesdir = dirFunc('files'); const logsdir = dirFunc('logs', 3600 * 24 * 7); From 64ceea37795cac7f4ca23d5ea7b36d4ea69c17cf Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 26 Feb 2023 09:05:36 +0100 Subject: [PATCH 36/40] fiuxed dependency --- plugins/dbgate-plugin-oracle/package.json | 2 +- yarn.lock | 21 +-------------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/plugins/dbgate-plugin-oracle/package.json b/plugins/dbgate-plugin-oracle/package.json index 9ed66790c..8fcf8cab0 100644 --- a/plugins/dbgate-plugin-oracle/package.json +++ b/plugins/dbgate-plugin-oracle/package.json @@ -32,7 +32,7 @@ "devDependencies": { "dbgate-plugin-tools": "^1.0.8", "dbgate-query-splitter": "^4.9.0", - "dbgate-tools": "^5.1.6", + "dbgate-tools": "^5.0.0-alpha.1", "lodash": "^4.17.21", "webpack": "^4.42.0", "webpack-cli": "^3.3.11" diff --git a/yarn.lock b/yarn.lock index 598064f71..6f5286890 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3349,30 +3349,11 @@ dbgate-plugin-xml@^5.0.0-alpha.1: resolved "https://registry.yarnpkg.com/dbgate-plugin-xml/-/dbgate-plugin-xml-5.0.9.tgz#c3abf6ed8cd1450c45058d35c9326458833ed27e" integrity sha512-P8Em1A6HhF0BfxEDDEUyzdgFeJHEC5vbg12frANpWHjO3V1HGdygsT2z1ukLK8FS5BLW/vcCdOFldXZGh+wWvg== -dbgate-query-splitter@^4.9.0, dbgate-query-splitter@^4.9.2, dbgate-query-splitter@^4.9.3: +dbgate-query-splitter@^4.9.0, dbgate-query-splitter@^4.9.3: version "4.9.3" resolved "https://registry.yarnpkg.com/dbgate-query-splitter/-/dbgate-query-splitter-4.9.3.tgz#f66396da9ae3cc8f775a282143bfca3441248aa2" integrity sha512-QMppAy3S6NGQMawNokmhbpZURvLCETyu/8yTfqWUHGdlK963fdSpmoX1A+9SjCDp62sX0vYntfD7uzd6jVSRcw== -dbgate-sqltree@^5.1.6: - version "5.1.6" - resolved "https://registry.yarnpkg.com/dbgate-sqltree/-/dbgate-sqltree-5.1.6.tgz#469fe6e06b5146afb9104114564e64aa46c13e7c" - integrity sha512-D2ffjeT5HsHBOeW0ORYkCLbDhyIvH6hrE1A/IG3kRZD1iX8Dy6c0h/BWNa4xOjz3pbJYkaB0ju8rIKJqi0BYog== - dependencies: - lodash "^4.17.21" - -dbgate-tools@^5.1.6: - version "5.1.6" - resolved "https://registry.yarnpkg.com/dbgate-tools/-/dbgate-tools-5.1.6.tgz#43c1e0575db550da5ba38627eaa19839df05d7f1" - integrity sha512-KTEqSnNzIdGfYJaIvEusch3/KnO0p376VRykJcK1+/+UtXf9Dk+azaWFjEoACBK3DlK1hmL1pEGfMh5FHDm7Qw== - dependencies: - dbgate-query-splitter "^4.9.2" - dbgate-sqltree "^5.1.6" - debug "^4.3.4" - json-stable-stringify "^1.0.1" - lodash "^4.17.21" - uuid "^3.4.0" - debug@2.6.9, debug@^2.2.0, debug@^2.3.3: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" From 622773fccdf664fec81b7b654cb8e6726ab146c3 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 26 Feb 2023 09:40:12 +0100 Subject: [PATCH 37/40] optimalization of loading oracle structure --- .../api/src/proc/databaseConnectionProcess.js | 2 +- packages/tools/src/DatabaseAnalyser.ts | 5 ++++- .../src/backend/Analyser.js | 20 +++++++++++-------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index 9684cfe1a..9a8756431 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -271,7 +271,7 @@ async function handleSqlPreview({ msgid, objects, options }) { process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated }); if (generator.isUnhandledException) { setTimeout(() => { - getLogger.info('Exiting because of unhandled exception'); + logger.error('Exiting because of unhandled exception'); process.exit(0); }, 500); } diff --git a/packages/tools/src/DatabaseAnalyser.ts b/packages/tools/src/DatabaseAnalyser.ts index c3e1f24aa..f113f7a4b 100644 --- a/packages/tools/src/DatabaseAnalyser.ts +++ b/packages/tools/src/DatabaseAnalyser.ts @@ -235,7 +235,10 @@ export class DatabaseAnalyser { if (this.pool.feedback) { this.pool.feedback(obj); } - } + if (obj && obj.analysingMessage) { + logger.debug(obj.analysingMessage); + } + } async getModifications() { const snapshot = await this._getFastSnapshot(); diff --git a/plugins/dbgate-plugin-oracle/src/backend/Analyser.js b/plugins/dbgate-plugin-oracle/src/backend/Analyser.js index 45aeb777f..e5e39e8f5 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-oracle/src/backend/Analyser.js @@ -104,7 +104,7 @@ class Analyser extends DatabaseAnalyser { const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']); this.feedback({ analysingMessage: 'Finalizing DB structure' }); - const columnColumnsMapped = fkColumns.rows.map(x => ({ + const fkColumnsMapped = fkColumns.rows.map(x => ({ pureName: x.pure_name, schemaName: x.schema_name, constraintSchema: x.constraint_schema, @@ -124,6 +124,9 @@ class Analyser extends DatabaseAnalyser { columnName: x.column_name, })); + const columnGroup = col => `${col.schema_name}||${col.pure_name}`; + const columnsGrouped = _.groupBy(columns.rows, columnGroup); + const res = { tables: tables.rows.map(table => { const newTable = { @@ -134,11 +137,11 @@ class Analyser extends DatabaseAnalyser { }; return { ...newTable, - columns: columns.rows - .filter(col => col.pure_name == table.pure_name && col.schema_name == table.schema_name) - .map(col => getColumnInfo(col, newTable, geometryColumns, geographyColumns)), + columns: (columnsGrouped[columnGroup(table)] || []).map(col => + getColumnInfo(col, newTable, geometryColumns, geographyColumns) + ), primaryKey: DatabaseAnalyser.extractPrimaryKeys(newTable, pkColumnsMapped), - foreignKeys: DatabaseAnalyser.extractForeignKeys(newTable, columnColumnsMapped), + foreignKeys: DatabaseAnalyser.extractForeignKeys(newTable, fkColumnsMapped), indexes: _.uniqBy( indexes.rows.filter( idx => @@ -176,9 +179,7 @@ class Analyser extends DatabaseAnalyser { schemaName: view.schema_name, contentHash: view.hash_code, createSql: `CREATE VIEW "${view.schema_name}"."${view.pure_name}"\nAS\n${view.create_sql}`, - columns: columns.rows - .filter(col => col.pure_name == view.pure_name && col.schema_name == view.schema_name) - .map(col => getColumnInfo(col)), + columns: (columnsGrouped[columnGroup(view)] || []).map(col => getColumnInfo(col)), })), matviews: matviews ? matviews.rows.map(matview => ({ @@ -212,6 +213,9 @@ class Analyser extends DatabaseAnalyser { })), }; + // this.feedback({ analysingMessage: 'Debug sleep' }); + // await new Promise(resolve => setTimeout(resolve, 90 * 1000)); + this.feedback({ analysingMessage: null }); return res; From 7a606cf8efbd08539cccd65a8acd74eefc10c448 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 26 Feb 2023 10:01:53 +0100 Subject: [PATCH 38/40] oracle port config #496 --- .../src/backend/drivers.js | 110 +++++------------- .../src/frontend/drivers.js | 13 +-- 2 files changed, 34 insertions(+), 89 deletions(-) diff --git a/plugins/dbgate-plugin-oracle/src/backend/drivers.js b/plugins/dbgate-plugin-oracle/src/backend/drivers.js index 90fc9cb46..c5ac36533 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/drivers.js +++ b/plugins/dbgate-plugin-oracle/src/backend/drivers.js @@ -52,52 +52,10 @@ const drivers = driverBases.map(driverBase => ({ authType, socketPath, }) { - let options = null; - - if (engine == 'redshift@dbgate-plugin-oracle') { - let url = databaseUrl; - if (url && url.startsWith('jdbc:redshift://')) { - url = url.substring('jdbc:redshift://'.length); - } - if (user && password) { - url = `oracle://${user}:${password}@${url}`; - } else if (user) { - url = `oracle://${user}@${url}`; - } else { - url = `oracle://${url}`; - } - - options = { - connectionString: url, - }; - } else { - options = useDatabaseUrl - ? { - connectionString: databaseUrl, - } - : { - host: authType == 'socket' ? socketPath || driverBase.defaultSocketPath : server, - port: authType == 'socket' ? null : port, - user, - password, - database: database || 'oracle', - ssl, - }; - } - - // console.log('OPTIONS', options); -/* - const client = new pg.Client(options); - await client.connect(); - - if (isReadOnly) { - await this.query(client, 'SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY'); - } -*/ - client = await oracledb.getConnection( { - user : options.user, - password : options.password, - connectString : options.host + client = await oracledb.getConnection({ + user, + password, + connectString: useDatabaseUrl ? databaseUrl : port ? `${server}:${port}` : server, }); return client; }, @@ -105,28 +63,25 @@ const drivers = driverBases.map(driverBase => ({ return pool.end(); }, async query(client, sql) { - //console.log('query sql', sql); + //console.log('query sql', sql); if (sql == null) { return { rows: [], columns: [], }; } -try { + try { //console.log('sql3', sql); - const res = await client.execute(sql); - //console.log('res', res); - const columns = extractOracleColumns(res.metaData); - //console.log('columns', columns); - return { rows: (res.rows || []).map(row => zipDataRow(row, columns)), columns }; -} -catch(err) { - console.log('Error query', err, sql); -} -finally { - //console.log('finally', sql); -} - + const res = await client.execute(sql); + //console.log('res', res); + const columns = extractOracleColumns(res.metaData); + //console.log('columns', columns); + return { rows: (res.rows || []).map(row => zipDataRow(row, columns)), columns }; + } catch (err) { + console.log('Error query', err, sql); + } finally { + //console.log('finally', sql); + } }, stream(client, sql, options) { /* @@ -137,8 +92,8 @@ finally { */ // console.log('queryStream', sql); const query = client.queryStream(sql); - // const consumeStream = new Promise((resolve, reject) => { - let rowcount = 0; + // const consumeStream = new Promise((resolve, reject) => { + let rowcount = 0; let wasHeader = false; query.on('metadata', row => { @@ -200,13 +155,12 @@ finally { }); options.done(); }); - query.on('close', function() { - //console.log("stream 'close' event"); - // The underlying ResultSet has been closed, so the connection can now - // be closed, if desired. Note: do not close connections on 'end'. - //resolve(rowcount); - ; - }); + query.on('close', function () { + //console.log("stream 'close' event"); + // The underlying ResultSet has been closed, so the connection can now + // be closed, if desired. Note: do not close connections on 'end'. + //resolve(rowcount); + }); //}); //const numrows = await consumeStream; @@ -215,7 +169,7 @@ finally { }, async getVersion(client) { //const { rows } = await this.query(client, "SELECT banner as version FROM v$version WHERE banner LIKE 'Oracle%'"); - const { rows } = await this.query(client, "SELECT version as \"version\" FROM v$instance"); + const { rows } = await this.query(client, 'SELECT version as "version" FROM v$instance'); const { version } = rows[0]; const isCockroach = false; //version.toLowerCase().includes('cockroachdb'); @@ -245,7 +199,7 @@ finally { }; }, async readQuery(client, sql, structure) { -/* + /* const query = new pg.Query({ text: sql, rowMode: 'array', @@ -267,10 +221,10 @@ finally { if (!wasHeader) { columns = extractOracleColumns(row); if (columns && columns.length > 0) { - pass.write({ - __isStreamHeader: true, - ...(structure || { columns }), - }); + pass.write({ + __isStreamHeader: true, + ...(structure || { columns }), + }); } wasHeader = true; } @@ -301,7 +255,7 @@ finally { return createBulkInsertStreamBase(this, stream, pool, name, options); }, async listDatabases(client) { - const { rows } = await this.query(client, 'SELECT instance_name AS \"name\" FROM v$instance'); + const { rows } = await this.query(client, 'SELECT instance_name AS "name" FROM v$instance'); return rows; }, @@ -319,7 +273,7 @@ finally { }, })); -drivers.initialize = (dbgateEnv) => { +drivers.initialize = dbgateEnv => { if (dbgateEnv.nativeModules && dbgateEnv.nativeModules.oracledb) { oracledb = dbgateEnv.nativeModules.oracledb(); } diff --git a/plugins/dbgate-plugin-oracle/src/frontend/drivers.js b/plugins/dbgate-plugin-oracle/src/frontend/drivers.js index fd60b2aa2..d9f7c9e44 100644 --- a/plugins/dbgate-plugin-oracle/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-oracle/src/frontend/drivers.js @@ -118,11 +118,7 @@ const oracleDriverBase = { return ['databaseUrl', 'isReadOnly'].includes(field); } - return ( - ['authType', 'user', 'password', 'defaultDatabase', 'singleDatabase', 'isReadOnly'].includes(field) || - (values.authType == 'socket' && ['socketPath'].includes(field)) || - (values.authType != 'socket' && ['server', 'port'].includes(field)) - ); + return ['user', 'password', 'defaultDatabase', 'singleDatabase', 'isReadOnly', 'server', 'port'].includes(field); }, beforeConnectionSave: connection => { @@ -166,17 +162,13 @@ $$ LANGUAGE plpgsql;`, }, ]; }, - - authTypeLabel: 'Connection mode', - defaultAuthTypeName: 'hostPort', - defaultSocketPath: '/var/run/oracledb', }; /** @type {import('dbgate-types').EngineDriver} */ const oracleDriver = { ...oracleDriverBase, engine: 'oracle@dbgate-plugin-oracle', - title: 'OracleDB', + title: 'OracleDB (BETA)', defaultPort: 1521, dialect: { ...dialect, @@ -198,5 +190,4 @@ const oracleDriver = { }, }; - module.exports = [oracleDriver]; From 1417f53c568c017721e684d35a7b9ab9b67f62ee Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 26 Feb 2023 10:03:25 +0100 Subject: [PATCH 39/40] disable SSL tab for oracle --- plugins/dbgate-plugin-oracle/src/frontend/drivers.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/dbgate-plugin-oracle/src/frontend/drivers.js b/plugins/dbgate-plugin-oracle/src/frontend/drivers.js index d9f7c9e44..0b130fe98 100644 --- a/plugins/dbgate-plugin-oracle/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-oracle/src/frontend/drivers.js @@ -188,6 +188,8 @@ const oracleDriver = { } return dialect; }, + + showConnectionTab: (field) => field == 'sshTunnel', }; module.exports = [oracleDriver]; From d1ae7fe6e924557a6c46a97ff3bac9ef4fcff44c Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 26 Feb 2023 10:08:41 +0100 Subject: [PATCH 40/40] oracle support marked as experimental --- plugins/dbgate-plugin-oracle/src/frontend/drivers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/dbgate-plugin-oracle/src/frontend/drivers.js b/plugins/dbgate-plugin-oracle/src/frontend/drivers.js index 0b130fe98..4f253b57b 100644 --- a/plugins/dbgate-plugin-oracle/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-oracle/src/frontend/drivers.js @@ -168,7 +168,7 @@ $$ LANGUAGE plpgsql;`, const oracleDriver = { ...oracleDriverBase, engine: 'oracle@dbgate-plugin-oracle', - title: 'OracleDB (BETA)', + title: 'OracleDB (Experimental)', defaultPort: 1521, dialect: { ...dialect,