From 0629123cc11a859e8ef527463fa3bbee00dd297c Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 12 Mar 2022 20:45:12 +0100 Subject: [PATCH 01/15] apps improvements --- packages/api/src/controllers/apps.js | 12 ++++++++ packages/api/src/controllers/files.js | 1 + packages/web/src/tabs/JsonEditorTab.svelte | 30 +++++++++++++------- packages/web/src/widgets/AppFilesList.svelte | 16 +++++++++++ 4 files changed, 48 insertions(+), 11 deletions(-) diff --git a/packages/api/src/controllers/apps.js b/packages/api/src/controllers/apps.js index 94b9f050b..ba2afe587 100644 --- a/packages/api/src/controllers/apps.js +++ b/packages/api/src/controllers/apps.js @@ -265,4 +265,16 @@ module.exports = { return true; }, + + createConfigFile_meta: true, + async createConfigFile({ appFolder, fileName, content }) { + const file = path.join(appdir(), appFolder, fileName); + if (!(await fs.exists(file))) { + await fs.writeFile(file, JSON.stringify(content, undefined, 2)); + socket.emitChanged(`app-files-changed-${appFolder}`); + socket.emitChanged('used-apps-changed'); + return true; + } + return false; + }, }; diff --git a/packages/api/src/controllers/files.js b/packages/api/src/controllers/files.js index 6d82c2e29..350f72262 100644 --- a/packages/api/src/controllers/files.js +++ b/packages/api/src/controllers/files.js @@ -98,6 +98,7 @@ module.exports = { const app = folder.substring('app:'.length); await fs.writeFile(path.join(appdir(), app, file), serialize(format, data)); socket.emitChanged(`app-files-changed-${app}`); + socket.emitChanged('used-apps-changed'); apps.emitChangedDbApp(folder); return true; } else { diff --git a/packages/web/src/tabs/JsonEditorTab.svelte b/packages/web/src/tabs/JsonEditorTab.svelte index e9a1e51e4..030a0c7cb 100644 --- a/packages/web/src/tabs/JsonEditorTab.svelte +++ b/packages/web/src/tabs/JsonEditorTab.svelte @@ -22,6 +22,8 @@ import useEditorData from '../query/useEditorData'; import invalidateCommands from '../commands/invalidateCommands'; import createActivator, { getActiveComponent } from '../utility/createActivator'; + import ToolStripContainer from '../buttons/ToolStripContainer.svelte'; + import ToolStripSaveButton from '../buttons/ToolStripSaveButton.svelte'; export let tabid; @@ -70,14 +72,20 @@ } - setEditorData(e.detail)} - on:focus={() => { - activator.activate(); - invalidateCommands(); - }} - bind:this={domEditor} - mode="json" -/> + + setEditorData(e.detail)} + on:focus={() => { + activator.activate(); + invalidateCommands(); + }} + bind:this={domEditor} + mode="json" + /> + + + + + diff --git a/packages/web/src/widgets/AppFilesList.svelte b/packages/web/src/widgets/AppFilesList.svelte index c4f34cbe8..3c50951ab 100644 --- a/packages/web/src/widgets/AppFilesList.svelte +++ b/packages/web/src/widgets/AppFilesList.svelte @@ -37,6 +37,7 @@ import { useAppFiles, useArchiveFolders } from '../utility/metadataLoaders'; import openNewTab from '../utility/openNewTab'; import WidgetsInnerContainer from './WidgetsInnerContainer.svelte'; + import { showSnackbarError } from '../utility/snackbar'; let filter = ''; @@ -66,12 +67,27 @@ }); } + async function handleNewConfigFile(fileName, content) { + if (!(await apiCall('apps/create-config-file', { fileName, content, appFolder: $currentApplication }))) { + showSnackbarError('File not created, probably already exists'); + } + } + function createAddMenu() { return [ { text: 'New SQL command', onClick: () => handleNewSqlFile('command.sql', 'Create new SQL command', COMMAND_TEMPLATE), }, + { + text: 'New virtual references file', + onClick: () => handleNewConfigFile('virtual-references.config.json', []), + }, + { + text: 'New disctionary descriptions file', + onClick: () => handleNewConfigFile('dictionary-descriptions.config.json', []), + }, + // { text: 'New query view', onClick: () => handleNewSqlFile('query.sql', 'Create new SQL query', QUERY_TEMPLATE) }, ]; } From 7efd4be401e4be71bacb69e49b5eae9425f2eaa0 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 12 Mar 2022 21:12:20 +0100 Subject: [PATCH 02/15] faxed db group close buttpm style --- packages/web/src/widgets/TabsPanel.svelte | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/web/src/widgets/TabsPanel.svelte b/packages/web/src/widgets/TabsPanel.svelte index 17220e4bf..bf3345e15 100644 --- a/packages/web/src/widgets/TabsPanel.svelte +++ b/packages/web/src/widgets/TabsPanel.svelte @@ -387,15 +387,17 @@ draggingDbGroupTarget = null; }} > - - {tabGroup.tabDbName} +
+ + {tabGroup.tabDbName} +
- closeMultipleTabs(tab => tabGroup.tabs.find(x => x.tabid == tab.tabid))} > - +
{#each tabGroup.tabs as tab} @@ -505,7 +507,10 @@ overflow: hidden; text-overflow: ellipsis; white-space: nowrap; + } + .db-name-inner { justify-content: center; + flex-grow: 1; } /* .db-name:hover { background-color: var(--theme-bg-3); @@ -541,7 +546,6 @@ margin-left: 5px; margin-right: 5px; color: var(--theme-font-3); - float: right; } .close-button:hover { From cd7864b889bc3454a4edd64f0ac773713e44ad70 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 10:56:01 +0100 Subject: [PATCH 03/15] column manager fix --- packages/web/src/datagrid/ColumnManager.svelte | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/web/src/datagrid/ColumnManager.svelte b/packages/web/src/datagrid/ColumnManager.svelte index 397b4a89b..024374ae3 100644 --- a/packages/web/src/datagrid/ColumnManager.svelte +++ b/packages/web/src/datagrid/ColumnManager.svelte @@ -175,10 +175,12 @@ if (domFocusField) domFocusField.focus(); }} on:setvisibility={e => { - for (const name of selectedColumns) { - const column = items.find(x => x.uniqueName == name); - if (column) { - display.setColumnVisibility(column.uniquePath, e.detail); + if (selectedColumns.includes(column.uniqueName)) { + for (const name of selectedColumns) { + const column = items.find(x => x.uniqueName == name); + if (column) { + display.setColumnVisibility(column.uniquePath, e.detail); + } } } }} From 750a37a27f9e43f8ec242a44194a99e58812779a Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 11:35:04 +0100 Subject: [PATCH 04/15] grid columns display fix --- packages/datalib/src/GridDisplay.ts | 3 ++- packages/web/src/datagrid/DataGridCore.svelte | 1 + packages/web/src/datagrid/SeriesSizes.ts | 3 +++ packages/web/src/datagrid/gridutil.ts | 4 ++++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 7e58fad1b..2c566e658 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -166,11 +166,12 @@ export abstract class GridDisplay { hideAllColumns() { this.setConfig(cfg => ({ ...cfg, - hiddenColumns: this.columns.map(x => x.uniqueName), + hiddenColumns: this.columns.filter(x => x.uniquePath.length == 1).map(x => x.uniqueName), })); } get hiddenColumnIndexes() { + // console.log('GridDisplay.hiddenColumn', this.config.hiddenColumns); return (this.config.hiddenColumns || []).map(x => _.findIndex(this.allColumns, y => y.uniqueName == x)); } diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index bd3cf4b00..ce4bfd485 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -800,6 +800,7 @@ // $: console.log('containerHeight', containerHeight); // $: console.log('COLUMNS', columns); + // $: console.log('columnSizes.realCount', columnSizes.realCount); // $: console.log('realColumnUniqueNames', realColumnUniqueNames); // $: console.log('columnSizes.realCount', columnSizes.realCount); diff --git a/packages/web/src/datagrid/SeriesSizes.ts b/packages/web/src/datagrid/SeriesSizes.ts index ec62a7404..bb2466e6c 100644 --- a/packages/web/src/datagrid/SeriesSizes.ts +++ b/packages/web/src/datagrid/SeriesSizes.ts @@ -99,9 +99,12 @@ export class SeriesSizes { } this.modelIndexes = _.range(0, this.count); + // console.log('SeriesSize:build:this.modelIndexes-before', this.modelIndexes); + // console.log('SeriesSize:build:this.hiddenAndFrozenModelIndexes', this.hiddenAndFrozenModelIndexes); if (this.hiddenAndFrozenModelIndexes) { this.modelIndexes = this.modelIndexes.filter(col => !this.hiddenAndFrozenModelIndexes.includes(col)); } + // console.log('SeriesSize:build:this.modelIndexes-result', this.modelIndexes); } public getScrollIndexOnPosition(position: number): number { diff --git a/packages/web/src/datagrid/gridutil.ts b/packages/web/src/datagrid/gridutil.ts index 3b9e26ef3..499368d65 100644 --- a/packages/web/src/datagrid/gridutil.ts +++ b/packages/web/src/datagrid/gridutil.ts @@ -17,6 +17,7 @@ export function countColumnSizes(grider: Grider, columns, containerWidth, displa //return this.context.measureText(txt).width; // console.log('countColumnSizes', loadedRows.length, containerWidth); + // console.log('countColumnSizes:columns', columns); columnSizes.maxSize = (containerWidth * 2) / 3; columnSizes.count = columns.length; @@ -114,10 +115,12 @@ export function countVisibleRealColumns(columnSizes, firstVisibleColumnScrollInd ) { visibleRealColumnIndexes.push(colIndex + columnSizes.frozenCount); } + // console.log('countVisibleRealColumns:visibleRealColumnIndexes', visibleRealColumnIndexes); // real columns for (let colIndex of visibleRealColumnIndexes) { let modelColumnIndex = columnSizes.realToModel(colIndex); + // console.log('countVisibleRealColumns:modelColumnIndex', modelColumnIndex); modelIndexes[colIndex] = modelColumnIndex; let col = columns[modelColumnIndex]; @@ -129,6 +132,7 @@ export function countVisibleRealColumns(columnSizes, firstVisibleColumnScrollInd width, }); } + // console.log('countVisibleRealColumns:realColumns', realColumns); return realColumns; } From 34dae68a62c7a412778bd489c7f1657316eecc90 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 14:02:09 +0100 Subject: [PATCH 05/15] export/import column map support --- packages/api/src/shell/copyStream.js | 43 +++++++-- .../src/utility/ColumnMapTransformStream.js | 21 +++++ packages/tools/src/tableTransforms.ts | 11 +++ packages/web/src/elements/TableControl.svelte | 6 ++ packages/web/src/forms/TextField.svelte | 4 +- .../impexp/ImportExportConfigurator.svelte | 20 +++++ packages/web/src/impexp/ScriptWriter.ts | 12 ++- packages/web/src/impexp/createImpExpScript.ts | 9 +- packages/web/src/modals/ColumnMapModal.svelte | 88 +++++++++++++++++++ 9 files changed, 202 insertions(+), 12 deletions(-) create mode 100644 packages/api/src/utility/ColumnMapTransformStream.js create mode 100644 packages/web/src/modals/ColumnMapModal.svelte diff --git a/packages/api/src/shell/copyStream.js b/packages/api/src/shell/copyStream.js index 60e49deb7..637eb0bb9 100644 --- a/packages/api/src/shell/copyStream.js +++ b/packages/api/src/shell/copyStream.js @@ -1,18 +1,47 @@ const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream'); +const Stream = require('stream'); +const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream'); + +function copyStream(input, output, options) { + const { columns } = options || {}; + + const transforms = []; + if (columns) { + transforms.push(new ColumnMapTransformStream(columns)); + } + if (output.requireFixedStructure) { + transforms.push(new EnsureStreamHeaderStream()); + } + + // return new Promise((resolve, reject) => { + // Stream.pipeline(input, ...transforms, output, err => { + // if (err) { + // reject(err); + // } else { + // resolve(); + // } + // }); + // }); -function copyStream(input, output) { return new Promise((resolve, reject) => { const finisher = output['finisher'] || output; finisher.on('finish', resolve); finisher.on('error', reject); - if (output.requireFixedStructure) { - const ensureHeader = new EnsureStreamHeaderStream(); - input.pipe(ensureHeader); - ensureHeader.pipe(output); - } else { - input.pipe(output); + let lastStream = input; + for (const tran of transforms) { + lastStream.pipe(tran); + lastStream = tran; } + lastStream.pipe(output); + + // if (output.requireFixedStructure) { + // const ensureHeader = new EnsureStreamHeaderStream(); + // input.pipe(ensureHeader); + // ensureHeader.pipe(output); + // } else { + // input.pipe(output); + // } }); } diff --git a/packages/api/src/utility/ColumnMapTransformStream.js b/packages/api/src/utility/ColumnMapTransformStream.js new file mode 100644 index 000000000..1bbd913df --- /dev/null +++ b/packages/api/src/utility/ColumnMapTransformStream.js @@ -0,0 +1,21 @@ +const stream = require('stream'); +const { transformRowUsingColumnMap } = require('dbgate-tools'); + +class ColumnMapTransformStream extends stream.Transform { + constructor(columns) { + super({ objectMode: true }); + this.columns = columns; + } + _transform(chunk, encoding, done) { + if (chunk.__isStreamHeader) { + // skip stream header + done(); + return; + } + + this.push(transformRowUsingColumnMap(chunk, this.columns)); + done(); + } +} + +module.exports = ColumnMapTransformStream; diff --git a/packages/tools/src/tableTransforms.ts b/packages/tools/src/tableTransforms.ts index 403426cff..54f46ed03 100644 --- a/packages/tools/src/tableTransforms.ts +++ b/packages/tools/src/tableTransforms.ts @@ -1,5 +1,7 @@ import { TableInfo } from 'dbgate-types'; import _cloneDeep from 'lodash/cloneDeep'; +import _fromPairs from 'lodash/fromPairs'; +import _get from 'lodash/get'; export function prepareTableForImport(table: TableInfo): TableInfo { const res = _cloneDeep(table); @@ -10,3 +12,12 @@ export function prepareTableForImport(table: TableInfo): TableInfo { if (res.primaryKey) res.primaryKey.constraintName = null; return res; } + +interface TransformColumnDefinition { + src: string; + dst: string; +} + +export function transformRowUsingColumnMap(row, columns: TransformColumnDefinition[]) { + return _fromPairs(columns.map(col => [col.dst, _get(row, col.src)])); +} diff --git a/packages/web/src/elements/TableControl.svelte b/packages/web/src/elements/TableControl.svelte index e68377345..d22a01524 100644 --- a/packages/web/src/elements/TableControl.svelte +++ b/packages/web/src/elements/TableControl.svelte @@ -24,6 +24,7 @@ export let selectedIndex = 0; export let clickable = false; export let disableFocusOutline = false; + export let emptyMessage = null; export let domTable = undefined; @@ -99,6 +100,11 @@ {/each} {/each} + {#if emptyMessage && rows.length == 0} + + {emptyMessage} + + {/if} diff --git a/packages/web/src/forms/TextField.svelte b/packages/web/src/forms/TextField.svelte index 59a0b0ed4..6ef2cccf0 100644 --- a/packages/web/src/forms/TextField.svelte +++ b/packages/web/src/forms/TextField.svelte @@ -2,8 +2,8 @@ import { onMount } from 'svelte'; export let value; - export let focused; - export let domEditor; + export let focused = false; + export let domEditor = undefined; if (focused) onMount(() => domEditor.focus()); diff --git a/packages/web/src/impexp/ImportExportConfigurator.svelte b/packages/web/src/impexp/ImportExportConfigurator.svelte index dfbddfc4e..4ed7fcabc 100644 --- a/packages/web/src/impexp/ImportExportConfigurator.svelte +++ b/packages/web/src/impexp/ImportExportConfigurator.svelte @@ -42,11 +42,14 @@ + + + +
{header}
+ +
+ When no columns are defined in this mapping, source row is copied to target without any modifications +
+ + + + (value = (value || []).map((x, i) => (i == index ? { ...x, skip: !e.target.checked } : x)))} + /> + + + (value = (value || []).map((x, i) => (i == index ? { ...x, src: e.target.value } : x)))} + /> + + + (value = (value || []).map((x, i) => (i == index ? { ...x, dst: e.target.value } : x)))} + /> + + + { + value = value.filter((x, i) => i != index); + }}>Remove + + + + + { + closeCurrentModal(); + onConfirm(!value || value.length == 0 ? null : value); + }} + /> + + { + value = [...(value || []), {}]; + }} + /> + { + value = []; + }} + /> + + + From 84725f05862e35f50e420d95e3a71cef267c6bc5 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 14:33:21 +0100 Subject: [PATCH 06/15] export preserves column settings --- packages/datalib/src/GridDisplay.ts | 14 ++++++++++++++ .../web/src/datagrid/CollectionDataGridCore.svelte | 1 + packages/web/src/datagrid/JslDataGridCore.svelte | 3 +++ packages/web/src/datagrid/SqlDataGridCore.svelte | 1 + packages/web/src/impexp/createImpExpScript.ts | 8 +++++++- packages/web/src/modals/ColumnMapModal.svelte | 9 ++++++--- 6 files changed, 32 insertions(+), 4 deletions(-) diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 2c566e658..6fce029af 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -570,6 +570,20 @@ export abstract class GridDisplay { return sql; } + getExportColumnMap() { + const changesDefined = this.config.hiddenColumns?.length > 0 || this.config.addedColumns?.length > 0; + if (this.isDynamicStructure && !changesDefined) { + return null; + } + return this.getColumns(null) + .filter(col => col.isChecked) + .map(col => ({ + dst: col.headerText, + src: col.uniqueName, + ignore: !changesDefined, + })); + } + resizeColumn(uniqueName: string, computedSize: number, diff: number) { this.setConfig(cfg => { const columnWidths = { diff --git a/packages/web/src/datagrid/CollectionDataGridCore.svelte b/packages/web/src/datagrid/CollectionDataGridCore.svelte index a3f798ba2..3e60ab055 100644 --- a/packages/web/src/datagrid/CollectionDataGridCore.svelte +++ b/packages/web/src/datagrid/CollectionDataGridCore.svelte @@ -174,6 +174,7 @@ initialValues.sourceDatabaseName = database; initialValues.sourceSql = getExportQuery(); initialValues.sourceList = [pureName]; + initialValues[`columns_${pureName}`] = display.getExportColumnMap(); showModal(ImportExportModal, { initialValues }); } diff --git a/packages/web/src/datagrid/JslDataGridCore.svelte b/packages/web/src/datagrid/JslDataGridCore.svelte index 90116b39a..568ab93d4 100644 --- a/packages/web/src/datagrid/JslDataGridCore.svelte +++ b/packages/web/src/datagrid/JslDataGridCore.svelte @@ -55,6 +55,7 @@ import RowsArrayGrider from './RowsArrayGrider'; export let jslid; + export let display; export const activator = createActivator('JslDataGridCore', false); @@ -93,10 +94,12 @@ initialValues.sourceStorageType = 'archive'; initialValues.sourceArchiveFolder = archiveMatch[1]; initialValues.sourceList = [archiveMatch[2]]; + initialValues[`columns_${archiveMatch[2]}`] = display.getExportColumnMap(); } else { initialValues.sourceStorageType = 'jsldata'; initialValues.sourceJslId = jslid; initialValues.sourceList = ['query-data']; + initialValues[`columns_query-data`] = display.getExportColumnMap(); } showModal(ImportExportModal, { initialValues }); } diff --git a/packages/web/src/datagrid/SqlDataGridCore.svelte b/packages/web/src/datagrid/SqlDataGridCore.svelte index 0013c3451..f78361363 100644 --- a/packages/web/src/datagrid/SqlDataGridCore.svelte +++ b/packages/web/src/datagrid/SqlDataGridCore.svelte @@ -135,6 +135,7 @@ initialValues.sourceDatabaseName = database; initialValues.sourceSql = display.getExportQuery(); initialValues.sourceList = display.baseTableOrSimilar ? [display.baseTableOrSimilar.pureName] : []; + initialValues[`columns_${pureName}`] = display.getExportColumnMap(); showModal(ImportExportModal, { initialValues }); } diff --git a/packages/web/src/impexp/createImpExpScript.ts b/packages/web/src/impexp/createImpExpScript.ts index c7238a13d..0a9bbe364 100644 --- a/packages/web/src/impexp/createImpExpScript.ts +++ b/packages/web/src/impexp/createImpExpScript.ts @@ -186,10 +186,16 @@ export default async function createImpExpScript(extensions, values, addEditorIn // @ts-ignore script.assign(targetVar, ...getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver)); - const colmap = (values[`columns_${sourceName}`] || []).filter(x => !x.skip); + let colmap = values[`columns_${sourceName}`] || []; + if (!colmap.find(x => !x.ignore)) { + // all values are ignored, ignore column map + colmap = []; + } + colmap = colmap.filter(x => !x.skip); let colmapVar = null; if (colmap.length > 0) { colmapVar = script.allocVariable(); + colmap = colmap.map(x => _.omit(x, ['ignore'])); script.assignValue(colmapVar, colmap); } diff --git a/packages/web/src/modals/ColumnMapModal.svelte b/packages/web/src/modals/ColumnMapModal.svelte index ecd895a8d..7b2ac200c 100644 --- a/packages/web/src/modals/ColumnMapModal.svelte +++ b/packages/web/src/modals/ColumnMapModal.svelte @@ -36,19 +36,22 @@ (value = (value || []).map((x, i) => (i == index ? { ...x, skip: !e.target.checked } : x)))} + on:change={e => + (value = (value || []).map((x, i) => (i == index ? { ...x, skip: !e.target.checked, ignore: false } : x)))} /> (value = (value || []).map((x, i) => (i == index ? { ...x, src: e.target.value } : x)))} + on:change={e => + (value = (value || []).map((x, i) => (i == index ? { ...x, src: e.target.value, ignore: false } : x)))} /> (value = (value || []).map((x, i) => (i == index ? { ...x, dst: e.target.value } : x)))} + on:change={e => + (value = (value || []).map((x, i) => (i == index ? { ...x, dst: e.target.value, ignore: false } : x)))} /> From cf047cb7b5da11e71318e1caf53358a1ffd94d55 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 14:44:33 +0100 Subject: [PATCH 07/15] quick export respect shown columns --- .../datagrid/CollectionDataGridCore.svelte | 3 ++- .../web/src/datagrid/JslDataGridCore.svelte | 6 +++-- .../web/src/datagrid/SqlDataGridCore.svelte | 3 ++- packages/web/src/impexp/ScriptWriter.ts | 2 +- packages/web/src/impexp/createImpExpScript.ts | 26 +++++++++++++------ packages/web/src/utility/exportFileTools.ts | 12 +++++++-- 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/packages/web/src/datagrid/CollectionDataGridCore.svelte b/packages/web/src/datagrid/CollectionDataGridCore.svelte index 3e60ab055..9872660c9 100644 --- a/packages/web/src/datagrid/CollectionDataGridCore.svelte +++ b/packages/web/src/datagrid/CollectionDataGridCore.svelte @@ -209,7 +209,8 @@ sql: getExportQuery(), }, }, - fmt + fmt, + display.getExportColumnMap() ); }; diff --git a/packages/web/src/datagrid/JslDataGridCore.svelte b/packages/web/src/datagrid/JslDataGridCore.svelte index 568ab93d4..a0112eb7d 100644 --- a/packages/web/src/datagrid/JslDataGridCore.svelte +++ b/packages/web/src/datagrid/JslDataGridCore.svelte @@ -116,7 +116,8 @@ fileName: archiveMatch[2], }, }, - fmt + fmt, + display.getExportColumnMap() ); } else { exportQuickExportFile( @@ -127,7 +128,8 @@ jslid, }, }, - fmt + fmt, + display.getExportColumnMap() ); } }; diff --git a/packages/web/src/datagrid/SqlDataGridCore.svelte b/packages/web/src/datagrid/SqlDataGridCore.svelte index f78361363..d23d206d7 100644 --- a/packages/web/src/datagrid/SqlDataGridCore.svelte +++ b/packages/web/src/datagrid/SqlDataGridCore.svelte @@ -194,7 +194,8 @@ sql: display.getExportQuery(), }, }, - fmt + fmt, + display.getExportColumnMap() ); }; registerQuickExportHandler(quickExportHandler); diff --git a/packages/web/src/impexp/ScriptWriter.ts b/packages/web/src/impexp/ScriptWriter.ts index edfc138ec..53dd3b6f5 100644 --- a/packages/web/src/impexp/ScriptWriter.ts +++ b/packages/web/src/impexp/ScriptWriter.ts @@ -33,7 +33,7 @@ export default class ScriptWriter { this.packageNames.push(packageName); } - copyStream(sourceVar, targetVar, colmapVar) { + copyStream(sourceVar, targetVar, colmapVar = null) { if (colmapVar) { this.put(`await dbgateApi.copyStream(${sourceVar}, ${targetVar}, {columns: ${colmapVar}});`); } else { diff --git a/packages/web/src/impexp/createImpExpScript.ts b/packages/web/src/impexp/createImpExpScript.ts index 0a9bbe364..197caa9d0 100644 --- a/packages/web/src/impexp/createImpExpScript.ts +++ b/packages/web/src/impexp/createImpExpScript.ts @@ -160,6 +160,21 @@ function getTargetExpr(extensions, sourceName, values, targetConnection, targetD throw new Error(`Unknown target storage type: ${targetStorageType}`); } +export function normalizeExportColumnMap(colmap) { + if (!colmap) { + return null; + } + if (!colmap.find(x => !x.ignore)) { + // all values are ignored, ignore column map + return null; + } + colmap = colmap.filter(x => !x.skip); + if (colmap.length > 0) { + return colmap.map(x => _.omit(x, ['ignore'])); + } + return null; +} + export default async function createImpExpScript(extensions, values, addEditorInfo = true) { const script = new ScriptWriter(values.startVariableIndex || 0); @@ -186,16 +201,11 @@ export default async function createImpExpScript(extensions, values, addEditorIn // @ts-ignore script.assign(targetVar, ...getTargetExpr(extensions, sourceName, values, targetConnection, targetDriver)); - let colmap = values[`columns_${sourceName}`] || []; - if (!colmap.find(x => !x.ignore)) { - // all values are ignored, ignore column map - colmap = []; - } - colmap = colmap.filter(x => !x.skip); + const colmap = normalizeExportColumnMap(values[`columns_${sourceName}`] ); + let colmapVar = null; - if (colmap.length > 0) { + if (colmap) { colmapVar = script.allocVariable(); - colmap = colmap.map(x => _.omit(x, ['ignore'])); script.assignValue(colmapVar, colmap); } diff --git a/packages/web/src/utility/exportFileTools.ts b/packages/web/src/utility/exportFileTools.ts index 91cf2c798..9d3e76a53 100644 --- a/packages/web/src/utility/exportFileTools.ts +++ b/packages/web/src/utility/exportFileTools.ts @@ -3,8 +3,9 @@ import getElectron from './getElectron'; import { showSnackbar, showSnackbarInfo, showSnackbarError, closeSnackbar } from '../utility/snackbar'; import resolveApi from './resolveApi'; import { apiCall, apiOff, apiOn } from './api'; +import { normalizeExportColumnMap } from '../impexp/createImpExpScript'; -export async function exportQuickExportFile(dataName, reader, format) { +export async function exportQuickExportFile(dataName, reader, format, columnMap = null) { const electron = getElectron(); let filePath; @@ -31,7 +32,14 @@ export async function exportQuickExportFile(dataName, reader, format) { const writer = format.createWriter(filePath, dataName); script.assign(targetVar, writer.functionName, writer.props); - script.copyStream(sourceVar, targetVar); + const colmap = normalizeExportColumnMap(columnMap); + let colmapVar = null; + if (colmap) { + colmapVar = script.allocVariable(); + script.assignValue(colmapVar, colmap); + } + + script.copyStream(sourceVar, targetVar, colmapVar); script.put(); const resp = await apiCall('runners/start', { script: script.getScript() }); From 3ccb00854c720bef806a955e65ca0c881bca37bb Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 16:47:20 +0100 Subject: [PATCH 08/15] quick export from free table editor --- packages/api/src/utility/freeTableStorage.js | 2 +- .../src/freetable/FreeTableGridCore.svelte | 26 ++++++++++++++++++- packages/web/src/tabs/FreeTableTab.svelte | 6 +++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/packages/api/src/utility/freeTableStorage.js b/packages/api/src/utility/freeTableStorage.js index caf5d2d8e..ceba774ef 100644 --- a/packages/api/src/utility/freeTableStorage.js +++ b/packages/api/src/utility/freeTableStorage.js @@ -3,7 +3,7 @@ const fs = require('fs-extra'); async function saveFreeTableData(file, data) { const { structure, rows } = data; const fileStream = fs.createWriteStream(file); - await fileStream.write(JSON.stringify(structure) + '\n'); + await fileStream.write(JSON.stringify({ __isStreamHeader: true, ...structure }) + '\n'); for (const row of rows) { await fileStream.write(JSON.stringify(row) + '\n'); } diff --git a/packages/web/src/freetable/FreeTableGridCore.svelte b/packages/web/src/freetable/FreeTableGridCore.svelte index 31b8c0b8f..2352b47da 100644 --- a/packages/web/src/freetable/FreeTableGridCore.svelte +++ b/packages/web/src/freetable/FreeTableGridCore.svelte @@ -16,6 +16,7 @@ import { createGridCache, FreeTableGridDisplay } from 'dbgate-datalib'; import { writable } from 'svelte/store'; import uuidv1 from 'uuid/v1'; + import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte'; import registerCommand from '../commands/registerCommand'; import DataGridCore from '../datagrid/DataGridCore.svelte'; @@ -24,6 +25,8 @@ import { apiCall } from '../utility/api'; import { registerMenu } from '../utility/contextMenu'; import createActivator, { getActiveComponent } from '../utility/createActivator'; + import createQuickExportMenu from '../utility/createQuickExportMenu'; + import { exportQuickExportFile } from '../utility/exportFileTools'; import FreeTableGrider from './FreeTableGrider'; import MacroPreviewGrider from './MacroPreviewGrider'; @@ -51,10 +54,31 @@ initialValues.sourceStorageType = 'jsldata'; initialValues.sourceJslId = jslid; initialValues.sourceList = ['editor-data']; + initialValues[`columns_editor-data`] = display.getExportColumnMap(); showModal(ImportExportModal, { initialValues: initialValues }); } - registerMenu({ command: 'freeTableGrid.export', tag: 'export' }); + const quickExportHandler = fmt => async () => { + const jslid = uuidv1(); + await apiCall('jsldata/save-free-table', { jslid, data: modelState.value }); + exportQuickExportFile( + 'editor-data', + { + functionName: 'jslDataReader', + props: { + jslid, + }, + }, + fmt, + display.getExportColumnMap() + ); + }; + registerQuickExportHandler(quickExportHandler); + + registerMenu(() => ({ + ...createQuickExportMenu(quickExportHandler, { command: 'freeTableGrid.export' }), + tag: 'export', + })); diff --git a/packages/web/src/tabs/FreeTableTab.svelte b/packages/web/src/tabs/FreeTableTab.svelte index d01bc3005..cf632396e 100644 --- a/packages/web/src/tabs/FreeTableTab.svelte +++ b/packages/web/src/tabs/FreeTableTab.svelte @@ -34,7 +34,7 @@ import { writable } from 'svelte/store'; import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte'; import ToolStripContainer from '../buttons/ToolStripContainer.svelte'; -import ToolStripExportButton from '../buttons/ToolStripExportButton.svelte'; + import ToolStripExportButton, { createQuickExportHandlerRef } from '../buttons/ToolStripExportButton.svelte'; import registerCommand from '../commands/registerCommand'; import DataGrid from '../datagrid/DataGrid.svelte'; import ErrorInfo from '../elements/ErrorInfo.svelte'; @@ -137,6 +137,8 @@ import ToolStripExportButton from '../buttons/ToolStripExportButton.svelte'; // display is overridden in FreeTableGridCore, this is because of column manager $: display = new FreeTableGridDisplay($modelState.value, $config, config.update, null, null); + + const quickExportHandlerRef = createQuickExportHandlerRef(); {#if isLoading} @@ -161,7 +163,7 @@ import ToolStripExportButton from '../buttons/ToolStripExportButton.svelte'; /> - + {/if} From 772a72dfd89a5aad62f9ebdd077c94571daf7641 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 16:49:33 +0100 Subject: [PATCH 09/15] dbgate.js => dbgate-serve.js --- packages/dbgate/README.md | 2 +- packages/dbgate/bin/{dbgate.js => dbgate-serve.js} | 0 packages/dbgate/package.json | 3 ++- 3 files changed, 3 insertions(+), 2 deletions(-) rename packages/dbgate/bin/{dbgate.js => dbgate-serve.js} (100%) diff --git a/packages/dbgate/README.md b/packages/dbgate/README.md index c48ade878..dab3776b0 100644 --- a/packages/dbgate/README.md +++ b/packages/dbgate/README.md @@ -11,7 +11,7 @@ npm install -g dbgate After installing, you can run dbgate with command: ```sh -dbgate +dbgate-serve ``` Then open http://localhost:5000 in your browser diff --git a/packages/dbgate/bin/dbgate.js b/packages/dbgate/bin/dbgate-serve.js similarity index 100% rename from packages/dbgate/bin/dbgate.js rename to packages/dbgate/bin/dbgate-serve.js diff --git a/packages/dbgate/package.json b/packages/dbgate/package.json index 3809631b4..dcbab9e0f 100644 --- a/packages/dbgate/package.json +++ b/packages/dbgate/package.json @@ -10,7 +10,7 @@ "author": "Jan Prochazka", "license": "MIT", "bin": { - "dbgate": "./bin/dbgate.js" + "dbgate": "./bin/dbgate-serve.js" }, "keywords": [ "sql", @@ -21,6 +21,7 @@ "dbgate-api": "^4.1.1", "dbgate-web": "^4.1.1", "dbgate-plugin-csv": "^4.1.1", + "dbgate-plugin-xml": "^4.1.1", "dbgate-plugin-excel": "^4.1.1", "dbgate-plugin-mongo": "^4.1.1", "dbgate-plugin-mysql": "^4.1.1", From ae719157c06742a763102b9c7e2c70bf42f8f911 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 16:50:45 +0100 Subject: [PATCH 10/15] v4.7.3-alpha.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f3f391254..14f5c3d42 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "private": true, - "version": "4.7.3-beta.1", + "version": "4.7.3-alpha.2", "name": "dbgate-all", "workspaces": [ "packages/*", From 25015f35d54dedead062cd8c95ee35603d01773b Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 17:05:50 +0100 Subject: [PATCH 11/15] dbgate-serve fix --- packages/api/src/controllers/plugins.js | 6 ++++++ packages/dbgate/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/api/src/controllers/plugins.js b/packages/api/src/controllers/plugins.js index e9d62b20d..4a537decf 100644 --- a/packages/api/src/controllers/plugins.js +++ b/packages/api/src/controllers/plugins.js @@ -89,6 +89,12 @@ module.exports = { encoding: 'utf-8', }) .then(x => JSON.parse(x)); + if (!manifest.keywords) { + continue; + } + if (!manifest.keywords.includes('dbgateplugin')) { + continue; + } const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md'); // @ts-ignore if (await fs.exists(readmeFile)) { diff --git a/packages/dbgate/package.json b/packages/dbgate/package.json index dcbab9e0f..1e8ae9dc2 100644 --- a/packages/dbgate/package.json +++ b/packages/dbgate/package.json @@ -10,7 +10,7 @@ "author": "Jan Prochazka", "license": "MIT", "bin": { - "dbgate": "./bin/dbgate-serve.js" + "dbgate-serve": "./bin/dbgate-serve.js" }, "keywords": [ "sql", From b6cc77c7fe3441033dfa6f74cae0faeaf410bcd5 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 17:14:04 +0100 Subject: [PATCH 12/15] dotenv config in dbgate-serve --- packages/api/src/main.js | 7 ++++++- packages/dbgate/bin/dbgate-serve.js | 1 + packages/dbgate/package.json | 11 ++++++----- yarn.lock | 10 ++++++++++ 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/api/src/main.js b/packages/api/src/main.js index b3268cc7b..275d6c672 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -95,7 +95,12 @@ function start() { if (platformInfo.isNpmDist) { app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public'))); - getPort({ port: 5000 }).then(port => { + getPort({ + port: parseInt( + // @ts-ignore + process.env.PORT || 3000 + ), + }).then(port => { server.listen(port, () => { console.log(`DbGate API listening on port ${port}`); }); diff --git a/packages/dbgate/bin/dbgate-serve.js b/packages/dbgate/bin/dbgate-serve.js index 4130a04cc..4231b81be 100755 --- a/packages/dbgate/bin/dbgate-serve.js +++ b/packages/dbgate/bin/dbgate-serve.js @@ -1,6 +1,7 @@ #!/usr/bin/env node const path = require('path'); +require('dotenv').config(); global.dbgateApiModulePath = path.dirname(path.dirname(require.resolve('dbgate-api'))); global.dbgateApiPackagedPluginsPath = path.dirname(global.dbgateApiModulePath); diff --git a/packages/dbgate/package.json b/packages/dbgate/package.json index 1e8ae9dc2..b1afa4d6a 100644 --- a/packages/dbgate/package.json +++ b/packages/dbgate/package.json @@ -19,13 +19,14 @@ ], "dependencies": { "dbgate-api": "^4.1.1", - "dbgate-web": "^4.1.1", "dbgate-plugin-csv": "^4.1.1", - "dbgate-plugin-xml": "^4.1.1", "dbgate-plugin-excel": "^4.1.1", "dbgate-plugin-mongo": "^4.1.1", - "dbgate-plugin-mysql": "^4.1.1", "dbgate-plugin-mssql": "^4.1.1", - "dbgate-plugin-postgres": "^4.1.1" + "dbgate-plugin-mysql": "^4.1.1", + "dbgate-plugin-postgres": "^4.1.1", + "dbgate-plugin-xml": "^4.1.1", + "dbgate-web": "^4.1.1", + "dotenv": "^16.0.0" } -} \ No newline at end of file +} diff --git a/yarn.lock b/yarn.lock index b46d51761..99cbee280 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3189,6 +3189,11 @@ dbgate-plugin-tools@^1.0.4, dbgate-plugin-tools@^1.0.7: pacote "^11.1.13" rimraf "^3.0.2" +dbgate-plugin-xml@^4.1.1: + version "4.7.2" + resolved "https://registry.yarnpkg.com/dbgate-plugin-xml/-/dbgate-plugin-xml-4.7.2.tgz#4a1817a7fc3b1ac5191e45dc7b1016b2f97c3158" + integrity sha512-LA52/Wg+blaRp/H+v4ypT+l6kbi4sYsTD/JdPaTbfwErMXhGZew9wtDolvbDjKZZ/zfRig0TDXj9fhLILw2cGw== + debug@2.6.9, debug@^2.2.0, debug@^2.3.3: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -3466,6 +3471,11 @@ dot-prop@^4.1.0: dependencies: is-obj "^1.0.0" +dotenv@^16.0.0: + version "16.0.0" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.0.tgz#c619001253be89ebb638d027b609c75c26e47411" + integrity sha512-qD9WU0MPM4SWLPJy/r2Be+2WgQj8plChsyrCNQzW/0WjvcJQiKQJ9mH3ZgB3fxbUUxgc/11ZJ0Fi5KiimWGz2Q== + duplexer2@~0.0.2: version "0.0.2" resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.0.2.tgz#c614dcf67e2fb14995a91711e5a617e8a60a31db" From cbe0ea7c9b7e4906e38f7d0d1470ef7b71e84014 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 17:14:32 +0100 Subject: [PATCH 13/15] v4.7.3-alpha.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 14f5c3d42..2e1b0ee0c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "private": true, - "version": "4.7.3-alpha.2", + "version": "4.7.3-alpha.3", "name": "dbgate-all", "workspaces": [ "packages/*", From 32c06fdf4d101b31dec813fbf7016bf8bb6c1330 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 17:32:26 +0100 Subject: [PATCH 14/15] test config refactor --- package.json | 2 +- packages/api/{.env-portal => env/portal/.env} | 0 packages/api/{.env-singledb => env/singledb/.env} | 0 packages/api/package.json | 4 ++-- 4 files changed, 3 insertions(+), 3 deletions(-) rename packages/api/{.env-portal => env/portal/.env} (100%) rename packages/api/{.env-singledb => env/singledb/.env} (100%) diff --git a/package.json b/package.json index 2e1b0ee0c..8d8e86e72 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "start:api": "yarn workspace dbgate-api start", "start:app": "cd app && yarn start", "start:api:portal": "yarn workspace dbgate-api start:portal", - "start:api:covid": "yarn workspace dbgate-api start:covid", + "start:api:singledb": "yarn workspace dbgate-api start:singledb", "start:web": "yarn workspace dbgate-web dev", "start:sqltree": "yarn workspace dbgate-sqltree start", "start:tools": "yarn workspace dbgate-tools start", diff --git a/packages/api/.env-portal b/packages/api/env/portal/.env similarity index 100% rename from packages/api/.env-portal rename to packages/api/env/portal/.env diff --git a/packages/api/.env-singledb b/packages/api/env/singledb/.env similarity index 100% rename from packages/api/.env-singledb rename to packages/api/env/singledb/.env diff --git a/packages/api/package.json b/packages/api/package.json index 721a664d3..6c8238eed 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -53,8 +53,8 @@ }, "scripts": { "start": "env-cmd node src/index.js", - "start:portal": "env-cmd -f .env-portal node src/index.js", - "start:singledb": "env-cmd -f .env-singledb node src/index.js", + "start:portal": "env-cmd -f env/portal/.env node src/index.js", + "start:singledb": "env-cmd -f env/singledb/.env node src/index.js", "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db", "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test", "ts": "tsc", From cf734a26eed77f5e1d5615d66f896b3e18afb3d1 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sun, 13 Mar 2022 17:33:19 +0100 Subject: [PATCH 15/15] v4.7.3-beta.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8d8e86e72..237f28c1a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "private": true, - "version": "4.7.3-alpha.3", + "version": "4.7.3-beta.4", "name": "dbgate-all", "workspaces": [ "packages/*",