diff --git a/package.json b/package.json index f3f391254..237f28c1a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "private": true, - "version": "4.7.3-beta.1", + "version": "4.7.3-beta.4", "name": "dbgate-all", "workspaces": [ "packages/*", @@ -11,7 +11,7 @@ "start:api": "yarn workspace dbgate-api start", "start:app": "cd app && yarn start", "start:api:portal": "yarn workspace dbgate-api start:portal", - "start:api:covid": "yarn workspace dbgate-api start:covid", + "start:api:singledb": "yarn workspace dbgate-api start:singledb", "start:web": "yarn workspace dbgate-web dev", "start:sqltree": "yarn workspace dbgate-sqltree start", "start:tools": "yarn workspace dbgate-tools start", diff --git a/packages/api/.env-portal b/packages/api/env/portal/.env similarity index 100% rename from packages/api/.env-portal rename to packages/api/env/portal/.env diff --git a/packages/api/.env-singledb b/packages/api/env/singledb/.env similarity index 100% rename from packages/api/.env-singledb rename to packages/api/env/singledb/.env diff --git a/packages/api/package.json b/packages/api/package.json index 721a664d3..6c8238eed 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -53,8 +53,8 @@ }, "scripts": { "start": "env-cmd node src/index.js", - "start:portal": "env-cmd -f .env-portal node src/index.js", - "start:singledb": "env-cmd -f .env-singledb node src/index.js", + "start:portal": "env-cmd -f env/portal/.env node src/index.js", + "start:singledb": "env-cmd -f env/singledb/.env node src/index.js", "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db", "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test", "ts": "tsc", diff --git a/packages/api/src/controllers/apps.js b/packages/api/src/controllers/apps.js index 94b9f050b..ba2afe587 100644 --- a/packages/api/src/controllers/apps.js +++ b/packages/api/src/controllers/apps.js @@ -265,4 +265,16 @@ module.exports = { return true; }, + + createConfigFile_meta: true, + async createConfigFile({ appFolder, fileName, content }) { + const file = path.join(appdir(), appFolder, fileName); + if (!(await fs.exists(file))) { + await fs.writeFile(file, JSON.stringify(content, undefined, 2)); + socket.emitChanged(`app-files-changed-${appFolder}`); + socket.emitChanged('used-apps-changed'); + return true; + } + return false; + }, }; diff --git a/packages/api/src/controllers/files.js b/packages/api/src/controllers/files.js index 6d82c2e29..350f72262 100644 --- a/packages/api/src/controllers/files.js +++ b/packages/api/src/controllers/files.js @@ -98,6 +98,7 @@ module.exports = { const app = folder.substring('app:'.length); await fs.writeFile(path.join(appdir(), app, file), serialize(format, data)); socket.emitChanged(`app-files-changed-${app}`); + socket.emitChanged('used-apps-changed'); apps.emitChangedDbApp(folder); return true; } else { diff --git a/packages/api/src/controllers/plugins.js b/packages/api/src/controllers/plugins.js index e9d62b20d..4a537decf 100644 --- a/packages/api/src/controllers/plugins.js +++ b/packages/api/src/controllers/plugins.js @@ -89,6 +89,12 @@ module.exports = { encoding: 'utf-8', }) .then(x => JSON.parse(x)); + if (!manifest.keywords) { + continue; + } + if (!manifest.keywords.includes('dbgateplugin')) { + continue; + } const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md'); // @ts-ignore if (await fs.exists(readmeFile)) { diff --git a/packages/api/src/main.js b/packages/api/src/main.js index b3268cc7b..275d6c672 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -95,7 +95,12 @@ function start() { if (platformInfo.isNpmDist) { app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public'))); - getPort({ port: 5000 }).then(port => { + getPort({ + port: parseInt( + // @ts-ignore + process.env.PORT || 3000 + ), + }).then(port => { server.listen(port, () => { console.log(`DbGate API listening on port ${port}`); }); diff --git a/packages/api/src/shell/copyStream.js b/packages/api/src/shell/copyStream.js index 60e49deb7..637eb0bb9 100644 --- a/packages/api/src/shell/copyStream.js +++ b/packages/api/src/shell/copyStream.js @@ -1,18 +1,47 @@ const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream'); +const Stream = require('stream'); +const ColumnMapTransformStream = require('../utility/ColumnMapTransformStream'); + +function copyStream(input, output, options) { + const { columns } = options || {}; + + const transforms = []; + if (columns) { + transforms.push(new ColumnMapTransformStream(columns)); + } + if (output.requireFixedStructure) { + transforms.push(new EnsureStreamHeaderStream()); + } + + // return new Promise((resolve, reject) => { + // Stream.pipeline(input, ...transforms, output, err => { + // if (err) { + // reject(err); + // } else { + // resolve(); + // } + // }); + // }); -function copyStream(input, output) { return new Promise((resolve, reject) => { const finisher = output['finisher'] || output; finisher.on('finish', resolve); finisher.on('error', reject); - if (output.requireFixedStructure) { - const ensureHeader = new EnsureStreamHeaderStream(); - input.pipe(ensureHeader); - ensureHeader.pipe(output); - } else { - input.pipe(output); + let lastStream = input; + for (const tran of transforms) { + lastStream.pipe(tran); + lastStream = tran; } + lastStream.pipe(output); + + // if (output.requireFixedStructure) { + // const ensureHeader = new EnsureStreamHeaderStream(); + // input.pipe(ensureHeader); + // ensureHeader.pipe(output); + // } else { + // input.pipe(output); + // } }); } diff --git a/packages/api/src/utility/ColumnMapTransformStream.js b/packages/api/src/utility/ColumnMapTransformStream.js new file mode 100644 index 000000000..1bbd913df --- /dev/null +++ b/packages/api/src/utility/ColumnMapTransformStream.js @@ -0,0 +1,21 @@ +const stream = require('stream'); +const { transformRowUsingColumnMap } = require('dbgate-tools'); + +class ColumnMapTransformStream extends stream.Transform { + constructor(columns) { + super({ objectMode: true }); + this.columns = columns; + } + _transform(chunk, encoding, done) { + if (chunk.__isStreamHeader) { + // skip stream header + done(); + return; + } + + this.push(transformRowUsingColumnMap(chunk, this.columns)); + done(); + } +} + +module.exports = ColumnMapTransformStream; diff --git a/packages/api/src/utility/freeTableStorage.js b/packages/api/src/utility/freeTableStorage.js index caf5d2d8e..ceba774ef 100644 --- a/packages/api/src/utility/freeTableStorage.js +++ b/packages/api/src/utility/freeTableStorage.js @@ -3,7 +3,7 @@ const fs = require('fs-extra'); async function saveFreeTableData(file, data) { const { structure, rows } = data; const fileStream = fs.createWriteStream(file); - await fileStream.write(JSON.stringify(structure) + '\n'); + await fileStream.write(JSON.stringify({ __isStreamHeader: true, ...structure }) + '\n'); for (const row of rows) { await fileStream.write(JSON.stringify(row) + '\n'); } diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 7e58fad1b..6fce029af 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -166,11 +166,12 @@ export abstract class GridDisplay { hideAllColumns() { this.setConfig(cfg => ({ ...cfg, - hiddenColumns: this.columns.map(x => x.uniqueName), + hiddenColumns: this.columns.filter(x => x.uniquePath.length == 1).map(x => x.uniqueName), })); } get hiddenColumnIndexes() { + // console.log('GridDisplay.hiddenColumn', this.config.hiddenColumns); return (this.config.hiddenColumns || []).map(x => _.findIndex(this.allColumns, y => y.uniqueName == x)); } @@ -569,6 +570,20 @@ export abstract class GridDisplay { return sql; } + getExportColumnMap() { + const changesDefined = this.config.hiddenColumns?.length > 0 || this.config.addedColumns?.length > 0; + if (this.isDynamicStructure && !changesDefined) { + return null; + } + return this.getColumns(null) + .filter(col => col.isChecked) + .map(col => ({ + dst: col.headerText, + src: col.uniqueName, + ignore: !changesDefined, + })); + } + resizeColumn(uniqueName: string, computedSize: number, diff: number) { this.setConfig(cfg => { const columnWidths = { diff --git a/packages/dbgate/README.md b/packages/dbgate/README.md index c48ade878..dab3776b0 100644 --- a/packages/dbgate/README.md +++ b/packages/dbgate/README.md @@ -11,7 +11,7 @@ npm install -g dbgate After installing, you can run dbgate with command: ```sh -dbgate +dbgate-serve ``` Then open http://localhost:5000 in your browser diff --git a/packages/dbgate/bin/dbgate.js b/packages/dbgate/bin/dbgate-serve.js similarity index 91% rename from packages/dbgate/bin/dbgate.js rename to packages/dbgate/bin/dbgate-serve.js index 4130a04cc..4231b81be 100755 --- a/packages/dbgate/bin/dbgate.js +++ b/packages/dbgate/bin/dbgate-serve.js @@ -1,6 +1,7 @@ #!/usr/bin/env node const path = require('path'); +require('dotenv').config(); global.dbgateApiModulePath = path.dirname(path.dirname(require.resolve('dbgate-api'))); global.dbgateApiPackagedPluginsPath = path.dirname(global.dbgateApiModulePath); diff --git a/packages/dbgate/package.json b/packages/dbgate/package.json index 3809631b4..b1afa4d6a 100644 --- a/packages/dbgate/package.json +++ b/packages/dbgate/package.json @@ -10,7 +10,7 @@ "author": "Jan Prochazka", "license": "MIT", "bin": { - "dbgate": "./bin/dbgate.js" + "dbgate-serve": "./bin/dbgate-serve.js" }, "keywords": [ "sql", @@ -19,12 +19,14 @@ ], "dependencies": { "dbgate-api": "^4.1.1", - "dbgate-web": "^4.1.1", "dbgate-plugin-csv": "^4.1.1", "dbgate-plugin-excel": "^4.1.1", "dbgate-plugin-mongo": "^4.1.1", - "dbgate-plugin-mysql": "^4.1.1", "dbgate-plugin-mssql": "^4.1.1", - "dbgate-plugin-postgres": "^4.1.1" + "dbgate-plugin-mysql": "^4.1.1", + "dbgate-plugin-postgres": "^4.1.1", + "dbgate-plugin-xml": "^4.1.1", + "dbgate-web": "^4.1.1", + "dotenv": "^16.0.0" } -} \ No newline at end of file +} diff --git a/packages/tools/src/tableTransforms.ts b/packages/tools/src/tableTransforms.ts index 403426cff..54f46ed03 100644 --- a/packages/tools/src/tableTransforms.ts +++ b/packages/tools/src/tableTransforms.ts @@ -1,5 +1,7 @@ import { TableInfo } from 'dbgate-types'; import _cloneDeep from 'lodash/cloneDeep'; +import _fromPairs from 'lodash/fromPairs'; +import _get from 'lodash/get'; export function prepareTableForImport(table: TableInfo): TableInfo { const res = _cloneDeep(table); @@ -10,3 +12,12 @@ export function prepareTableForImport(table: TableInfo): TableInfo { if (res.primaryKey) res.primaryKey.constraintName = null; return res; } + +interface TransformColumnDefinition { + src: string; + dst: string; +} + +export function transformRowUsingColumnMap(row, columns: TransformColumnDefinition[]) { + return _fromPairs(columns.map(col => [col.dst, _get(row, col.src)])); +} diff --git a/packages/web/src/datagrid/CollectionDataGridCore.svelte b/packages/web/src/datagrid/CollectionDataGridCore.svelte index a3f798ba2..9872660c9 100644 --- a/packages/web/src/datagrid/CollectionDataGridCore.svelte +++ b/packages/web/src/datagrid/CollectionDataGridCore.svelte @@ -174,6 +174,7 @@ initialValues.sourceDatabaseName = database; initialValues.sourceSql = getExportQuery(); initialValues.sourceList = [pureName]; + initialValues[`columns_${pureName}`] = display.getExportColumnMap(); showModal(ImportExportModal, { initialValues }); } @@ -208,7 +209,8 @@ sql: getExportQuery(), }, }, - fmt + fmt, + display.getExportColumnMap() ); }; diff --git a/packages/web/src/datagrid/ColumnManager.svelte b/packages/web/src/datagrid/ColumnManager.svelte index 397b4a89b..024374ae3 100644 --- a/packages/web/src/datagrid/ColumnManager.svelte +++ b/packages/web/src/datagrid/ColumnManager.svelte @@ -175,10 +175,12 @@ if (domFocusField) domFocusField.focus(); }} on:setvisibility={e => { - for (const name of selectedColumns) { - const column = items.find(x => x.uniqueName == name); - if (column) { - display.setColumnVisibility(column.uniquePath, e.detail); + if (selectedColumns.includes(column.uniqueName)) { + for (const name of selectedColumns) { + const column = items.find(x => x.uniqueName == name); + if (column) { + display.setColumnVisibility(column.uniquePath, e.detail); + } } } }} diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index bd3cf4b00..ce4bfd485 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -800,6 +800,7 @@ // $: console.log('containerHeight', containerHeight); // $: console.log('COLUMNS', columns); + // $: console.log('columnSizes.realCount', columnSizes.realCount); // $: console.log('realColumnUniqueNames', realColumnUniqueNames); // $: console.log('columnSizes.realCount', columnSizes.realCount); diff --git a/packages/web/src/datagrid/JslDataGridCore.svelte b/packages/web/src/datagrid/JslDataGridCore.svelte index 90116b39a..a0112eb7d 100644 --- a/packages/web/src/datagrid/JslDataGridCore.svelte +++ b/packages/web/src/datagrid/JslDataGridCore.svelte @@ -55,6 +55,7 @@ import RowsArrayGrider from './RowsArrayGrider'; export let jslid; + export let display; export const activator = createActivator('JslDataGridCore', false); @@ -93,10 +94,12 @@ initialValues.sourceStorageType = 'archive'; initialValues.sourceArchiveFolder = archiveMatch[1]; initialValues.sourceList = [archiveMatch[2]]; + initialValues[`columns_${archiveMatch[2]}`] = display.getExportColumnMap(); } else { initialValues.sourceStorageType = 'jsldata'; initialValues.sourceJslId = jslid; initialValues.sourceList = ['query-data']; + initialValues[`columns_query-data`] = display.getExportColumnMap(); } showModal(ImportExportModal, { initialValues }); } @@ -113,7 +116,8 @@ fileName: archiveMatch[2], }, }, - fmt + fmt, + display.getExportColumnMap() ); } else { exportQuickExportFile( @@ -124,7 +128,8 @@ jslid, }, }, - fmt + fmt, + display.getExportColumnMap() ); } }; diff --git a/packages/web/src/datagrid/SeriesSizes.ts b/packages/web/src/datagrid/SeriesSizes.ts index ec62a7404..bb2466e6c 100644 --- a/packages/web/src/datagrid/SeriesSizes.ts +++ b/packages/web/src/datagrid/SeriesSizes.ts @@ -99,9 +99,12 @@ export class SeriesSizes { } this.modelIndexes = _.range(0, this.count); + // console.log('SeriesSize:build:this.modelIndexes-before', this.modelIndexes); + // console.log('SeriesSize:build:this.hiddenAndFrozenModelIndexes', this.hiddenAndFrozenModelIndexes); if (this.hiddenAndFrozenModelIndexes) { this.modelIndexes = this.modelIndexes.filter(col => !this.hiddenAndFrozenModelIndexes.includes(col)); } + // console.log('SeriesSize:build:this.modelIndexes-result', this.modelIndexes); } public getScrollIndexOnPosition(position: number): number { diff --git a/packages/web/src/datagrid/SqlDataGridCore.svelte b/packages/web/src/datagrid/SqlDataGridCore.svelte index 0013c3451..d23d206d7 100644 --- a/packages/web/src/datagrid/SqlDataGridCore.svelte +++ b/packages/web/src/datagrid/SqlDataGridCore.svelte @@ -135,6 +135,7 @@ initialValues.sourceDatabaseName = database; initialValues.sourceSql = display.getExportQuery(); initialValues.sourceList = display.baseTableOrSimilar ? [display.baseTableOrSimilar.pureName] : []; + initialValues[`columns_${pureName}`] = display.getExportColumnMap(); showModal(ImportExportModal, { initialValues }); } @@ -193,7 +194,8 @@ sql: display.getExportQuery(), }, }, - fmt + fmt, + display.getExportColumnMap() ); }; registerQuickExportHandler(quickExportHandler); diff --git a/packages/web/src/datagrid/gridutil.ts b/packages/web/src/datagrid/gridutil.ts index 3b9e26ef3..499368d65 100644 --- a/packages/web/src/datagrid/gridutil.ts +++ b/packages/web/src/datagrid/gridutil.ts @@ -17,6 +17,7 @@ export function countColumnSizes(grider: Grider, columns, containerWidth, displa //return this.context.measureText(txt).width; // console.log('countColumnSizes', loadedRows.length, containerWidth); + // console.log('countColumnSizes:columns', columns); columnSizes.maxSize = (containerWidth * 2) / 3; columnSizes.count = columns.length; @@ -114,10 +115,12 @@ export function countVisibleRealColumns(columnSizes, firstVisibleColumnScrollInd ) { visibleRealColumnIndexes.push(colIndex + columnSizes.frozenCount); } + // console.log('countVisibleRealColumns:visibleRealColumnIndexes', visibleRealColumnIndexes); // real columns for (let colIndex of visibleRealColumnIndexes) { let modelColumnIndex = columnSizes.realToModel(colIndex); + // console.log('countVisibleRealColumns:modelColumnIndex', modelColumnIndex); modelIndexes[colIndex] = modelColumnIndex; let col = columns[modelColumnIndex]; @@ -129,6 +132,7 @@ export function countVisibleRealColumns(columnSizes, firstVisibleColumnScrollInd width, }); } + // console.log('countVisibleRealColumns:realColumns', realColumns); return realColumns; } diff --git a/packages/web/src/elements/TableControl.svelte b/packages/web/src/elements/TableControl.svelte index e68377345..d22a01524 100644 --- a/packages/web/src/elements/TableControl.svelte +++ b/packages/web/src/elements/TableControl.svelte @@ -24,6 +24,7 @@ export let selectedIndex = 0; export let clickable = false; export let disableFocusOutline = false; + export let emptyMessage = null; export let domTable = undefined; @@ -99,6 +100,11 @@ {/each} {/each} + {#if emptyMessage && rows.length == 0} +