diff --git a/integration-tests/__tests__/data-duplicator.spec.js b/integration-tests/__tests__/data-duplicator.spec.js new file mode 100644 index 000000000..3227cd984 --- /dev/null +++ b/integration-tests/__tests__/data-duplicator.spec.js @@ -0,0 +1,94 @@ +const engines = require('../engines'); +const stream = require('stream'); +const { testWrapper } = require('../tools'); +const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator'); +const { runCommandOnDriver } = require('dbgate-tools'); + +describe('Data duplicator', () => { + test.each(engines.map(engine => [engine.label, engine]))( + 'Insert simple data - %s', + testWrapper(async (conn, driver, engine) => { + runCommandOnDriver(conn, driver, dmp => + dmp.createTable({ + pureName: 't1', + columns: [ + { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true }, + { columnName: 'val', dataType: 'varchar(50)' }, + ], + primaryKey: { + columns: [{ columnName: 'id' }], + }, + }) + ); + runCommandOnDriver(conn, driver, dmp => + dmp.createTable({ + pureName: 't2', + columns: [ + { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true }, + { columnName: 'val', dataType: 'varchar(50)' }, + { columnName: 'valfk', dataType: 'int', notNull: true }, + ], + primaryKey: { + columns: [{ columnName: 'id' }], + }, + foreignKeys: [{ refTableName: 't1', columns: [{ columnName: 'valfk', refColumnName: 'id' }] }], + }) + ); + + const gett1 = () => + stream.Readable.from([ + { __isStreamHeader: true, __isDynamicStructure: true }, + { id: 1, val: 'v1' }, + { id: 2, val: 'v2' }, + { id: 3, val: 'v3' }, + ]); + const gett2 = () => + stream.Readable.from([ + { __isStreamHeader: true, __isDynamicStructure: true }, + { id: 1, val: 'v1', valfk: 1 }, + { id: 2, val: 'v2', valfk: 2 }, + { id: 3, val: 'v3', valfk: 3 }, + ]); + + await dataDuplicator({ + systemConnection: conn, + driver, + items: [ + { + name: 't1', + operation: 'copy', + openStream: gett1, + }, + { + name: 't2', + operation: 'copy', + openStream: gett2, + }, + ], + }); + + await dataDuplicator({ + systemConnection: conn, + driver, + items: [ + { + name: 't1', + operation: 'copy', + openStream: gett1, + }, + { + name: 't2', + operation: 'copy', + openStream: gett2, + }, + ], + }); + + const res1 = await driver.query(conn, `select count(*) as cnt from t1`); + expect(res1.rows[0].cnt.toString()).toEqual('6'); + + const res2 = await driver.query(conn, `select count(*) as cnt from t2`); + expect(res2.rows[0].cnt.toString()).toEqual('6'); + }) + ); +}); diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 8f10950d5..196919223 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -136,8 +136,8 @@ const filterLocal = [ '-MySQL', '-MariaDB', '-PostgreSQL', - 'SQL Server', - '-SQLite', + '-SQL Server', + 'SQLite', '-CockroachDB', ]; diff --git a/integration-tests/package.json b/integration-tests/package.json index b647b30ab..81ed8d4a4 100644 --- a/integration-tests/package.json +++ b/integration-tests/package.json @@ -13,6 +13,8 @@ "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js", "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest", + "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js", + "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults", "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local" diff --git a/package.json b/package.json index 59f808cee..5a946d80e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "private": true, - "version": "5.2.3-beta.6", + "version": "5.2.3-beta.8", "name": "dbgate-all", "workspaces": [ "packages/*", diff --git a/packages/api/src/controllers/archive.js b/packages/api/src/controllers/archive.js index e520871e5..eb1a546e2 100644 --- a/packages/api/src/controllers/archive.js +++ b/packages/api/src/controllers/archive.js @@ -3,13 +3,13 @@ const readline = require('readline'); const path = require('path'); const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories'); const socket = require('../utility/socket'); -const { saveFreeTableData } = require('../utility/freeTableStorage'); const loadFilesRecursive = require('../utility/loadFilesRecursive'); const getJslFileName = require('../utility/getJslFileName'); const { getLogger } = require('dbgate-tools'); const uuidv1 = require('uuid/v1'); const dbgateApi = require('../shell'); const jsldata = require('./jsldata'); +const platformInfo = require('../utility/platformInfo'); const logger = getLogger('archive'); @@ -137,8 +137,13 @@ module.exports = { }); const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath }); await dbgateApi.copyStream(reader, writer); - await fs.unlink(changedFilePath); - await fs.rename(path.join(tmpchangedFilePath), path.join(changedFilePath)); + if (platformInfo.isWindows) { + await fs.copyFile(tmpchangedFilePath, changedFilePath); + await fs.unlink(tmpchangedFilePath); + } else { + await fs.unlink(changedFilePath); + await fs.rename(tmpchangedFilePath, changedFilePath); + } return true; }, @@ -162,34 +167,6 @@ module.exports = { return true; }, - saveFreeTable_meta: true, - async saveFreeTable({ folder, file, data }) { - await saveFreeTableData(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), data); - socket.emitChanged(`archive-files-changed`, { folder }); - return true; - }, - - loadFreeTable_meta: true, - async loadFreeTable({ folder, file }) { - return new Promise((resolve, reject) => { - const fileStream = fs.createReadStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`)); - const liner = readline.createInterface({ - input: fileStream, - }); - let structure = null; - const rows = []; - liner.on('line', line => { - const data = JSON.parse(line); - if (structure) rows.push(data); - else structure = data; - }); - liner.on('close', () => { - resolve({ structure, rows }); - fileStream.close(); - }); - }); - }, - saveText_meta: true, async saveText({ folder, file, text }) { await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text); @@ -198,10 +175,30 @@ module.exports = { }, saveJslData_meta: true, - async saveJslData({ folder, file, jslid }) { + async saveJslData({ folder, file, jslid, changeSet }) { const source = getJslFileName(jslid); const target = path.join(resolveArchiveFolder(folder), `${file}.jsonl`); - await fs.copyFile(source, target); + if (changeSet) { + const reader = await dbgateApi.modifyJsonLinesReader({ + fileName: source, + changeSet, + }); + const writer = await dbgateApi.jsonLinesWriter({ fileName: target }); + await dbgateApi.copyStream(reader, writer); + } else { + await fs.copyFile(source, target); + socket.emitChanged(`archive-files-changed`, { folder }); + } + return true; + }, + + saveRows_meta: true, + async saveRows({ folder, file, rows }) { + const fileStream = fs.createWriteStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`)); + for (const row of rows) { + await fileStream.write(JSON.stringify(row) + '\n'); + } + await fileStream.close(); socket.emitChanged(`archive-files-changed`, { folder }); return true; }, diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index d591c43d7..0cf04a978 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -4,7 +4,6 @@ const lineReader = require('line-reader'); const _ = require('lodash'); const { __ } = require('lodash/fp'); const DatastoreProxy = require('../utility/DatastoreProxy'); -const { saveFreeTableData } = require('../utility/freeTableStorage'); const getJslFileName = require('../utility/getJslFileName'); const JsonLinesDatastore = require('../utility/JsonLinesDatastore'); const requirePluginFunction = require('../utility/requirePluginFunction'); @@ -148,6 +147,12 @@ module.exports = { return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters, _.isEmpty(sort) ? null : sort); }, + exists_meta: true, + async exists({ jslid }) { + const fileName = getJslFileName(jslid); + return fs.existsSync(fileName); + }, + getStats_meta: true, getStats({ jslid }) { const file = `${getJslFileName(jslid)}.stats`; @@ -189,18 +194,22 @@ module.exports = { // } }, - saveFreeTable_meta: true, - async saveFreeTable({ jslid, data }) { - saveFreeTableData(getJslFileName(jslid), data); - return true; - }, - saveText_meta: true, async saveText({ jslid, text }) { await fs.promises.writeFile(getJslFileName(jslid), text); return true; }, + saveRows_meta: true, + async saveRows({ jslid, rows }) { + const fileStream = fs.createWriteStream(getJslFileName(jslid)); + for (const row of rows) { + await fileStream.write(JSON.stringify(row) + '\n'); + } + await fileStream.close(); + return true; + }, + extractTimelineChart_meta: true, async extractTimelineChart({ jslid, timestampFunction, aggregateFunction, measures }) { const timestamp = requirePluginFunction(timestampFunction); diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index 393c64f84..d10eac39d 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -70,15 +70,20 @@ module.exports = { if (message) { const json = safeJsonParse(message.message); - if (json) logger.info(json); + if (json) logger.log(json); else logger.info(message.message); - socket.emit(`runner-info-${runid}`, { + const toEmit = { time: new Date(), - severity: 'info', ...message, message: json ? json.msg : message.message, - }); + }; + + if (json && json.level >= 50) { + toEmit.severity = 'error'; + } + + socket.emit(`runner-info-${runid}`, toEmit); } }, @@ -125,8 +130,9 @@ module.exports = { }, } ); - const pipeDispatcher = severity => data => - this.dispatchMessage(runid, { severity, message: data.toString().trim() }); + const pipeDispatcher = severity => data => { + return this.dispatchMessage(runid, { severity, message: data.toString().trim() }); + }; byline(subprocess.stdout).on('data', pipeDispatcher('info')); byline(subprocess.stderr).on('data', pipeDispatcher('error')); diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index 9684cfe1a..9a8756431 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -271,7 +271,7 @@ async function handleSqlPreview({ msgid, objects, options }) { process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated }); if (generator.isUnhandledException) { setTimeout(() => { - getLogger.info('Exiting because of unhandled exception'); + logger.error('Exiting because of unhandled exception'); process.exit(0); }, 500); } diff --git a/packages/api/src/shell/dataDuplicator.js b/packages/api/src/shell/dataDuplicator.js index 15f308c69..60b68daa4 100644 --- a/packages/api/src/shell/dataDuplicator.js +++ b/packages/api/src/shell/dataDuplicator.js @@ -9,9 +9,18 @@ const copyStream = require('./copyStream'); const jsonLinesReader = require('./jsonLinesReader'); const { resolveArchiveFolder } = require('../utility/directories'); -async function dataDuplicator({ connection, archive, items, analysedStructure = null }) { - const driver = requireEngineDriver(connection); - const pool = await connectUtility(driver, connection, 'write'); +async function dataDuplicator({ + connection, + archive, + items, + options, + analysedStructure = null, + driver, + systemConnection, +}) { + if (!driver) driver = requireEngineDriver(connection); + const pool = systemConnection || (await connectUtility(driver, connection, 'write')); + logger.info(`Connected.`); if (!analysedStructure) { @@ -26,10 +35,13 @@ async function dataDuplicator({ connection, archive, items, analysedStructure = name: item.name, operation: item.operation, matchColumns: item.matchColumns, - openStream: () => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) }), + openStream: + item.openStream || + (() => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) })), })), stream, - copyStream + copyStream, + options ); await dupl.run(); diff --git a/packages/api/src/shell/fakeObjectReader.js b/packages/api/src/shell/fakeObjectReader.js index 18e2d70a4..d1bdb593e 100644 --- a/packages/api/src/shell/fakeObjectReader.js +++ b/packages/api/src/shell/fakeObjectReader.js @@ -1,18 +1,26 @@ const stream = require('stream'); -async function fakeObjectReader({ delay = 0 } = {}) { +async function fakeObjectReader({ delay = 0, dynamicData = null } = {}) { const pass = new stream.PassThrough({ objectMode: true, }); function doWrite() { - pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true }); - pass.write({ id: 1, country: 'Czechia' }); - pass.write({ id: 2, country: 'Austria' }); - pass.write({ country: 'Germany', id: 3 }); - pass.write({ country: 'Romania', id: 4 }); - pass.write({ country: 'Great Britain', id: 5 }); - pass.write({ country: 'Bosna, Hecegovina', id: 6 }); - pass.end(); + if (dynamicData) { + pass.write({ __isStreamHeader: true, __isDynamicStructure: true }); + for (const item of dynamicData) { + pass.write(item); + } + pass.end(); + } else { + pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true }); + pass.write({ id: 1, country: 'Czechia' }); + pass.write({ id: 2, country: 'Austria' }); + pass.write({ country: 'Germany', id: 3 }); + pass.write({ country: 'Romania', id: 4 }); + pass.write({ country: 'Great Britain', id: 5 }); + pass.write({ country: 'Bosna, Hecegovina', id: 6 }); + pass.end(); + } } if (delay) { diff --git a/packages/api/src/shell/jsonLinesWriter.js b/packages/api/src/shell/jsonLinesWriter.js index a559e3174..44c56e8fd 100644 --- a/packages/api/src/shell/jsonLinesWriter.js +++ b/packages/api/src/shell/jsonLinesWriter.js @@ -12,7 +12,9 @@ class StringifyStream extends stream.Transform { _transform(chunk, encoding, done) { let skip = false; if (!this.wasHeader) { - skip = (chunk.__isStreamHeader && !this.header) || (chunk.__isStreamHeader && chunk.__isDynamicStructure); + skip = + (chunk.__isStreamHeader && !this.header) || + (chunk.__isStreamHeader && chunk.__isDynamicStructure && !chunk.__keepDynamicStreamHeader); this.wasHeader = true; } if (!skip) { diff --git a/packages/api/src/shell/modifyJsonLinesReader.js b/packages/api/src/shell/modifyJsonLinesReader.js index d76ed8c8a..acffb32f9 100644 --- a/packages/api/src/shell/modifyJsonLinesReader.js +++ b/packages/api/src/shell/modifyJsonLinesReader.js @@ -2,7 +2,7 @@ const fs = require('fs'); const _ = require('lodash'); const stream = require('stream'); const byline = require('byline'); -const { getLogger } = require('dbgate-tools'); +const { getLogger, processJsonDataUpdateCommands, removeTablePairingId } = require('dbgate-tools'); const logger = getLogger('modifyJsonLinesReader'); const stableStringify = require('json-stable-stringify'); @@ -11,6 +11,7 @@ class ParseStream extends stream.Transform { super({ objectMode: true }); this.limitRows = limitRows; this.changeSet = changeSet; + this.wasHeader = false; this.currentRowIndex = 0; if (mergeMode == 'merge') { if (mergedRows && mergeKey) { @@ -28,12 +29,28 @@ class ParseStream extends stream.Transform { _transform(chunk, encoding, done) { let obj = JSON.parse(chunk); if (obj.__isStreamHeader) { - this.push(obj); + if (this.changeSet && this.changeSet.structure) { + this.push({ + ...removeTablePairingId(this.changeSet.structure), + __isStreamHeader: true, + }); + } else { + this.push(obj); + } + this.wasHeader = true; done(); return; } if (this.changeSet) { + if (!this.wasHeader && this.changeSet.structure) { + this.push({ + ...removeTablePairingId(this.changeSet.structure), + __isStreamHeader: true, + }); + this.wasHeader = true; + } + if (!this.limitRows || this.currentRowIndex < this.limitRows) { if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) { obj = null; @@ -41,13 +58,20 @@ class ParseStream extends stream.Transform { const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex); if (update) { - obj = { - ...obj, - ...update.fields, - }; + if (update.document) { + obj = update.document; + } else { + obj = { + ...obj, + ...update.fields, + }; + } } if (obj) { + if (this.changeSet.dataUpdateCommands) { + obj = processJsonDataUpdateCommands(obj, this.changeSet.dataUpdateCommands); + } this.push(obj); } this.currentRowIndex += 1; diff --git a/packages/api/src/shell/runScript.js b/packages/api/src/shell/runScript.js index 331e4e1c8..5ce4c1a80 100644 --- a/packages/api/src/shell/runScript.js +++ b/packages/api/src/shell/runScript.js @@ -11,7 +11,7 @@ async function runScript(func) { await func(); process.exit(0); } catch (err) { - logger.error('Error running script', err); + logger.error({ err }, `Error running script: ${err.message}`); process.exit(1); } } diff --git a/packages/api/src/utility/JsonLinesDatastore.js b/packages/api/src/utility/JsonLinesDatastore.js index b7f9c109d..c34cb3300 100644 --- a/packages/api/src/utility/JsonLinesDatastore.js +++ b/packages/api/src/utility/JsonLinesDatastore.js @@ -2,7 +2,6 @@ const fs = require('fs'); const os = require('os'); const rimraf = require('rimraf'); const path = require('path'); -const lineReader = require('line-reader'); const AsyncLock = require('async-lock'); const lock = new AsyncLock(); const stableStringify = require('json-stable-stringify'); @@ -11,23 +10,7 @@ const requirePluginFunction = require('./requirePluginFunction'); const esort = require('external-sorting'); const uuidv1 = require('uuid/v1'); const { jsldir } = require('./directories'); - -function fetchNextLineFromReader(reader) { - return new Promise((resolve, reject) => { - if (!reader.hasNextLine()) { - resolve(null); - return; - } - - reader.nextLine((err, line) => { - if (err) { - reject(err); - } else { - resolve(line); - } - }); - }); -} +const LineReader = require('./LineReader'); class JsonLinesDatastore { constructor(file, formatterFunction) { @@ -74,7 +57,7 @@ class JsonLinesDatastore { await new Promise(resolve => rimraf(tempDir, resolve)); } - _closeReader() { + async _closeReader() { // console.log('CLOSING READER', this.reader); if (!this.reader) return; const reader = this.reader; @@ -84,7 +67,7 @@ class JsonLinesDatastore { // this.firstRowToBeReturned = null; this.currentFilter = null; this.currentSort = null; - return new Promise(resolve => reader.close(resolve)); + await reader.close(); } async notifyChanged(callback) { @@ -100,12 +83,9 @@ class JsonLinesDatastore { async _openReader(fileName) { // console.log('OPENING READER', fileName); // console.log(fs.readFileSync(fileName, 'utf-8')); - return new Promise((resolve, reject) => - lineReader.open(fileName, (err, reader) => { - if (err) reject(err); - resolve(reader); - }) - ); + + const fileStream = fs.createReadStream(fileName); + return new LineReader(fileStream); } parseLine(line) { @@ -120,7 +100,7 @@ class JsonLinesDatastore { // return res; // } for (;;) { - const line = await fetchNextLineFromReader(this.reader); + const line = await this.reader.readLine(); if (!line) { // EOF return null; @@ -240,6 +220,7 @@ class JsonLinesDatastore { // console.log(JSON.stringify(this.currentFilter, undefined, 2)); for (let i = 0; i < limit; i += 1) { const line = await this._readLine(true); + // console.log('READED LINE', i); if (line == null) break; res.push(line); } diff --git a/packages/api/src/utility/LineReader.js b/packages/api/src/utility/LineReader.js new file mode 100644 index 000000000..7fbed5b48 --- /dev/null +++ b/packages/api/src/utility/LineReader.js @@ -0,0 +1,88 @@ +const readline = require('readline'); + +class Queue { + constructor() { + this.elements = {}; + this.head = 0; + this.tail = 0; + } + enqueue(element) { + this.elements[this.tail] = element; + this.tail++; + } + dequeue() { + const item = this.elements[this.head]; + delete this.elements[this.head]; + this.head++; + return item; + } + peek() { + return this.elements[this.head]; + } + getLength() { + return this.tail - this.head; + } + isEmpty() { + return this.getLength() === 0; + } +} + +class LineReader { + constructor(input) { + this.input = input; + this.queue = new Queue(); + this.resolve = null; + this.isEnded = false; + this.rl = readline.createInterface({ + input, + }); + this.input.pause(); + + this.rl.on('line', line => { + this.input.pause(); + if (this.resolve) { + const resolve = this.resolve; + this.resolve = null; + resolve(line); + return; + } + this.queue.enqueue(line); + }); + + this.rl.on('close', () => { + if (this.resolve) { + const resolve = this.resolve; + this.resolve = null; + this.isEnded = true; + resolve(null); + return; + } + this.queue.enqueue(null); + }); + } + + readLine() { + if (this.isEnded) { + return Promise.resolve(null); + } + + if (!this.queue.isEmpty()) { + const res = this.queue.dequeue(); + if (res == null) this.isEnded = true; + return Promise.resolve(res); + } + + this.input.resume(); + + return new Promise(resolve => { + this.resolve = resolve; + }); + } + + close() { + this.isEnded = true; + return new Promise(resolve => this.input.close(resolve)); + } +} + +module.exports = LineReader; diff --git a/packages/api/src/utility/directories.js b/packages/api/src/utility/directories.js index 841ccb846..3d1f4f516 100644 --- a/packages/api/src/utility/directories.js +++ b/packages/api/src/utility/directories.js @@ -42,18 +42,23 @@ function datadir() { return dir; } -const dirFunc = (dirname, clean) => () => { - const dir = path.join(datadir(), dirname); - ensureDirectory(dir, clean); +const dirFunc = + (dirname, clean, subdirs = []) => + () => { + const dir = path.join(datadir(), dirname); + ensureDirectory(dir, clean); + for (const subdir of subdirs) { + ensureDirectory(path.join(dir, subdir), false); + } - return dir; -}; + return dir; + }; const jsldir = dirFunc('jsl', true); const rundir = dirFunc('run', true); const uploadsdir = dirFunc('uploads', true); const pluginsdir = dirFunc('plugins'); -const archivedir = dirFunc('archive'); +const archivedir = dirFunc('archive', false, ['default']); const appdir = dirFunc('apps'); const filesdir = dirFunc('files'); const logsdir = dirFunc('logs', 3600 * 24 * 7); diff --git a/packages/api/src/utility/freeTableStorage.js b/packages/api/src/utility/freeTableStorage.js deleted file mode 100644 index ceba774ef..000000000 --- a/packages/api/src/utility/freeTableStorage.js +++ /dev/null @@ -1,15 +0,0 @@ -const fs = require('fs-extra'); - -async function saveFreeTableData(file, data) { - const { structure, rows } = data; - const fileStream = fs.createWriteStream(file); - await fileStream.write(JSON.stringify({ __isStreamHeader: true, ...structure }) + '\n'); - for (const row of rows) { - await fileStream.write(JSON.stringify(row) + '\n'); - } - await fileStream.close(); -} - -module.exports = { - saveFreeTableData, -}; diff --git a/packages/datalib/src/ChangeSet.ts b/packages/datalib/src/ChangeSet.ts index 707db5db1..28396351a 100644 --- a/packages/datalib/src/ChangeSet.ts +++ b/packages/datalib/src/ChangeSet.ts @@ -9,7 +9,8 @@ import { AllowIdentityInsert, Expression, } from 'dbgate-sqltree'; -import type { NamedObjectInfo, DatabaseInfo } from 'dbgate-types'; +import type { NamedObjectInfo, DatabaseInfo, TableInfo } from 'dbgate-types'; +import { JsonDataObjectUpdateCommand } from 'dbgate-tools'; export interface ChangeSetItem { pureName: string; @@ -21,12 +22,18 @@ export interface ChangeSetItem { fields?: { [column: string]: string }; } -export interface ChangeSet { +export interface ChangeSetItemFields { inserts: ChangeSetItem[]; updates: ChangeSetItem[]; deletes: ChangeSetItem[]; } +export interface ChangeSet extends ChangeSetItemFields { + structure?: TableInfo; + dataUpdateCommands?: JsonDataObjectUpdateCommand[]; + setColumnMode?: 'fixed' | 'variable'; +} + export function createChangeSet(): ChangeSet { return { inserts: [], @@ -51,7 +58,7 @@ export interface ChangeSetFieldDefinition extends ChangeSetRowDefinition { export function findExistingChangeSetItem( changeSet: ChangeSet, definition: ChangeSetRowDefinition -): [keyof ChangeSet, ChangeSetItem] { +): [keyof ChangeSetItemFields, ChangeSetItem] { if (!changeSet || !definition) return ['updates', null]; if (definition.insertedRowIndex != null) { return [ @@ -456,5 +463,12 @@ export function changeSetInsertDocuments(changeSet: ChangeSet, documents: any[], export function changeSetContainsChanges(changeSet: ChangeSet) { if (!changeSet) return false; - return changeSet.deletes.length > 0 || changeSet.updates.length > 0 || changeSet.inserts.length > 0; + return ( + changeSet.deletes.length > 0 || + changeSet.updates.length > 0 || + changeSet.inserts.length > 0 || + !!changeSet.structure || + !!changeSet.setColumnMode || + changeSet.dataUpdateCommands?.length > 0 + ); } diff --git a/packages/datalib/src/DataDuplicator.ts b/packages/datalib/src/DataDuplicator.ts index faa475708..f78c42149 100644 --- a/packages/datalib/src/DataDuplicator.ts +++ b/packages/datalib/src/DataDuplicator.ts @@ -12,6 +12,11 @@ export interface DataDuplicatorItem { matchColumns: string[]; } +export interface DataDuplicatorOptions { + rollbackAfterFinish?: boolean; + skipRowsWithUnresolvedRefs?: boolean; +} + class DuplicatorReference { constructor( public base: DuplicatorItemHolder, @@ -78,6 +83,13 @@ class DuplicatorItemHolder { if (ref) { // remap id res[key] = ref.ref.idMap[res[key]]; + if (ref.isMandatory && res[key] == null) { + // mandatory refertence not matched + if (this.duplicator.options.skipRowsWithUnresolvedRefs) { + return null; + } + throw new Error(`Unresolved reference, base=${ref.base.name}, ref=${ref.ref.name}, ${key}=${chunk[key]}`); + } } } @@ -91,6 +103,8 @@ class DuplicatorItemHolder { let inserted = 0; let mapped = 0; let missing = 0; + let skipped = 0; + let lastLogged = new Date(); const writeStream = createAsyncWriteStream(this.duplicator.stream, { processItem: async chunk => { @@ -99,18 +113,35 @@ class DuplicatorItemHolder { } const doCopy = async () => { + // console.log('chunk', this.name, JSON.stringify(chunk)); const insertedObj = this.createInsertObject(chunk); - await runCommandOnDriver(pool, driver, dmp => - dmp.putCmd( + // console.log('insertedObj', this.name, JSON.stringify(insertedObj)); + if (insertedObj == null) { + skipped += 1; + return; + } + let res = await runQueryOnDriver(pool, driver, dmp => { + dmp.put( '^insert ^into %f (%,i) ^values (%,v)', this.table, Object.keys(insertedObj), Object.values(insertedObj) - ) - ); + ); + + if ( + this.autoColumn && + this.isReferenced && + !this.duplicator.driver.dialect.requireStandaloneSelectForScopeIdentity + ) { + dmp.selectScopeIdentity(this.table); + } + }); inserted += 1; if (this.autoColumn && this.isReferenced) { - const res = await runQueryOnDriver(pool, driver, dmp => dmp.selectScopeIdentity(this.table)); + if (this.duplicator.driver.dialect.requireStandaloneSelectForScopeIdentity) { + res = await runQueryOnDriver(pool, driver, dmp => dmp.selectScopeIdentity(this.table)); + } + // console.log('IDRES', JSON.stringify(res)); const resId = Object.entries(res?.rows?.[0])?.[0]?.[1]; if (resId != null) { this.idMap[chunk[this.autoColumn]] = resId; @@ -146,6 +177,13 @@ class DuplicatorItemHolder { break; } } + + if (new Date().getTime() - lastLogged.getTime() > 5000) { + logger.info( + `Duplicating ${this.item.name} in progress, inserted ${inserted} rows, mapped ${mapped} rows, missing ${missing} rows, skipped ${skipped} rows` + ); + lastLogged = new Date(); + } // this.idMap[oldId] = newId; }, }); @@ -158,7 +196,7 @@ class DuplicatorItemHolder { // }, // }); - return { inserted, mapped, missing }; + return { inserted, mapped, missing, skipped }; } } @@ -172,7 +210,8 @@ export class DataDuplicator { public db: DatabaseInfo, public items: DataDuplicatorItem[], public stream, - public copyStream: (input, output) => Promise + public copyStream: (input, output) => Promise, + public options: DataDuplicatorOptions = {} ) { this.itemHolders = items.map(x => new DuplicatorItemHolder(x, this)); this.itemHolders.forEach(x => x.initializeReferences()); @@ -212,13 +251,20 @@ export class DataDuplicator { for (const item of this.itemPlan) { const stats = await item.runImport(); logger.info( - `Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows` + `Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows, skipped ${stats.skipped} rows` ); } } catch (err) { - logger.error({ err }, 'Failed duplicator job, rollbacking'); + logger.error({ err }, `Failed duplicator job, rollbacking. ${err.message}`); await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction()); + return; + } + if (this.options.rollbackAfterFinish) { + logger.info('Rollbacking transaction, nothing was changed'); + await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction()); + } else { + logger.info('Committing duplicator transaction'); + await runCommandOnDriver(this.pool, this.driver, dmp => dmp.commitTransaction()); } - await runCommandOnDriver(this.pool, this.driver, dmp => dmp.commitTransaction()); } } diff --git a/packages/datalib/src/FreeTableGridDisplay.ts b/packages/datalib/src/FreeTableGridDisplay.ts deleted file mode 100644 index 607abf4e9..000000000 --- a/packages/datalib/src/FreeTableGridDisplay.ts +++ /dev/null @@ -1,48 +0,0 @@ -import _ from 'lodash'; -import type { EngineDriver, ViewInfo, ColumnInfo } from 'dbgate-types'; -import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay'; -import { GridConfig, GridCache } from './GridConfig'; -import { FreeTableModel } from './FreeTableModel'; -import { analyseCollectionDisplayColumns } from '.'; - -export class FreeTableGridDisplay extends GridDisplay { - constructor( - public model: FreeTableModel, - config: GridConfig, - setConfig: ChangeConfigFunc, - cache: GridCache, - setCache: ChangeCacheFunc - ) { - super(config, setConfig, cache, setCache); - this.columns = model?.structure?.__isDynamicStructure - ? analyseCollectionDisplayColumns(model?.rows, this) - : this.getDisplayColumns(model); - this.filterable = false; - this.sortable = false; - } - - getDisplayColumns(model: FreeTableModel) { - return _.uniqBy( - model?.structure?.columns - ?.map(col => this.getDisplayColumn(col)) - ?.map(col => ({ - ...col, - isChecked: this.isColumnChecked(col), - })) || [], - col => col.uniqueName - ); - } - - getDisplayColumn(col: ColumnInfo) { - const uniquePath = [col.columnName]; - const uniqueName = uniquePath.join('.'); - return { - ...col, - pureName: 'data', - schemaName: '', - headerText: col.columnName, - uniqueName, - uniquePath, - }; - } -} diff --git a/packages/datalib/src/FreeTableModel.ts b/packages/datalib/src/FreeTableModel.ts deleted file mode 100644 index 616e0c8e2..000000000 --- a/packages/datalib/src/FreeTableModel.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { TableInfo } from 'dbgate-types'; - -export interface FreeTableModel { - structure: TableInfo; - rows: any[]; -} - -export function createFreeTableModel() { - return { - structure: { - columns: [ - { - columnName: 'col1', - }, - ], - foreignKeys: [], - }, - rows: [ - { - col1: 'val1', - }, - { - col1: 'val2', - }, - ], - }; -} diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 4f505ebca..9a7fc7707 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -84,6 +84,7 @@ export abstract class GridDisplay { return this.baseTable || this.baseView; } changeSetKeyFields: string[] = null; + editableStructure: TableInfo = null; sortable = false; groupable = false; filterable = false; diff --git a/packages/datalib/src/JslGridDisplay.ts b/packages/datalib/src/JslGridDisplay.ts index 9a34f490d..030471bea 100644 --- a/packages/datalib/src/JslGridDisplay.ts +++ b/packages/datalib/src/JslGridDisplay.ts @@ -24,6 +24,7 @@ export class JslGridDisplay extends GridDisplay { this.isDynamicStructure = isDynamicStructure; this.filterTypeOverride = 'eval'; this.editable = editable; + this.editableStructure = editable ? structure : null; if (structure?.columns) { this.columns = _.uniqBy( diff --git a/packages/datalib/src/MacroDefinition.ts b/packages/datalib/src/MacroDefinition.ts index e7fa0b5b1..cf1839631 100644 --- a/packages/datalib/src/MacroDefinition.ts +++ b/packages/datalib/src/MacroDefinition.ts @@ -11,7 +11,7 @@ export interface MacroDefinition { name: string; group: string; description?: string; - type: 'transformValue'; + type: 'transformValue' | 'transformRow'; code: string; args?: MacroArgument[]; } diff --git a/packages/datalib/src/index.ts b/packages/datalib/src/index.ts index 347769a59..30daf5c30 100644 --- a/packages/datalib/src/index.ts +++ b/packages/datalib/src/index.ts @@ -6,12 +6,8 @@ export * from './TableGridDisplay'; export * from './ViewGridDisplay'; export * from './JslGridDisplay'; export * from './ChangeSet'; -export * from './FreeTableGridDisplay'; -export * from './FreeTableModel'; export * from './MacroDefinition'; export * from './runMacro'; -// export * from './FormViewDisplay'; -// export * from './TableFormViewDisplay'; export * from './CollectionGridDisplay'; export * from './deleteCascade'; export * from './PerspectiveDisplay'; diff --git a/packages/datalib/src/runMacro.ts b/packages/datalib/src/runMacro.ts index 19d70385c..a85c1d7ab 100644 --- a/packages/datalib/src/runMacro.ts +++ b/packages/datalib/src/runMacro.ts @@ -1,10 +1,9 @@ -import { FreeTableModel } from './FreeTableModel'; import _ from 'lodash'; import uuidv1 from 'uuid/v1'; import uuidv4 from 'uuid/v4'; import moment from 'moment'; import { MacroDefinition, MacroSelectedCell } from './MacroDefinition'; -import { ChangeSet, setChangeSetValue } from './ChangeSet'; +import { ChangeSet, setChangeSetValue, setChangeSetRowData } from './ChangeSet'; import { GridDisplay } from './GridDisplay'; const getMacroFunction = { @@ -13,13 +12,8 @@ const getMacroFunction = { ${code} } `, - transformRows: code => ` -(rows, args, modules, selectedCells, cols, columns) => { - ${code} -} -`, - transformData: code => ` -(rows, args, modules, selectedCells, cols, columns) => { + transformRow: code => ` +(row, args, modules, rowIndex, columns) => { ${code} } `, @@ -32,160 +26,6 @@ const modules = { moment, }; -function runTramsformValue( - func, - macroArgs: {}, - data: FreeTableModel, - preview: boolean, - selectedCells: MacroSelectedCell[], - errors: string[] = [] -) { - const selectedRows = _.groupBy(selectedCells, 'row'); - const rows = data.rows.map((row, rowIndex) => { - const selectedRow = selectedRows[rowIndex]; - if (selectedRow) { - const modifiedFields = []; - let res = null; - for (const cell of selectedRow) { - const { column } = cell; - const oldValue = row[column]; - let newValue = oldValue; - try { - newValue = func(oldValue, macroArgs, modules, rowIndex, row, column); - } catch (err) { - errors.push(`Error processing column ${column} on row ${rowIndex}: ${err.message}`); - } - if (newValue != oldValue) { - if (res == null) { - res = { ...row }; - } - res[column] = newValue; - if (preview) modifiedFields.push(column); - } - } - if (res) { - if (modifiedFields.length > 0) { - return { - ...res, - __modifiedFields: new Set(modifiedFields), - }; - } - return res; - } - return row; - } else { - return row; - } - }); - - return { - structure: data.structure, - rows, - }; -} - -function removePreviewRowFlags(rows) { - rows = rows.filter(row => row.__rowStatus != 'deleted'); - rows = rows.map(row => { - if (row.__rowStatus || row.__modifiedFields || row.__insertedFields || row.__deletedFields) - return _.omit(row, ['__rowStatus', '__modifiedFields', '__insertedFields', '__deletedFields']); - return row; - }); - return rows; -} - -function runTramsformRows( - func, - macroArgs: {}, - data: FreeTableModel, - preview: boolean, - selectedCells: MacroSelectedCell[], - errors: string[] = [] -) { - let rows = data.rows; - try { - rows = func( - data.rows, - macroArgs, - modules, - selectedCells, - data.structure.columns.map(x => x.columnName), - data.structure.columns - ); - if (!preview) { - rows = removePreviewRowFlags(rows); - } - } catch (err) { - errors.push(`Error processing rows: ${err.message}`); - } - return { - structure: data.structure, - rows, - }; -} - -function runTramsformData( - func, - macroArgs: {}, - data: FreeTableModel, - preview: boolean, - selectedCells: MacroSelectedCell[], - errors: string[] = [] -) { - try { - let { rows, columns, cols } = func( - data.rows, - macroArgs, - modules, - selectedCells, - data.structure.columns.map(x => x.columnName), - data.structure.columns - ); - if (cols && !columns) { - columns = cols.map(columnName => ({ columnName })); - } - columns = _.uniqBy(columns, 'columnName'); - if (!preview) { - rows = removePreviewRowFlags(rows); - } - return { - structure: { columns }, - rows, - }; - } catch (err) { - errors.push(`Error processing data: ${err.message}`); - } - return data; -} - -export function runMacro( - macro: MacroDefinition, - macroArgs: {}, - data: FreeTableModel, - preview: boolean, - selectedCells: MacroSelectedCell[], - errors: string[] = [] -): FreeTableModel { - let func; - try { - func = eval(getMacroFunction[macro.type](macro.code)); - } catch (err) { - errors.push(`Error compiling macro ${macro.name}: ${err.message}`); - return data; - } - if (macro.type == 'transformValue') { - return runTramsformValue(func, macroArgs, data, preview, selectedCells, errors); - } - if (macro.type == 'transformRows') { - return runTramsformRows(func, macroArgs, data, preview, selectedCells, errors); - } - if (macro.type == 'transformData') { - // @ts-ignore - return runTramsformData(func, macroArgs, data, preview, selectedCells, errors); - } - return data; -} - export function compileMacroFunction(macro: MacroDefinition, errors = []) { if (!macro) return null; let func; @@ -198,7 +38,7 @@ export function compileMacroFunction(macro: MacroDefinition, errors = []) { } } -export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, column, errors = []) { +export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex: number, row, column: string, errors = []) { if (!compiledFunc) return value; try { const res = compiledFunc(value, macroArgs, modules, rowIndex, row, column); @@ -209,31 +49,62 @@ export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, c } } +export function runMacroOnRow(compiledFunc, macroArgs, rowIndex: number, row: any, columns: string[], errors = []) { + if (!compiledFunc) return row; + try { + const res = compiledFunc(row, macroArgs, modules, rowIndex, columns); + return res; + } catch (err) { + errors.push(`Error processing row ${rowIndex}: ${err.message}`); + return row; + } +} + export function runMacroOnChangeSet( macro: MacroDefinition, macroArgs: {}, selectedCells: MacroSelectedCell[], changeSet: ChangeSet, - display: GridDisplay + display: GridDisplay, + useRowIndexInsteaOfCondition: boolean ): ChangeSet { const errors = []; const compiledMacroFunc = compileMacroFunction(macro, errors); if (!compiledMacroFunc) return null; - let res = changeSet; - for (const cell of selectedCells) { - const definition = display.getChangeSetField(cell.rowData, cell.column, undefined); - const macroResult = runMacroOnValue( - compiledMacroFunc, - macroArgs, - cell.value, - cell.row, - cell.rowData, - cell.column, - errors - ); - res = setChangeSetValue(res, definition, macroResult); + if (macro.type == 'transformValue') { + let res = changeSet; + for (const cell of selectedCells) { + const definition = display.getChangeSetField( + cell.rowData, + cell.column, + undefined, + useRowIndexInsteaOfCondition ? cell.row : undefined, + useRowIndexInsteaOfCondition + ); + const macroResult = runMacroOnValue( + compiledMacroFunc, + macroArgs, + cell.value, + cell.row, + cell.rowData, + cell.column, + errors + ); + res = setChangeSetValue(res, definition, macroResult); + } + return res; + } + if (macro.type == 'transformRow') { + let res = changeSet; + const rowIndexes = _.uniq(selectedCells.map(x => x.row)); + for (const index of rowIndexes) { + const rowData = selectedCells.find(x => x.row == index)?.rowData; + const columns = _.uniq(selectedCells.map(x => x.column)); + const definition = display.getChangeSetRow(rowData, null, index, true); + const newRow = runMacroOnRow(compiledMacroFunc, macroArgs, index, rowData, columns); + res = setChangeSetRowData(res, definition, newRow); + } + return res; } - - return res; } diff --git a/packages/tools/src/DatabaseAnalyser.ts b/packages/tools/src/DatabaseAnalyser.ts index c3e1f24aa..f113f7a4b 100644 --- a/packages/tools/src/DatabaseAnalyser.ts +++ b/packages/tools/src/DatabaseAnalyser.ts @@ -235,7 +235,10 @@ export class DatabaseAnalyser { if (this.pool.feedback) { this.pool.feedback(obj); } - } + if (obj && obj.analysingMessage) { + logger.debug(obj.analysingMessage); + } + } async getModifications() { const snapshot = await this._getFastSnapshot(); diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index 590b62c2f..54cfd8d23 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -244,16 +244,7 @@ export class SqlDumper implements AlterProcessor { this.put('%i ', col.columnName); this.columnDefinition(col); }); - if (table.primaryKey) { - this.put(',&n'); - if (table.primaryKey.constraintName) { - this.put('^constraint %i', table.primaryKey.constraintName); - } - this.put( - ' ^primary ^key (%,i)', - table.primaryKey.columns.map(x => x.columnName) - ); - } + this.createTablePrimaryKeyCore(table); (table.foreignKeys || []).forEach(fk => { this.put(',&n'); @@ -275,6 +266,19 @@ export class SqlDumper implements AlterProcessor { }); } + createTablePrimaryKeyCore(table: TableInfo) { + if (table.primaryKey) { + this.put(',&n'); + if (table.primaryKey.constraintName) { + this.put('^constraint %i', table.primaryKey.constraintName); + } + this.put( + ' ^primary ^key (%,i)', + table.primaryKey.columns.map(x => x.columnName) + ); + } + } + createForeignKeyFore(fk: ForeignKeyInfo) { if (fk.constraintName != null) this.put('^constraint %i ', fk.constraintName); this.put( diff --git a/packages/tools/src/createAsyncWriteStream.ts b/packages/tools/src/createAsyncWriteStream.ts index 81dd1155e..46a0ce95e 100644 --- a/packages/tools/src/createAsyncWriteStream.ts +++ b/packages/tools/src/createAsyncWriteStream.ts @@ -14,23 +14,12 @@ export function createAsyncWriteStream(stream, options: AsyncWriteStreamOptions) }); writable._write = async (chunk, encoding, callback) => { - await options.processItem(chunk); - - // const { sql, id, newIdSql } = chunk; - // if (_isArray(sql)) { - // for (const item of sql) await driver.query(pool, item, { discardResult: true }); - // } else { - // await driver.query(pool, sql, { discardResult: true }); - // } - // if (newIdSql) { - // const res = await driver.query(pool, newIdSql); - // const resId = Object.entries(res?.rows?.[0])?.[0]?.[1]; - - // if (options?.mapResultId) { - // options?.mapResultId(id, resId as string); - // } - // } - callback(); + try { + await options.processItem(chunk); + callback(null); + } catch (err) { + callback(err); + } }; // writable._final = async callback => { diff --git a/packages/tools/src/diffTools.ts b/packages/tools/src/diffTools.ts index ecda30151..01081f663 100644 --- a/packages/tools/src/diffTools.ts +++ b/packages/tools/src/diffTools.ts @@ -72,6 +72,34 @@ export function generateTablePairingId(table: TableInfo): TableInfo { return table; } +export function removeTablePairingId(table: TableInfo): TableInfo { + if (!table) return table; + return { + ...table, + columns: table.columns?.map(col => ({ + ...col, + pairingId: undefined, + })), + foreignKeys: table.foreignKeys?.map(cnt => ({ + ...cnt, + pairingId: undefined, + })), + checks: table.checks?.map(cnt => ({ + ...cnt, + pairingId: undefined, + })), + indexes: table.indexes?.map(cnt => ({ + ...cnt, + pairingId: undefined, + })), + uniques: table.uniques?.map(cnt => ({ + ...cnt, + pairingId: undefined, + })), + pairingId: undefined, + }; +} + function generateObjectPairingId(obj) { if (obj.objectTypeField) return { @@ -346,7 +374,13 @@ function createPairs(oldList, newList, additionalCondition = null) { function planTablePreload(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo) { const key = newTable.preloadedRowsKey || newTable.primaryKey?.columns?.map(x => x.columnName); if (newTable.preloadedRows?.length > 0 && key?.length > 0) { - plan.fillPreloadedRows(newTable, oldTable?.preloadedRows, newTable.preloadedRows, key, newTable.preloadedRowsInsertOnly); + plan.fillPreloadedRows( + newTable, + oldTable?.preloadedRows, + newTable.preloadedRows, + key, + newTable.preloadedRowsInsertOnly + ); } } diff --git a/packages/tools/src/driverBase.ts b/packages/tools/src/driverBase.ts index ac7b312a6..626ed0b24 100644 --- a/packages/tools/src/driverBase.ts +++ b/packages/tools/src/driverBase.ts @@ -23,6 +23,7 @@ const dialect = { export async function runCommandOnDriver(pool, driver: EngineDriver, cmd: (dmp: SqlDumper) => void): Promise { const dmp = driver.createDumper(); cmd(dmp as any); + // console.log('CMD:', dmp.s); await driver.query(pool, dmp.s, { discardResult: true }); } @@ -33,6 +34,7 @@ export async function runQueryOnDriver( ): Promise { const dmp = driver.createDumper(); cmd(dmp as any); + // console.log('QUERY:', dmp.s); return await driver.query(pool, dmp.s); } diff --git a/packages/tools/src/schemaEditorTools.ts b/packages/tools/src/schemaEditorTools.ts index 5c5c92657..18e095198 100644 --- a/packages/tools/src/schemaEditorTools.ts +++ b/packages/tools/src/schemaEditorTools.ts @@ -10,6 +10,14 @@ import type { UniqueInfo, } from 'dbgate-types'; import _ from 'lodash'; +import { parseSqlDefaultValue } from './stringTools'; + +export interface JsonDataObjectUpdateCommand { + type: 'renameField' | 'deleteField' | 'setField' | 'setFieldIfNull'; + oldField?: string; + newField?: string; + value?: any; +} export interface EditorColumnInfo extends ColumnInfo { isPrimaryKey?: boolean; @@ -23,6 +31,41 @@ export function fillEditorColumnInfo(column: ColumnInfo, table: TableInfo): Edit }; } +export function processJsonDataUpdateCommands(obj: any, commands: JsonDataObjectUpdateCommand[] = []) { + for (const cmd of commands) { + switch (cmd.type) { + case 'deleteField': + obj = { + ...obj, + }; + delete obj[cmd.oldField]; + break; + case 'renameField': + obj = { + ...obj, + }; + obj[cmd.newField] = obj[cmd.oldField]; + delete obj[cmd.oldField]; + break; + case 'setField': + obj = { + ...obj, + }; + obj[cmd.newField] = cmd.value; + break; + case 'setFieldIfNull': + obj = { + ...obj, + }; + if (obj[cmd.newField] == null) { + obj[cmd.newField] = cmd.value; + } + break; + } + } + return obj; +} + function processPrimaryKey(table: TableInfo, oldColumn: EditorColumnInfo, newColumn: EditorColumnInfo): TableInfo { if (!oldColumn?.isPrimaryKey && newColumn?.isPrimaryKey) { let primaryKey = table?.primaryKey; @@ -71,7 +114,11 @@ function processPrimaryKey(table: TableInfo, oldColumn: EditorColumnInfo, newCol return table; } -export function editorAddColumn(table: TableInfo, column: EditorColumnInfo): TableInfo { +function defineDataCommand(table: TableInfo, cmd: () => JsonDataObjectUpdateCommand) { + table['__addDataCommands'] = [...(table['__addDataCommands'] || []), cmd()]; +} + +export function editorAddColumn(table: TableInfo, column: EditorColumnInfo, addDataCommand?: boolean): TableInfo { let res = { ...table, columns: [...(table?.columns || []), { ...column, pairingId: uuidv1() }], @@ -79,10 +126,18 @@ export function editorAddColumn(table: TableInfo, column: EditorColumnInfo): Tab res = processPrimaryKey(res, null, column); + if (addDataCommand && column.defaultValue) { + defineDataCommand(res, () => ({ + type: 'setField', + newField: column.columnName, + value: parseSqlDefaultValue(column.defaultValue), + })); + } + return res; } -export function editorModifyColumn(table: TableInfo, column: EditorColumnInfo): TableInfo { +export function editorModifyColumn(table: TableInfo, column: EditorColumnInfo, addDataCommand?: boolean): TableInfo { const oldColumn = table?.columns?.find(x => x.pairingId == column.pairingId); let res = { @@ -91,10 +146,26 @@ export function editorModifyColumn(table: TableInfo, column: EditorColumnInfo): }; res = processPrimaryKey(res, fillEditorColumnInfo(oldColumn, table), column); + if (addDataCommand && oldColumn.columnName != column.columnName) { + defineDataCommand(res, () => ({ + type: 'renameField', + oldField: oldColumn.columnName, + newField: column.columnName, + })); + } + + if (addDataCommand && !oldColumn.defaultValue && column.defaultValue) { + defineDataCommand(res, () => ({ + type: 'setFieldIfNull', + newField: column.columnName, + value: parseSqlDefaultValue(column.defaultValue), + })); + } + return res; } -export function editorDeleteColumn(table: TableInfo, column: EditorColumnInfo): TableInfo { +export function editorDeleteColumn(table: TableInfo, column: EditorColumnInfo, addDataCommand?: boolean): TableInfo { let res = { ...table, columns: table.columns.filter(col => col.pairingId != column.pairingId), @@ -102,6 +173,13 @@ export function editorDeleteColumn(table: TableInfo, column: EditorColumnInfo): res = processPrimaryKey(res, column, null); + if (addDataCommand) { + defineDataCommand(res, () => ({ + type: 'deleteField', + oldField: column.columnName, + })); + } + return res; } diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index 8705ac370..21be36385 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -115,3 +115,15 @@ export function getAsImageSrc(obj) { return null; } + +export function parseSqlDefaultValue(value: string) { + if (!value) return undefined; + if (!_isString(value)) return undefined; + if (value.startsWith("'") && value.endsWith("'")) { + return value.slice(1, -1); + } + if (!isNaN(value as any) && !isNaN(parseFloat(value))) { + return parseFloat(value); + } + return undefined; +} diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index 8669b4f6a..166d8298f 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -11,6 +11,7 @@ export interface SqlDialect { anonymousPrimaryKey?: boolean; defaultSchemaName?: string; enableConstraintsPerTable?: boolean; + requireStandaloneSelectForScopeIdentity?: boolean; dropColumnDependencies?: string[]; changeColumnDependencies?: string[]; diff --git a/packages/types/extensions.d.ts b/packages/types/extensions.d.ts index 84c95b732..0fa777891 100644 --- a/packages/types/extensions.d.ts +++ b/packages/types/extensions.d.ts @@ -37,8 +37,9 @@ export interface PluginDefinition { export interface QuickExportDefinition { label: string; - createWriter: (fileName: string) => { functionName: string; props: any }; + createWriter: (fileName: string, dataName?: string) => { functionName: string; props: any }; extension: string; + noFilenameDependency?: boolean; } export interface ExtensionsDirectory { diff --git a/packages/web/src/TabRegister.svelte b/packages/web/src/TabRegister.svelte index d5bf9de5d..3f3f3966d 100644 --- a/packages/web/src/TabRegister.svelte +++ b/packages/web/src/TabRegister.svelte @@ -1,16 +1,3 @@ - - {#each _.keys(mountedTabs) as tabid (tabid)} diff --git a/packages/web/src/appobj/ArchiveFileAppObject.svelte b/packages/web/src/appobj/ArchiveFileAppObject.svelte index 4cecca45d..20457d54f 100644 --- a/packages/web/src/appobj/ArchiveFileAppObject.svelte +++ b/packages/web/src/appobj/ArchiveFileAppObject.svelte @@ -54,11 +54,9 @@ 'matview.sql': 'img view', }; - function getArchiveIcon(archiveFilesAsDataSheets, data) { + function getArchiveIcon(data) { if (data.fileType == 'jsonl') { - return isArchiveFileMarkedAsDataSheet(archiveFilesAsDataSheets, data.folderName, data.fileName) - ? 'img free-table' - : 'img archive'; + return 'img archive'; } return ARCHIVE_ICONS[data.fileType]; } @@ -70,20 +68,14 @@ import ImportExportModal from '../modals/ImportExportModal.svelte'; import { showModal } from '../modals/modalTools'; - import { archiveFilesAsDataSheets, currentArchive, extensions, getCurrentDatabase, getExtensions } from '../stores'; + import { getExtensions } from '../stores'; import createQuickExportMenu from '../utility/createQuickExportMenu'; import { exportQuickExportFile } from '../utility/exportFileTools'; import openNewTab from '../utility/openNewTab'; import AppObjectCore from './AppObjectCore.svelte'; - import getConnectionLabel from '../utility/getConnectionLabel'; import InputTextModal from '../modals/InputTextModal.svelte'; import ConfirmModal from '../modals/ConfirmModal.svelte'; - import { - isArchiveFileMarkedAsDataSheet, - markArchiveFileAsDataSheet, - markArchiveFileAsReadonly, - } from '../utility/archiveTools'; import { apiCall } from '../utility/api'; export let data; @@ -116,36 +108,12 @@ }, }); }; - const handleOpenRead = () => { - markArchiveFileAsReadonly(data.folderName, data.fileName); + const handleOpenArchive = () => { openArchive(data.fileName, data.folderName); }; - const handleOpenWrite = () => { - markArchiveFileAsDataSheet(data.folderName, data.fileName); - openNewTab({ - title: data.fileName, - icon: 'img free-table', - tabComponent: 'FreeTableTab', - props: { - initialArgs: { - functionName: 'archiveReader', - props: { - fileName: data.fileName, - folderName: data.folderName, - }, - }, - archiveFile: data.fileName, - archiveFolder: data.folderName, - }, - }); - }; const handleClick = () => { if (data.fileType == 'jsonl') { - if (isArchiveFileMarkedAsDataSheet($archiveFilesAsDataSheets, data.folderName, data.fileName)) { - handleOpenWrite(); - } else { - handleOpenRead(); - } + handleOpenArchive(); } if (data.fileType.endsWith('.sql')) { handleOpenSqlFile(); @@ -166,8 +134,7 @@ function createMenu() { return [ - data.fileType == 'jsonl' && { text: 'Open (readonly)', onClick: handleOpenRead }, - data.fileType == 'jsonl' && { text: 'Open as data sheet', onClick: handleOpenWrite }, + data.fileType == 'jsonl' && { text: 'Open', onClick: handleOpenArchive }, data.fileType == 'jsonl' && { text: 'Open in text editor', onClick: handleOpenJsonLinesText }, { text: 'Delete', onClick: handleDelete }, { text: 'Rename', onClick: handleRename }, @@ -232,7 +199,7 @@ {...$$restProps} {data} title={data.fileLabel} - icon={getArchiveIcon($archiveFilesAsDataSheets, data)} + icon={getArchiveIcon(data)} menu={createMenu} on:click={handleClick} /> diff --git a/packages/web/src/appobj/DatabaseObjectAppObject.svelte b/packages/web/src/appobj/DatabaseObjectAppObject.svelte index 57cbce1f2..650308a9a 100644 --- a/packages/web/src/appobj/DatabaseObjectAppObject.svelte +++ b/packages/web/src/appobj/DatabaseObjectAppObject.svelte @@ -102,10 +102,6 @@ isImport: true, requiresWriteAccess: true, }, - { - label: 'Open as data sheet', - isOpenFreeTable: true, - }, { label: 'Open active chart', isActiveChart: true, @@ -176,10 +172,6 @@ isExport: true, functionName: 'tableReader', }, - { - label: 'Open as data sheet', - isOpenFreeTable: true, - }, { label: 'Open active chart', isActiveChart: true, @@ -242,10 +234,6 @@ isExport: true, functionName: 'tableReader', }, - { - label: 'Open as data sheet', - isOpenFreeTable: true, - }, { label: 'Open active chart', isActiveChart: true, @@ -409,27 +397,7 @@ return driver; }; - if (menu.isOpenFreeTable) { - const coninfo = await getConnectionInfo(data); - openNewTab({ - title: data.pureName, - icon: 'img free-table', - tabComponent: 'FreeTableTab', - props: { - initialArgs: { - functionName: 'tableReader', - props: { - connection: { - ...coninfo, - database: data.database, - }, - schemaName: data.schemaName, - pureName: data.pureName, - }, - }, - }, - }); - } else if (menu.isActiveChart) { + if (menu.isActiveChart) { const driver = await getDriver(); const dmp = driver.createDumper(); dmp.put('^select * from %f', data); @@ -734,6 +702,22 @@ }, { onClick: () => { + // openNewTab( + // { + // tabComponent: 'ImportExportTab', + // title: 'Import/Export', + // icon: 'img export', + // }, + // { + // editor: { + // sourceStorageType: 'database', + // sourceConnectionId: data.conid, + // sourceDatabaseName: data.database, + // sourceSchemaName: data.schemaName, + // sourceList: [data.pureName], + // }, + // } + // ); showModal(ImportExportModal, { initialValues: { sourceStorageType: 'database', diff --git a/packages/web/src/commands/stdCommands.ts b/packages/web/src/commands/stdCommands.ts index 62302dfc2..46f23cb48 100644 --- a/packages/web/src/commands/stdCommands.ts +++ b/packages/web/src/commands/stdCommands.ts @@ -58,7 +58,7 @@ registerCommand({ category: 'Theme', name: 'Change', toolbarName: 'Change theme', - onClick: () => showModal(SettingsModal, { selectedTab: 1 }), + onClick: () => showModal(SettingsModal, { selectedTab: 2 }), // getSubCommands: () => get(extensions).themes.map(themeCommand), }); @@ -329,21 +329,6 @@ registerCommand({ }, }); -registerCommand({ - id: 'new.freetable', - category: 'New', - icon: 'img markdown', - name: 'Data sheet', - menuName: 'New data sheet', - onClick: () => { - openNewTab({ - title: 'Data #', - icon: 'img free-table', - tabComponent: 'FreeTableTab', - }); - }, -}); - registerCommand({ id: 'new.jsonl', category: 'New', diff --git a/packages/web/src/datagrid/ChangeSetGrider.ts b/packages/web/src/datagrid/ChangeSetGrider.ts index 5ba218b56..6d7d1abc9 100644 --- a/packages/web/src/datagrid/ChangeSetGrider.ts +++ b/packages/web/src/datagrid/ChangeSetGrider.ts @@ -1,4 +1,4 @@ -import type { ChangeSet, MacroDefinition, MacroSelectedCell } from 'dbgate-datalib'; +import { ChangeSet, MacroDefinition, MacroSelectedCell, runMacroOnRow } from 'dbgate-datalib'; import { changeSetContainsChanges, @@ -17,6 +17,7 @@ import { } from 'dbgate-datalib'; import Grider from './Grider'; import type { GriderRowStatus } from './Grider'; +import _ from 'lodash'; function getRowFromItem(row, matchedChangeSetItem) { return matchedChangeSetItem.document @@ -38,7 +39,7 @@ export default class ChangeSetGrider extends Grider { private rowStatusCache; private rowDefinitionsCache; private batchChangeSet: ChangeSet; - private _errors = null; + private _errors = []; private compiledMacroFunc; constructor( @@ -89,7 +90,7 @@ export default class ChangeSetGrider extends Grider { this.useRowIndexInsteaOfCondition ); const [matchedField, matchedChangeSetItem] = findExistingChangeSetItem(this.changeSet, rowDefinition); - const rowUpdated = matchedChangeSetItem + let rowUpdated = matchedChangeSetItem ? getRowFromItem(row, matchedChangeSetItem) : this.compiledMacroFunc ? { ...row } @@ -105,18 +106,32 @@ export default class ChangeSetGrider extends Grider { }; if (this.compiledMacroFunc) { - for (const cell of this.selectedCells) { - if (cell.row != index) continue; - const newValue = runMacroOnValue( - this.compiledMacroFunc, - this.macroArgs, - rowUpdated[cell.column], - index, - rowUpdated, - cell.column, - this._errors - ); - rowUpdated[cell.column] = newValue; + if (this.macro?.type == 'transformValue') { + for (const cell of this.selectedCells) { + if (cell.row != index) continue; + const newValue = runMacroOnValue( + this.compiledMacroFunc, + this.macroArgs, + rowUpdated[cell.column], + index, + rowUpdated, + cell.column, + this._errors + ); + rowUpdated[cell.column] = newValue; + } + } + if (this.macro?.type == 'transformRow') { + if (this.selectedCells.find(x => x.row == index)) { + rowUpdated = runMacroOnRow( + this.compiledMacroFunc, + this.macroArgs, + index, + rowUpdated, + _.uniq(this.selectedCells.map(x => x.column)), + this._errors + ); + } } } diff --git a/packages/web/src/datagrid/ColumnManager.svelte b/packages/web/src/datagrid/ColumnManager.svelte index 896295da2..b404084ce 100644 --- a/packages/web/src/datagrid/ColumnManager.svelte +++ b/packages/web/src/datagrid/ColumnManager.svelte @@ -1,6 +1,6 @@ +{#if allowChangeChangeSetStructure} +
+ { + dispatchChangeSet({ + type: 'set', + value: { + ...changeSetState?.value, + structure: { + ...display?.editableStructure, + __isDynamicStructure: e.detail == 'variable', + // __keepDynamicStreamHeader: true, + }, + }, + }); + }} + /> +
+{/if} @@ -122,6 +187,9 @@ }}>Add {/if} + {#if allowChangeChangeSetStructure && !isDynamicStructure} + Add + {/if} display.hideAllColumns()}>Hide display.showAllColumns()}>Show @@ -139,12 +207,19 @@ /> {#each items as column (column.uniqueName)} + {@const columnIndex = items.indexOf(column)} { if (domFocusField) domFocusField.focus(); @@ -198,4 +273,14 @@ left: -1000px; top: -1000px; } + + .selectwrap :global(select) { + flex: 1; + padding: 3px 0px; + border: none; + } + + .selectwrap { + border-bottom: 1px solid var(--theme-border); + } diff --git a/packages/web/src/datagrid/ColumnManagerRow.svelte b/packages/web/src/datagrid/ColumnManagerRow.svelte index 06ee30bcf..5ec95caf2 100644 --- a/packages/web/src/datagrid/ColumnManagerRow.svelte +++ b/packages/web/src/datagrid/ColumnManagerRow.svelte @@ -4,6 +4,9 @@ import FontIcon from '../icons/FontIcon.svelte'; import ColumnLabel from '../elements/ColumnLabel.svelte'; import { createEventDispatcher } from 'svelte'; + import { showModal } from '../modals/modalTools'; + import ColumnEditorModal from '../tableeditor/ColumnEditorModal.svelte'; + import { editorDeleteColumn } from 'dbgate-tools'; export let column; export let display; @@ -11,6 +14,28 @@ export let isSelected = false; export let conid; export let database; + export let isDynamicStructure; + + export let tableInfo; + export let setTableInfo; + + export let columnInfo = null; + export let columnIndex = -1; + + export let allowChangeChangeSetStructure = false; + $: addDataCommand = allowChangeChangeSetStructure; + + function handleEditColumn() { + showModal(ColumnEditorModal, { columnInfo, tableInfo, setTableInfo, addDataCommand }); + } + + function exchange(array, i1, i2) { + const i1r = (i1 + array.length) % array.length; + const i2r = (i2 + array.length) % array.length; + const res = [...array]; + [res[i1r], res[i2r]] = [res[i2r], res[i1r]]; + return res; + } const dispatch = createEventDispatcher(); @@ -29,32 +54,59 @@ on:mousemove on:mouseup > - - display.toggleExpandedColumn(column.uniqueName)} - /> - - {#if isJsonView} - - {:else} - { - e.stopPropagation(); - }} - on:mousedown={e => { - e.stopPropagation(); - }} - on:change={() => { - const newValue = !column.isChecked; - display.setColumnVisibility(column.uniquePath, newValue); - dispatch('setvisibility', newValue); - }} - /> +
+ + display.toggleExpandedColumn(column.uniqueName)} + /> + + {#if isJsonView} + + {:else} + { + e.stopPropagation(); + }} + on:mousedown={e => { + e.stopPropagation(); + }} + on:change={() => { + const newValue = !column.isChecked; + display.setColumnVisibility(column.uniquePath, newValue); + dispatch('setvisibility', newValue); + }} + /> + {/if} + +
+ + {#if allowChangeChangeSetStructure && !isDynamicStructure} +
+ + + + setTableInfo(info => editorDeleteColumn(info, columnInfo, addDataCommand))}> + + + + setTableInfo(info => ({ ...info, columns: exchange(info.columns, columnIndex, columnIndex - 1) }))} + > + + + + setTableInfo(info => ({ ...info, columns: exchange(info.columns, columnIndex, columnIndex + 1) }))} + > + + +
{/if} - diff --git a/packages/web/src/datagrid/DataGrid.svelte b/packages/web/src/datagrid/DataGrid.svelte index 6a1d851ad..1d6c05e2f 100644 --- a/packages/web/src/datagrid/DataGrid.svelte +++ b/packages/web/src/datagrid/DataGrid.svelte @@ -57,13 +57,12 @@ import HorizontalSplitter from '../elements/HorizontalSplitter.svelte'; import VerticalSplitter from '../elements/VerticalSplitter.svelte'; import FormViewFilters from '../formview/FormViewFilters.svelte'; - import MacroDetail from '../freetable/MacroDetail.svelte'; - import MacroManager from '../freetable/MacroManager.svelte'; + import MacroDetail from '../macro/MacroDetail.svelte'; + import MacroManager from '../macro/MacroManager.svelte'; import WidgetColumnBar from '../widgets/WidgetColumnBar.svelte'; import WidgetColumnBarItem from '../widgets/WidgetColumnBarItem.svelte'; import ColumnManager from './ColumnManager.svelte'; import ReferenceManager from './ReferenceManager.svelte'; - import FreeTableColumnEditor from '../freetable/FreeTableColumnEditor.svelte'; import JsonViewFilters from '../jsonview/JsonViewFilters.svelte'; import createActivator, { getActiveComponent } from '../utility/createActivator'; import _ from 'lodash'; @@ -201,7 +200,6 @@ height="40%" show={freeTableColumn && !isDynamicStructure} > - diff --git a/packages/web/src/datagrid/DataGridCell.svelte b/packages/web/src/datagrid/DataGridCell.svelte index b785de0ac..f22b394c2 100644 --- a/packages/web/src/datagrid/DataGridCell.svelte +++ b/packages/web/src/datagrid/DataGridCell.svelte @@ -7,6 +7,7 @@ import CellValue from './CellValue.svelte'; import { showModal } from '../modals/modalTools'; import EditCellDataModal from '../modals/EditCellDataModal.svelte'; + import { openJsonLinesData } from '../utility/openJsonLinesData'; export let rowIndex; export let col; @@ -96,20 +97,7 @@ icon="icon open-in-new" on:click={() => { if (_.every(jsonParsedValue || value, x => _.isPlainObject(x))) { - openNewTab( - { - title: 'Data #', - icon: 'img free-table', - tabComponent: 'FreeTableTab', - props: {}, - }, - { - editor: { - rows: jsonParsedValue || value, - structure: { __isDynamicStructure: true, columns: [] }, - }, - } - ); + openJsonLinesData(jsonParsedValue || value); } else { openJsonDocument(jsonParsedValue || value, undefined, true); } diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index bd016e309..8f611a233 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -161,7 +161,7 @@ registerCommand({ id: 'dataGrid.openJsonArrayInSheet', category: 'Data grid', - name: 'Open array as data sheet', + name: 'Open array as table', testEnabled: () => getCurrentDataGrid()?.openJsonArrayInSheetEnabled(), onClick: () => getCurrentDataGrid().openJsonArrayInSheet(), }); @@ -238,7 +238,7 @@ registerCommand({ id: 'dataGrid.openFreeTable', category: 'Data grid', - name: 'Edit selection as data sheet', + name: 'Edit selection as table', testEnabled: () => getCurrentDataGrid() != null, onClick: () => getCurrentDataGrid().openFreeTable(), }); @@ -398,6 +398,7 @@ import EditCellDataModal, { shouldOpenMultilineDialog } from '../modals/EditCellDataModal.svelte'; import { getDatabaseInfo, useDatabaseStatus } from '../utility/metadataLoaders'; import { showSnackbarSuccess } from '../utility/snackbar'; + import { openJsonLinesData } from '../utility/openJsonLinesData'; export let onLoadNextData = undefined; export let grider = undefined; @@ -645,15 +646,7 @@ } export function openFreeTable() { - openNewTab( - { - title: 'Data #', - icon: 'img free-table', - tabComponent: 'FreeTableTab', - props: {}, - }, - { editor: getSelectedFreeData() } - ); + openJsonLinesData(getSelectedFreeDataRows()); } export function openChartFromSelection() { @@ -784,20 +777,7 @@ } export function openJsonArrayInSheet() { - openNewTab( - { - title: 'Data #', - icon: 'img free-table', - tabComponent: 'FreeTableTab', - props: {}, - }, - { - editor: { - rows: getSelectedDataJson(true), - structure: { __isDynamicStructure: true, columns: [] }, - }, - } - ); + openJsonLinesData(getSelectedDataJson(true)); } export function editJsonEnabled() { @@ -1109,6 +1089,12 @@ }; }; + const getSelectedFreeDataRows = () => { + const columns = getSelectedColumns(); + const rows = getSelectedRowData().map(row => _.pickBy(row, (v, col) => columns.find(x => x.columnName == col))); + return rows; + }; + function getCellsPublished(cells) { const regular = cellsToRegularCells(cells); const res = regular diff --git a/packages/web/src/datagrid/JslDataGrid.svelte b/packages/web/src/datagrid/JslDataGrid.svelte index 6ae1a2a31..e9f34fe9f 100644 --- a/packages/web/src/datagrid/JslDataGrid.svelte +++ b/packages/web/src/datagrid/JslDataGrid.svelte @@ -1,5 +1,6 @@ { + on:click={e => { if (onClick) onClick(e); else openWebLink(href); }} + use:contextMenu={menu} > diff --git a/packages/web/src/freetable/ColumnManagerRow.svelte b/packages/web/src/freetable/ColumnManagerRow.svelte deleted file mode 100644 index c255bd588..000000000 --- a/packages/web/src/freetable/ColumnManagerRow.svelte +++ /dev/null @@ -1,50 +0,0 @@ - - -
-
{column.columnName}
-
- - - - - - - - - - - - -
-
- - diff --git a/packages/web/src/freetable/ColumnNameEditor.svelte b/packages/web/src/freetable/ColumnNameEditor.svelte deleted file mode 100644 index 0cb5975f7..000000000 --- a/packages/web/src/freetable/ColumnNameEditor.svelte +++ /dev/null @@ -1,55 +0,0 @@ - - - - - diff --git a/packages/web/src/freetable/FreeTableColumnEditor.svelte b/packages/web/src/freetable/FreeTableColumnEditor.svelte deleted file mode 100644 index fece23122..000000000 --- a/packages/web/src/freetable/FreeTableColumnEditor.svelte +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - {#each structure?.columns || [] as column, index} - {#if index == editingColumn} - { - dispatchChangeColumns( - $$props, - cols => cols.map((col, i) => (index == i ? { columnName } : col)), - row => _.mapKeys(row, (v, k) => (k == column.columnName ? columnName : k)) - ); - }} - onBlur={() => (editingColumn = null)} - focusOnCreate - blurOnEnter - existingNames={structure?.columns.map(x => x.columnName)} - /> - {:else} - (editingColumn = index)} - onRemove={() => { - dispatchChangeColumns($$props, cols => cols.filter((c, i) => i != index)); - }} - onUp={() => { - dispatchChangeColumns($$props, cols => exchange(cols, index, index - 1)); - }} - onDown={() => { - dispatchChangeColumns($$props, cols => exchange(cols, index, index + 1)); - }} - /> - {/if} - {/each} - { - dispatchChangeColumns($$props, cols => [...cols, { columnName }]); - }} - placeholder="New column" - existingNames={(structure?.columns || []).map(x => x.columnName)} - /> - diff --git a/packages/web/src/freetable/FreeTableGridCore.svelte b/packages/web/src/freetable/FreeTableGridCore.svelte deleted file mode 100644 index c791b16f6..000000000 --- a/packages/web/src/freetable/FreeTableGridCore.svelte +++ /dev/null @@ -1,84 +0,0 @@ - - - - - diff --git a/packages/web/src/freetable/FreeTableGrider.ts b/packages/web/src/freetable/FreeTableGrider.ts deleted file mode 100644 index 3d7e62b11..000000000 --- a/packages/web/src/freetable/FreeTableGrider.ts +++ /dev/null @@ -1,104 +0,0 @@ -import type { FreeTableModel } from 'dbgate-datalib'; -import Grider from '../datagrid/Grider'; - -export default class FreeTableGrider extends Grider { - public model: FreeTableModel; - private batchModel: FreeTableModel; - - constructor(public modelState, public dispatchModel) { - super(); - this.model = modelState && modelState.value; - } - getRowData(index: any) { - return this.model.rows?.[index]; - } - get rowCount() { - return this.model.rows?.length; - } - get currentModel(): FreeTableModel { - return this.batchModel || this.model; - } - set currentModel(value) { - if (this.batchModel) this.batchModel = value; - else this.dispatchModel({ type: 'set', value }); - } - setCellValue(index: number, uniqueName: string, value: any) { - const model = this.currentModel; - if (model.rows[index]) { - this.currentModel = { - ...model, - rows: model.rows.map((row, i) => (index == i ? { ...row, [uniqueName]: value } : row)), - }; - } - } - setRowData(index: number, document: any) { - const model = this.currentModel; - if (model.rows[index]) { - this.currentModel = { - ...model, - rows: model.rows.map((row, i) => (index == i ? document : row)), - }; - } - } - get editable() { - return true; - } - get canInsert() { - return true; - } - get allowSave() { - return true; - } - insertRow(): number { - const model = this.currentModel; - this.currentModel = { - ...model, - rows: [...model.rows, {}], - }; - return this.currentModel.rows.length - 1; - } - insertDocuments(documents: any[]): number { - const model = this.currentModel; - this.currentModel = { - ...model, - rows: [...model.rows, ...documents], - }; - return this.currentModel.rows.length - documents.length; - } - - deleteRow(index: number) { - const model = this.currentModel; - this.currentModel = { - ...model, - rows: model.rows.filter((row, i) => index != i), - }; - } - beginUpdate() { - this.batchModel = this.model; - } - endUpdate() { - if (this.model != this.batchModel) { - this.dispatchModel({ type: 'set', value: this.batchModel }); - this.batchModel = null; - } - } - - // static factory({ modelState, dispatchModel }): FreeTableGrider { - // return new FreeTableGrider(modelState, dispatchModel); - // } - // static factoryDeps({ modelState, dispatchModel }) { - // return [modelState, dispatchModel]; - // } - undo() { - this.dispatchModel({ type: 'undo' }); - } - redo() { - this.dispatchModel({ type: 'redo' }); - } - get canUndo() { - return this.modelState.canUndo; - } - get canRedo() { - return this.modelState.canRedo; - } -} diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index 189098a5a..27eb71de3 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -228,6 +228,8 @@ 'img keydb': 'mdi mdi-key color-icon-blue', 'img duplicator': 'mdi mdi-content-duplicate color-icon-green', + 'img import': 'mdi mdi-database-import color-icon-green', + 'img export': 'mdi mdi-database-export color-icon-green', }; diff --git a/packages/web/src/impexp/ImportExportConfigurator.svelte b/packages/web/src/impexp/ImportExportConfigurator.svelte index 982253d67..daa58a338 100644 --- a/packages/web/src/impexp/ImportExportConfigurator.svelte +++ b/packages/web/src/impexp/ImportExportConfigurator.svelte @@ -22,7 +22,7 @@ for (const file of getAsArray(files)) { const format = findFileFormat(extensions, storage); if (format) { - await (format.addFileToSourceList || addFileToSourceListDefault)(file, newSources, newValues); + await (format.addFileToSourceList || addFileToSourceListDefault)(file, newSources, newValues, apiCall); } } newValues['sourceList'] = [...(values.sourceList || []).filter(x => !newSources.includes(x)), ...newSources]; @@ -58,6 +58,7 @@ import { showModal } from '../modals/modalTools'; import { findFileFormat } from '../plugins/fileformats'; import { extensions } from '../stores'; + import { apiCall } from '../utility/api'; import getAsArray from '../utility/getAsArray'; import { useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders'; import { setUploadListener } from '../utility/uploadFiles'; diff --git a/packages/web/src/jsontree/JSONTree.svelte b/packages/web/src/jsontree/JSONTree.svelte index 1ffe37588..06b7a74c6 100644 --- a/packages/web/src/jsontree/JSONTree.svelte +++ b/packages/web/src/jsontree/JSONTree.svelte @@ -5,6 +5,7 @@ import openNewTab from '../utility/openNewTab'; import _ from 'lodash'; import { copyTextToClipboard } from '../utility/clipboard'; + import { openJsonLinesData } from '../utility/openJsonLinesData'; setContext('json-tree-context-key', {}); @@ -49,22 +50,9 @@ if (value && _.isArray(value)) { res.push({ - text: 'Open as data sheet', + text: 'Open as table', onClick: () => { - openNewTab( - { - title: 'Data #', - icon: 'img free-table', - tabComponent: 'FreeTableTab', - props: {}, - }, - { - editor: { - rows: value, - structure: { __isDynamicStructure: true, columns: [] }, - }, - } - ); + openJsonLinesData(value); }, }); } diff --git a/packages/web/src/freetable/MacroDetail.svelte b/packages/web/src/macro/MacroDetail.svelte similarity index 100% rename from packages/web/src/freetable/MacroDetail.svelte rename to packages/web/src/macro/MacroDetail.svelte diff --git a/packages/web/src/freetable/MacroHeader.svelte b/packages/web/src/macro/MacroHeader.svelte similarity index 100% rename from packages/web/src/freetable/MacroHeader.svelte rename to packages/web/src/macro/MacroHeader.svelte diff --git a/packages/web/src/freetable/MacroInfoTab.svelte b/packages/web/src/macro/MacroInfoTab.svelte similarity index 100% rename from packages/web/src/freetable/MacroInfoTab.svelte rename to packages/web/src/macro/MacroInfoTab.svelte diff --git a/packages/web/src/freetable/MacroManager.svelte b/packages/web/src/macro/MacroManager.svelte similarity index 100% rename from packages/web/src/freetable/MacroManager.svelte rename to packages/web/src/macro/MacroManager.svelte diff --git a/packages/web/src/freetable/MacroParameters.svelte b/packages/web/src/macro/MacroParameters.svelte similarity index 100% rename from packages/web/src/freetable/MacroParameters.svelte rename to packages/web/src/macro/MacroParameters.svelte diff --git a/packages/web/src/freetable/MacroPreviewGrider.ts b/packages/web/src/macro/MacroPreviewGrider.ts similarity index 100% rename from packages/web/src/freetable/MacroPreviewGrider.ts rename to packages/web/src/macro/MacroPreviewGrider.ts diff --git a/packages/web/src/freetable/macros.js b/packages/web/src/macro/macros.js similarity index 66% rename from packages/web/src/freetable/macros.js rename to packages/web/src/macro/macros.js index 0cee83a8c..f131e3b6c 100644 --- a/packages/web/src/freetable/macros.js +++ b/packages/web/src/macro/macros.js @@ -203,65 +203,17 @@ return !!value; ], code: `return modules.moment().format(args.format)`, }, - { - title: 'Duplicate rows', - name: 'duplicateRows', - group: 'Tools', - description: 'Duplicate selected rows', - type: 'transformRows', - code: ` -const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row)); -const selectedRows = modules.lodash.groupBy(selectedCells, 'row'); -const maxIndex = modules.lodash.max(selectedRowIndexes); -return [ - ...rows.slice(0, maxIndex + 1), - ...selectedRowIndexes.map(index => ({ - ...modules.lodash.pick(rows[index], selectedRows[index].map(x => x.column)), - __rowStatus: 'inserted', - })), - ...rows.slice(maxIndex + 1), -] - `, - }, - { - title: 'Delete empty rows', - name: 'deleteEmptyRows', - group: 'Tools', - description: 'Delete empty rows - rows with all values null or empty string', - type: 'transformRows', - code: ` -return rows.map(row => { - if (cols.find(col => row[col])) return row; - return { - ...row, - __rowStatus: 'deleted', - }; -}) -`, - }, { title: 'Duplicate columns', name: 'duplicateColumns', group: 'Tools', description: 'Duplicate selected columns', - type: 'transformData', + type: 'transformRow', code: ` -const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column)); -const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row)); -const addedColumnNames = selectedColumnNames.map(col => (args.prefix || '') + col + (args.postfix || '')); -const resultRows = rows.map((row, rowIndex) => ({ - ...row, - ...(selectedRowIndexes.includes(rowIndex) ? modules.lodash.fromPairs(selectedColumnNames.map(col => [(args.prefix || '') + col + (args.postfix || ''), row[col]])) : {}), - __insertedFields: addedColumnNames, -})); -const resultCols = [ - ...cols, - ...addedColumnNames, -]; -return { - rows: resultRows, - cols: resultCols, -} + return { + ...row, + ...modules.lodash.fromPairs(columns.map(col=>[(args.prefix || '') + col + (args.postfix || ''), row[col]])) + } `, args: [ { @@ -282,42 +234,27 @@ return { name: 'splitColumns', group: 'Tools', description: 'Split selected columns', - type: 'transformData', + type: 'transformRow', code: ` -const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column)); -const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row)); - -const addedColumnNames = new Set(); - -const resultRows = modules.lodash.cloneDeep(rows); -resultRows.forEach((row, rowIndex) => { - for(const cell of selectedCells) { - if (cell.row == rowIndex && modules.lodash.isString(cell.value)) { - const splitted = cell.value.split(args.delimiter); - splitted.forEach((value, valueIndex) => { - const name = cell.column + '_' + (valueIndex + 1).toString(); - row[name] = value; - addedColumnNames.add(name); - }); - } - } -}); - -const resultCols = [ - ...cols, - ...addedColumnNames, -]; -return { - rows: resultRows, - cols: resultCols, -} + const res = {...row}; + for(const col of columns) { + const value = row[col]; + if (modules.lodash.isString(value)) { + const splitted = value.split(args.delimiter); + splitted.forEach((splitValue, valueIndex) => { + const name = col + '_' + (valueIndex + 1).toString(); + res[name] = splitValue; + }); + } + } + return res; `, args: [ { type: 'text', label: 'Delimiter', name: 'delimiter', - default: ',' + default: ',', }, ], }, @@ -342,53 +279,34 @@ return { name: 'extractDateFields', group: 'Tools', description: 'Extract yaear, month, day and other date/time fields from selection and adds it as new columns', - type: 'transformData', + type: 'transformRow', code: ` -const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column)); -const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row)); -const addedColumnNames = modules.lodash.compact([args.year, args.month, args.day, args.hour, args.minute, args.second]); -const selectedRows = modules.lodash.groupBy(selectedCells, 'row'); -const resultRows = rows.map((row, rowIndex) => { - if (!selectedRowIndexes.includes(rowIndex)) return { - ...row, - __insertedFields: addedColumnNames, - }; - let mom = null; - for(const cell of selectedRows[rowIndex]) { - const m = modules.moment(row[cell.column]); - if (m.isValid()) { - mom = m; - break; - } - } - if (!mom) return { - ...row, - __insertedFields: addedColumnNames, - }; + let mom = null; + for(const col of columns) { + const m = modules.moment(row[col]); + if (m.isValid()) { + mom = m; + break; + } + } - const fields = { - [args.year]: mom.year(), - [args.month]: mom.month() + 1, - [args.day]: mom.day(), - [args.hour]: mom.hour(), - [args.minute]: mom.minute(), - [args.second]: mom.second(), - }; + if (!mom) return row; - return { - ...row, - ...modules.lodash.pick(fields, addedColumnNames), - __insertedFields: addedColumnNames, - } -}); -const resultCols = [ - ...cols, - ...addedColumnNames, -]; -return { - rows: resultRows, - cols: resultCols, -} + const addedColumnNames = modules.lodash.compact([args.year, args.month, args.day, args.hour, args.minute, args.second]); + + const fields = { + [args.year]: mom.year(), + [args.month]: mom.month() + 1, + [args.day]: mom.day(), + [args.hour]: mom.hour(), + [args.minute]: mom.minute(), + [args.second]: mom.second(), + }; + + return { + ...row, + ...modules.lodash.pick(fields, addedColumnNames), + }; `, args: [ { diff --git a/packages/web/src/plugins/PluginsProvider.svelte b/packages/web/src/plugins/PluginsProvider.svelte index 28580b065..42a10bced 100644 --- a/packages/web/src/plugins/PluginsProvider.svelte +++ b/packages/web/src/plugins/PluginsProvider.svelte @@ -1,8 +1,4 @@ @@ -31,7 +33,10 @@ - + {#if driver?.dialect?.columnProperties?.isUnsigned} @@ -52,9 +57,9 @@ on:click={e => { closeCurrentModal(); if (columnInfo) { - setTableInfo(tbl => editorModifyColumn(tbl, e.detail)); + setTableInfo(tbl => editorModifyColumn(tbl, e.detail, addDataCommand)); } else { - setTableInfo(tbl => editorAddColumn(tbl, e.detail)); + setTableInfo(tbl => editorAddColumn(tbl, e.detail, addDataCommand)); if (onAddNext) onAddNext(); } }} @@ -65,7 +70,7 @@ value="Save" on:click={e => { closeCurrentModal(); - setTableInfo(tbl => editorAddColumn(tbl, e.detail)); + setTableInfo(tbl => editorAddColumn(tbl, e.detail, addDataCommand)); }} /> {/if} @@ -77,7 +82,7 @@ value="Remove" on:click={() => { closeCurrentModal(); - setTableInfo(tbl => editorDeleteColumn(tbl, columnInfo)); + setTableInfo(tbl => editorDeleteColumn(tbl, columnInfo, addDataCommand)); }} /> {/if} diff --git a/packages/web/src/tabs/ArchiveFileTab.svelte b/packages/web/src/tabs/ArchiveFileTab.svelte index 868404c26..1756d7484 100644 --- a/packages/web/src/tabs/ArchiveFileTab.svelte +++ b/packages/web/src/tabs/ArchiveFileTab.svelte @@ -14,13 +14,25 @@ testEnabled: () => getCurrentEditor()?.canSave(), onClick: () => getCurrentEditor().save(), }); + + registerCommand({ + id: 'archiveFile.saveAs', + category: 'Archive file', + name: 'Save as', + icon: 'icon save', + isRelatedToTab: true, + testEnabled: () => getCurrentEditor() != null, + onClick: () => getCurrentEditor().saveAs(), + }); - + {#if jslidChecked || !jslid} + + {/if} + diff --git a/packages/web/src/tabs/CollectionDataTab.svelte b/packages/web/src/tabs/CollectionDataTab.svelte index d4e606f09..113ea1442 100644 --- a/packages/web/src/tabs/CollectionDataTab.svelte +++ b/packages/web/src/tabs/CollectionDataTab.svelte @@ -154,7 +154,7 @@ } function handleRunMacro(macro, params, cells) { - const newChangeSet = runMacroOnChangeSet(macro, params, cells, $changeSetStore?.value, display); + const newChangeSet = runMacroOnChangeSet(macro, params, cells, $changeSetStore?.value, display, false); if (newChangeSet) { dispatchChangeSet({ type: 'set', value: newChangeSet }); } diff --git a/packages/web/src/tabs/DataDuplicatorTab.svelte b/packages/web/src/tabs/DataDuplicatorTab.svelte index 9e7390c29..f3b676224 100644 --- a/packages/web/src/tabs/DataDuplicatorTab.svelte +++ b/packages/web/src/tabs/DataDuplicatorTab.svelte @@ -12,6 +12,16 @@ testEnabled: () => getCurrentEditor()?.canRun(), onClick: () => getCurrentEditor().run(), }); + registerCommand({ + id: 'dataDuplicator.kill', + category: 'Data duplicator', + icon: 'icon close', + name: 'Kill', + toolbar: true, + isRelatedToTab: true, + testEnabled: () => getCurrentEditor()?.canKill(), + onClick: () => getCurrentEditor().kill(), + }); - +
-
Source archive
- { - setEditorData(old => ({ - ...old, - archiveFolder: e.detail, - })); + + { + setEditorData(old => ({ + ...old, + archiveFolder: e.detail, + })); + }} + options={$archiveFolders?.map(x => ({ + label: x.name, + value: x.name, + })) || []} + /> + + + { + setEditorData(old => ({ + ...old, + rollbackAfterFinish: !$editorState.value?.rollbackAfterFinish, + })); + }, }} - options={$archiveFolders?.map(x => ({ - label: x.name, - value: x.name, - })) || []} - /> + > + { + setEditorData(old => ({ + ...old, + rollbackAfterFinish: e.target.checked, + })); + }} + /> + + + { + setEditorData(old => ({ + ...old, + skipRowsWithUnresolvedRefs: !$editorState.value?.skipRowsWithUnresolvedRefs, + })); + }, + }} + > + { + setEditorData(old => ({ + ...old, + skipRowsWithUnresolvedRefs: e.target.checked, + })); + }} + /> + +
+ Check all + | + Uncheck all +
+ {/if} + + { + openNewTab({ + title: row.file, + icon: 'img archive', + tooltip: `${$editorState.value?.archiveFolder}\n${row.file}`, + tabComponent: 'ArchiveFileTab', + props: { + archiveFile: row.file, + archiveFolder: $editorState.value?.archiveFolder, + }, + }); + }}> {row.file} + + + { + openNewTab({ + title: row.pureName, + icon: 'img table', + tabComponent: 'TableDataTab', + props: { + schemaName: row.schemaName, + pureName: row.pureName, + conid, + database, + objectTypeField: 'tables', + }, + }); + }}> {row.table} +
@@ -247,6 +392,7 @@ + diff --git a/packages/web/src/tabs/FreeTableTab.svelte b/packages/web/src/tabs/FreeTableTab.svelte deleted file mode 100644 index 55b660b72..000000000 --- a/packages/web/src/tabs/FreeTableTab.svelte +++ /dev/null @@ -1,169 +0,0 @@ - - - - -{#if isLoading} - -{:else if errorMessage} - -{:else} - - - - - - - -{/if} diff --git a/packages/web/src/tabs/ImportExportTab.svelte b/packages/web/src/tabs/ImportExportTab.svelte new file mode 100644 index 000000000..2e065d33f --- /dev/null +++ b/packages/web/src/tabs/ImportExportTab.svelte @@ -0,0 +1,197 @@ + + + + +
+ + + {#if busy} + + {/if} +
+ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/web/src/tabs/index.js b/packages/web/src/tabs/index.js index 59243e4e4..b828fdb7e 100644 --- a/packages/web/src/tabs/index.js +++ b/packages/web/src/tabs/index.js @@ -5,7 +5,6 @@ import * as TableStructureTab from './TableStructureTab.svelte'; import * as QueryTab from './QueryTab.svelte'; import * as ShellTab from './ShellTab.svelte'; import * as ArchiveFileTab from './ArchiveFileTab.svelte'; -import * as FreeTableTab from './FreeTableTab.svelte'; import * as PluginTab from './PluginTab.svelte'; import * as ChartTab from './ChartTab.svelte'; import * as MarkdownEditorTab from './MarkdownEditorTab.svelte'; @@ -29,6 +28,7 @@ import * as PerspectiveTab from './PerspectiveTab.svelte'; import * as ServerSummaryTab from './ServerSummaryTab.svelte'; import * as ProfilerTab from './ProfilerTab.svelte'; import * as DataDuplicatorTab from './DataDuplicatorTab.svelte'; +import * as ImportExportTab from './ImportExportTab.svelte'; export default { TableDataTab, @@ -38,7 +38,6 @@ export default { QueryTab, ShellTab, ArchiveFileTab, - FreeTableTab, PluginTab, ChartTab, MarkdownEditorTab, @@ -62,4 +61,5 @@ export default { ServerSummaryTab, ProfilerTab, DataDuplicatorTab, + ImportExportTab, }; diff --git a/packages/web/src/utility/archiveTools.ts b/packages/web/src/utility/archiveTools.ts deleted file mode 100644 index a9a059439..000000000 --- a/packages/web/src/utility/archiveTools.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { archiveFilesAsDataSheets } from '../stores'; - -export function markArchiveFileAsDataSheet(folder, file) { - archiveFilesAsDataSheets.update(ar => - ar.find(x => x.folder == folder && x.file == file) ? ar : [...ar, { folder, file }] - ); -} - -export function markArchiveFileAsReadonly(folder, file) { - archiveFilesAsDataSheets.update(ar => ar.filter(x => x.folder != folder || x.file != file)); -} - -export function isArchiveFileMarkedAsDataSheet(store, folder, file) { - return !!store.find(x => x.folder == folder && x.file == file); -} diff --git a/packages/web/src/utility/createQuickExportMenu.ts b/packages/web/src/utility/createQuickExportMenu.ts index 2dc4185fc..69fa70788 100644 --- a/packages/web/src/utility/createQuickExportMenu.ts +++ b/packages/web/src/utility/createQuickExportMenu.ts @@ -1,5 +1,5 @@ import type { QuickExportDefinition } from 'dbgate-types'; -import { getExtensions } from '../stores'; +import { currentArchive, getCurrentArchive, getExtensions } from '../stores'; export function createQuickExportMenuItems(handler: (fmt: QuickExportDefinition) => Function, advancedExportMenuItem) { const extensions = getExtensions(); @@ -9,6 +9,22 @@ export function createQuickExportMenuItems(handler: (fmt: QuickExportDefinition) onClick: handler(fmt), })), { divider: true }, + { + text: 'Current archive', + onClick: handler({ + extension: 'jsonl', + label: 'Current archive', + noFilenameDependency: true, + createWriter: (fileName, dataName) => ({ + functionName: 'archiveWriter', + props: { + fileName: dataName, + folderName: getCurrentArchive(), + }, + }), + }), + }, + { divider: true }, { text: 'More...', ...advancedExportMenuItem, diff --git a/packages/web/src/utility/exportFileTools.ts b/packages/web/src/utility/exportFileTools.ts index f330bc267..4916baddd 100644 --- a/packages/web/src/utility/exportFileTools.ts +++ b/packages/web/src/utility/exportFileTools.ts @@ -7,6 +7,7 @@ import { normalizeExportColumnMap } from '../impexp/createImpExpScript'; import { getCurrentConfig } from '../stores'; import { showModal } from '../modals/modalTools'; import RunScriptModal from '../modals/RunScriptModal.svelte'; +import { QuickExportDefinition } from 'dbgate-types'; export async function importSqlDump(inputFile, connection) { const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson(); @@ -117,35 +118,53 @@ export async function saveExportedFile(filters, defaultPath, extension, dataName }); } -export async function exportQuickExportFile(dataName, reader, format, columnMap = null) { - await saveExportedFile( - [{ name: format.label, extensions: [format.extension] }], - `${dataName}.${format.extension}`, - format.extension, - dataName, - filePath => { - const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson(); +function generateQuickExportScript( + reader, + format: QuickExportDefinition, + filePath: string, + dataName: string, + columnMap +) { + const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson(); - const sourceVar = script.allocVariable(); - script.assign(sourceVar, reader.functionName, reader.props); + const sourceVar = script.allocVariable(); + script.assign(sourceVar, reader.functionName, reader.props); - const targetVar = script.allocVariable(); - const writer = format.createWriter(filePath, dataName); - script.assign(targetVar, writer.functionName, writer.props); + const targetVar = script.allocVariable(); + const writer = format.createWriter(filePath, dataName); + script.assign(targetVar, writer.functionName, writer.props); - const colmap = normalizeExportColumnMap(columnMap); - let colmapVar = null; - if (colmap) { - colmapVar = script.allocVariable(); - script.assignValue(colmapVar, colmap); - } + const colmap = normalizeExportColumnMap(columnMap); + let colmapVar = null; + if (colmap) { + colmapVar = script.allocVariable(); + script.assignValue(colmapVar, colmap); + } - script.copyStream(sourceVar, targetVar, colmapVar); - script.endLine(); + script.copyStream(sourceVar, targetVar, colmapVar); + script.endLine(); - return script.getScript(); - } - ); + return script.getScript(); +} + +export async function exportQuickExportFile(dataName, reader, format: QuickExportDefinition, columnMap = null) { + if (format.noFilenameDependency) { + const script = generateQuickExportScript(reader, format, null, dataName, columnMap); + runImportExportScript({ + script, + runningMessage: `Exporting ${dataName}`, + canceledMessage: `Export ${dataName} canceled`, + finishedMessage: `Export ${dataName} finished`, + }); + } else { + await saveExportedFile( + [{ name: format.label, extensions: [format.extension] }], + `${dataName}.${format.extension}`, + format.extension, + dataName, + filePath => generateQuickExportScript(reader, format, filePath, dataName, columnMap) + ); + } } // export async function exportSqlDump(connection, databaseName) { diff --git a/packages/web/src/utility/openJsonLinesData.ts b/packages/web/src/utility/openJsonLinesData.ts new file mode 100644 index 000000000..27c0b53cb --- /dev/null +++ b/packages/web/src/utility/openJsonLinesData.ts @@ -0,0 +1,22 @@ +import uuidv1 from 'uuid/v1'; +import { apiCall } from './api'; +import openNewTab from './openNewTab'; + +export async function openJsonLinesData(rows) { + const jslid = uuidv1(); + + // await apiCall('jsldata/save-rows', { jslid, rows }); + openNewTab( + { + tabComponent: 'ArchiveFileTab', + icon: 'img archive', + title: 'Data #', + props: { + jslid, + }, + }, + { + rows, + } + ); +} diff --git a/packages/web/src/utility/openNewTab.ts b/packages/web/src/utility/openNewTab.ts index 1fefd8798..080557fee 100644 --- a/packages/web/src/utility/openNewTab.ts +++ b/packages/web/src/utility/openNewTab.ts @@ -65,7 +65,7 @@ export default async function openNewTab(newTab, initialData = undefined, option const tabid = uuidv1(); if (initialData) { for (const key of _.keys(initialData)) { - if (key == 'editor') { + if (key == 'editor' || key == 'rows') { await localforage.setItem(`tabdata_${key}_${tabid}`, initialData[key]); } else { localStorage.setItem(`tabdata_${key}_${tabid}`, JSON.stringify(initialData[key])); diff --git a/packages/web/src/widgets/AppFilesList.svelte b/packages/web/src/widgets/AppFilesList.svelte index 2747ba499..f1535af7e 100644 --- a/packages/web/src/widgets/AppFilesList.svelte +++ b/packages/web/src/widgets/AppFilesList.svelte @@ -33,7 +33,6 @@ import newQuery from '../query/newQuery'; import { currentApplication } from '../stores'; import { apiCall } from '../utility/api'; - import { markArchiveFileAsDataSheet } from '../utility/archiveTools'; import { useAppFiles, useArchiveFolders } from '../utility/metadataLoaders'; import openNewTab from '../utility/openNewTab'; import WidgetsInnerContainer from './WidgetsInnerContainer.svelte'; diff --git a/packages/web/src/widgets/ArchiveFilesList.svelte b/packages/web/src/widgets/ArchiveFilesList.svelte index 53ebbcc61..8c39c8a25 100644 --- a/packages/web/src/widgets/ArchiveFilesList.svelte +++ b/packages/web/src/widgets/ArchiveFilesList.svelte @@ -29,7 +29,6 @@ import { showModal } from '../modals/modalTools'; import { currentArchive } from '../stores'; import { apiCall } from '../utility/api'; - import { markArchiveFileAsDataSheet } from '../utility/archiveTools'; import { useArchiveFiles, useArchiveFolders } from '../utility/metadataLoaders'; import openNewTab from '../utility/openNewTab'; import WidgetsInnerContainer from './WidgetsInnerContainer.svelte'; @@ -43,31 +42,26 @@ apiCall('archive/refresh-files', { folder }); }; - function handleNewDataSheet() { + function handleNewJsonLines() { showModal(InputTextModal, { value: '', label: 'New file name', - header: 'Create new data sheet', + header: 'Create new JSON lines', onConfirm: async file => { - await apiCall('archive/save-free-table', { + await apiCall('archive/save-rows', { folder: $currentArchive, file, - data: createFreeTableModel(), + rows: [ + { id: 1, value: 'val1' }, + { id: 1, value: 'val2' }, + ], }); - markArchiveFileAsDataSheet($currentArchive, file); openNewTab({ title: file, - icon: 'img free-table', - tabComponent: 'FreeTableTab', + icon: 'img archive', + tabComponent: 'ArchiveFileTab', props: { - initialArgs: { - functionName: 'archiveReader', - props: { - fileName: file, - folderName: $currentArchive, - }, - }, archiveFile: file, archiveFolder: $currentArchive, }, @@ -77,7 +71,7 @@ } function createAddMenu() { - return [{ text: 'New data sheet', onClick: handleNewDataSheet }]; + return [{ text: 'New NDJSON file', onClick: handleNewJsonLines }]; } diff --git a/packages/web/src/widgets/StatusBar.svelte b/packages/web/src/widgets/StatusBar.svelte index b448245d4..39d8d7e0b 100644 --- a/packages/web/src/widgets/StatusBar.svelte +++ b/packages/web/src/widgets/StatusBar.svelte @@ -6,7 +6,14 @@ import FontIcon from '../icons/FontIcon.svelte'; - import { activeTabId, currentDatabase, currentThemeDefinition, visibleCommandPalette } from '../stores'; + import { + activeTabId, + currentArchive, + currentDatabase, + currentThemeDefinition, + selectedWidget, + visibleCommandPalette, + } from '../stores'; import getConnectionLabel from '../utility/getConnectionLabel'; import { useConnectionList, useDatabaseServerVersion, useDatabaseStatus } from '../utility/metadataLoaders'; import { findCommand } from '../commands/runCommand'; @@ -140,6 +147,18 @@ {/if} + {#if $currentArchive} +
{ + $selectedWidget = 'archive'; + }} + > + + {$currentArchive} +
+ {/if}
{#each contextItems || [] as item} diff --git a/plugins/dbgate-plugin-excel/package.json b/plugins/dbgate-plugin-excel/package.json index 7cc8a2dd4..feb6155f9 100644 --- a/plugins/dbgate-plugin-excel/package.json +++ b/plugins/dbgate-plugin-excel/package.json @@ -33,7 +33,7 @@ }, "devDependencies": { "lodash": "^4.17.21", - "xlsx": "^0.16.8", + "xlsx": "^0.18.5", "dbgate-plugin-tools": "^1.0.7", "webpack": "^4.42.0", "webpack-cli": "^3.3.11" diff --git a/plugins/dbgate-plugin-excel/src/frontend/index.js b/plugins/dbgate-plugin-excel/src/frontend/index.js index 16d401fb1..cd6fa4f86 100644 --- a/plugins/dbgate-plugin-excel/src/frontend/index.js +++ b/plugins/dbgate-plugin-excel/src/frontend/index.js @@ -1,9 +1,3 @@ -let dbgateEnv; - -function initialize(dbgateEnv) { - dbgateEnv = dbgateEnv; -} - const fileFormat = { packageName: 'dbgate-plugin-excel', // file format identifier @@ -17,8 +11,8 @@ const fileFormat = { // function name from backend, which contains writer factory, postfixed by package name writerFunc: 'writer@dbgate-plugin-excel', - addFileToSourceList: async ({ fileName }, newSources, newValues) => { - const resp = await dbgateEnv.apiCall('plugins/command', { + addFileToSourceList: async ({ fileName }, newSources, newValues, apiCall) => { + const resp = await apiCall('plugins/command', { command: 'analyse', packageName: 'dbgate-plugin-excel', args: { @@ -85,5 +79,4 @@ export default { }), }, ], - initialize, }; diff --git a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js index 3a98fefaa..20106c99c 100644 --- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js @@ -42,6 +42,7 @@ const dialect = { dropCheck: true, dropReferencesWhenDropTable: false, + requireStandaloneSelectForScopeIdentity: true, columnProperties: { columnComment: true, diff --git a/plugins/dbgate-plugin-oracle/package.json b/plugins/dbgate-plugin-oracle/package.json index 9ed66790c..8fcf8cab0 100644 --- a/plugins/dbgate-plugin-oracle/package.json +++ b/plugins/dbgate-plugin-oracle/package.json @@ -32,7 +32,7 @@ "devDependencies": { "dbgate-plugin-tools": "^1.0.8", "dbgate-query-splitter": "^4.9.0", - "dbgate-tools": "^5.1.6", + "dbgate-tools": "^5.0.0-alpha.1", "lodash": "^4.17.21", "webpack": "^4.42.0", "webpack-cli": "^3.3.11" diff --git a/plugins/dbgate-plugin-oracle/src/backend/Analyser.js b/plugins/dbgate-plugin-oracle/src/backend/Analyser.js index 45aeb777f..e5e39e8f5 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-oracle/src/backend/Analyser.js @@ -104,7 +104,7 @@ class Analyser extends DatabaseAnalyser { const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']); this.feedback({ analysingMessage: 'Finalizing DB structure' }); - const columnColumnsMapped = fkColumns.rows.map(x => ({ + const fkColumnsMapped = fkColumns.rows.map(x => ({ pureName: x.pure_name, schemaName: x.schema_name, constraintSchema: x.constraint_schema, @@ -124,6 +124,9 @@ class Analyser extends DatabaseAnalyser { columnName: x.column_name, })); + const columnGroup = col => `${col.schema_name}||${col.pure_name}`; + const columnsGrouped = _.groupBy(columns.rows, columnGroup); + const res = { tables: tables.rows.map(table => { const newTable = { @@ -134,11 +137,11 @@ class Analyser extends DatabaseAnalyser { }; return { ...newTable, - columns: columns.rows - .filter(col => col.pure_name == table.pure_name && col.schema_name == table.schema_name) - .map(col => getColumnInfo(col, newTable, geometryColumns, geographyColumns)), + columns: (columnsGrouped[columnGroup(table)] || []).map(col => + getColumnInfo(col, newTable, geometryColumns, geographyColumns) + ), primaryKey: DatabaseAnalyser.extractPrimaryKeys(newTable, pkColumnsMapped), - foreignKeys: DatabaseAnalyser.extractForeignKeys(newTable, columnColumnsMapped), + foreignKeys: DatabaseAnalyser.extractForeignKeys(newTable, fkColumnsMapped), indexes: _.uniqBy( indexes.rows.filter( idx => @@ -176,9 +179,7 @@ class Analyser extends DatabaseAnalyser { schemaName: view.schema_name, contentHash: view.hash_code, createSql: `CREATE VIEW "${view.schema_name}"."${view.pure_name}"\nAS\n${view.create_sql}`, - columns: columns.rows - .filter(col => col.pure_name == view.pure_name && col.schema_name == view.schema_name) - .map(col => getColumnInfo(col)), + columns: (columnsGrouped[columnGroup(view)] || []).map(col => getColumnInfo(col)), })), matviews: matviews ? matviews.rows.map(matview => ({ @@ -212,6 +213,9 @@ class Analyser extends DatabaseAnalyser { })), }; + // this.feedback({ analysingMessage: 'Debug sleep' }); + // await new Promise(resolve => setTimeout(resolve, 90 * 1000)); + this.feedback({ analysingMessage: null }); return res; diff --git a/plugins/dbgate-plugin-oracle/src/backend/drivers.js b/plugins/dbgate-plugin-oracle/src/backend/drivers.js index 90fc9cb46..c5ac36533 100644 --- a/plugins/dbgate-plugin-oracle/src/backend/drivers.js +++ b/plugins/dbgate-plugin-oracle/src/backend/drivers.js @@ -52,52 +52,10 @@ const drivers = driverBases.map(driverBase => ({ authType, socketPath, }) { - let options = null; - - if (engine == 'redshift@dbgate-plugin-oracle') { - let url = databaseUrl; - if (url && url.startsWith('jdbc:redshift://')) { - url = url.substring('jdbc:redshift://'.length); - } - if (user && password) { - url = `oracle://${user}:${password}@${url}`; - } else if (user) { - url = `oracle://${user}@${url}`; - } else { - url = `oracle://${url}`; - } - - options = { - connectionString: url, - }; - } else { - options = useDatabaseUrl - ? { - connectionString: databaseUrl, - } - : { - host: authType == 'socket' ? socketPath || driverBase.defaultSocketPath : server, - port: authType == 'socket' ? null : port, - user, - password, - database: database || 'oracle', - ssl, - }; - } - - // console.log('OPTIONS', options); -/* - const client = new pg.Client(options); - await client.connect(); - - if (isReadOnly) { - await this.query(client, 'SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY'); - } -*/ - client = await oracledb.getConnection( { - user : options.user, - password : options.password, - connectString : options.host + client = await oracledb.getConnection({ + user, + password, + connectString: useDatabaseUrl ? databaseUrl : port ? `${server}:${port}` : server, }); return client; }, @@ -105,28 +63,25 @@ const drivers = driverBases.map(driverBase => ({ return pool.end(); }, async query(client, sql) { - //console.log('query sql', sql); + //console.log('query sql', sql); if (sql == null) { return { rows: [], columns: [], }; } -try { + try { //console.log('sql3', sql); - const res = await client.execute(sql); - //console.log('res', res); - const columns = extractOracleColumns(res.metaData); - //console.log('columns', columns); - return { rows: (res.rows || []).map(row => zipDataRow(row, columns)), columns }; -} -catch(err) { - console.log('Error query', err, sql); -} -finally { - //console.log('finally', sql); -} - + const res = await client.execute(sql); + //console.log('res', res); + const columns = extractOracleColumns(res.metaData); + //console.log('columns', columns); + return { rows: (res.rows || []).map(row => zipDataRow(row, columns)), columns }; + } catch (err) { + console.log('Error query', err, sql); + } finally { + //console.log('finally', sql); + } }, stream(client, sql, options) { /* @@ -137,8 +92,8 @@ finally { */ // console.log('queryStream', sql); const query = client.queryStream(sql); - // const consumeStream = new Promise((resolve, reject) => { - let rowcount = 0; + // const consumeStream = new Promise((resolve, reject) => { + let rowcount = 0; let wasHeader = false; query.on('metadata', row => { @@ -200,13 +155,12 @@ finally { }); options.done(); }); - query.on('close', function() { - //console.log("stream 'close' event"); - // The underlying ResultSet has been closed, so the connection can now - // be closed, if desired. Note: do not close connections on 'end'. - //resolve(rowcount); - ; - }); + query.on('close', function () { + //console.log("stream 'close' event"); + // The underlying ResultSet has been closed, so the connection can now + // be closed, if desired. Note: do not close connections on 'end'. + //resolve(rowcount); + }); //}); //const numrows = await consumeStream; @@ -215,7 +169,7 @@ finally { }, async getVersion(client) { //const { rows } = await this.query(client, "SELECT banner as version FROM v$version WHERE banner LIKE 'Oracle%'"); - const { rows } = await this.query(client, "SELECT version as \"version\" FROM v$instance"); + const { rows } = await this.query(client, 'SELECT version as "version" FROM v$instance'); const { version } = rows[0]; const isCockroach = false; //version.toLowerCase().includes('cockroachdb'); @@ -245,7 +199,7 @@ finally { }; }, async readQuery(client, sql, structure) { -/* + /* const query = new pg.Query({ text: sql, rowMode: 'array', @@ -267,10 +221,10 @@ finally { if (!wasHeader) { columns = extractOracleColumns(row); if (columns && columns.length > 0) { - pass.write({ - __isStreamHeader: true, - ...(structure || { columns }), - }); + pass.write({ + __isStreamHeader: true, + ...(structure || { columns }), + }); } wasHeader = true; } @@ -301,7 +255,7 @@ finally { return createBulkInsertStreamBase(this, stream, pool, name, options); }, async listDatabases(client) { - const { rows } = await this.query(client, 'SELECT instance_name AS \"name\" FROM v$instance'); + const { rows } = await this.query(client, 'SELECT instance_name AS "name" FROM v$instance'); return rows; }, @@ -319,7 +273,7 @@ finally { }, })); -drivers.initialize = (dbgateEnv) => { +drivers.initialize = dbgateEnv => { if (dbgateEnv.nativeModules && dbgateEnv.nativeModules.oracledb) { oracledb = dbgateEnv.nativeModules.oracledb(); } diff --git a/plugins/dbgate-plugin-oracle/src/frontend/drivers.js b/plugins/dbgate-plugin-oracle/src/frontend/drivers.js index fd60b2aa2..4f253b57b 100644 --- a/plugins/dbgate-plugin-oracle/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-oracle/src/frontend/drivers.js @@ -118,11 +118,7 @@ const oracleDriverBase = { return ['databaseUrl', 'isReadOnly'].includes(field); } - return ( - ['authType', 'user', 'password', 'defaultDatabase', 'singleDatabase', 'isReadOnly'].includes(field) || - (values.authType == 'socket' && ['socketPath'].includes(field)) || - (values.authType != 'socket' && ['server', 'port'].includes(field)) - ); + return ['user', 'password', 'defaultDatabase', 'singleDatabase', 'isReadOnly', 'server', 'port'].includes(field); }, beforeConnectionSave: connection => { @@ -166,17 +162,13 @@ $$ LANGUAGE plpgsql;`, }, ]; }, - - authTypeLabel: 'Connection mode', - defaultAuthTypeName: 'hostPort', - defaultSocketPath: '/var/run/oracledb', }; /** @type {import('dbgate-types').EngineDriver} */ const oracleDriver = { ...oracleDriverBase, engine: 'oracle@dbgate-plugin-oracle', - title: 'OracleDB', + title: 'OracleDB (Experimental)', defaultPort: 1521, dialect: { ...dialect, @@ -196,7 +188,8 @@ const oracleDriver = { } return dialect; }, + + showConnectionTab: (field) => field == 'sshTunnel', }; - module.exports = [oracleDriver]; diff --git a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js index 93aee6de7..fa2f2bb96 100644 --- a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js +++ b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js @@ -35,6 +35,7 @@ const dialect = { dropCheck: true, dropReferencesWhenDropTable: true, + requireStandaloneSelectForScopeIdentity: true, predefinedDataTypes: [ 'bigint', diff --git a/plugins/dbgate-plugin-sqlite/src/frontend/Dumper.js b/plugins/dbgate-plugin-sqlite/src/frontend/Dumper.js index e10b6ab7b..d2aa90e50 100644 --- a/plugins/dbgate-plugin-sqlite/src/frontend/Dumper.js +++ b/plugins/dbgate-plugin-sqlite/src/frontend/Dumper.js @@ -18,7 +18,23 @@ class Dumper extends SqlDumper { } selectScopeIdentity() { - this.put('^select last_insert_rowid()') + this.put('^select last_insert_rowid()'); + } + + columnDefinition(column, flags) { + if (column.dataType && column.dataType.toLowerCase().includes('int') && column.notNull && column.autoIncrement) { + this.put('^integer ^primary ^key ^autoincrement'); + return; + } + super.columnDefinition(column, flags); + } + + createTablePrimaryKeyCore(table) { + const column = table.columns.find((x) => x.autoIncrement); + if (column && column.dataType && column.dataType.toLowerCase().includes('int') && column.notNull) { + return; + } + super.createTablePrimaryKeyCore(table); } } diff --git a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js index de6071d77..a24bacb5a 100644 --- a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js +++ b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js @@ -22,6 +22,7 @@ const dialect = { return `[${s}]`; }, anonymousPrimaryKey: true, + requireStandaloneSelectForScopeIdentity: true, createColumn: true, dropColumn: true, diff --git a/yarn.lock b/yarn.lock index f781e9ab1..6f5286890 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1787,14 +1787,6 @@ activedirectory2@^2.1.0: ldapjs "^2.1.0" merge-options "^2.0.0" -adler-32@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.2.0.tgz#6a3e6bf0a63900ba15652808cb15c6813d1a5f25" - integrity sha512-/vUqU/UY4MVeFsg+SsK6c+/05RZXIHZMGJA+PX5JyWI0ZRcBpupnRuPLU/NXXoFwMYCPCoxIfElM2eS+DUXCqQ== - dependencies: - exit-on-epipe "~1.0.1" - printj "~1.1.0" - adler-32@~1.3.0: version "1.3.1" resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.3.1.tgz#1dbf0b36dda0012189a32b3679061932df1821e2" @@ -2732,7 +2724,7 @@ caseless@~0.12.0: resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== -cfb@^1.1.4: +cfb@~1.2.1: version "1.2.2" resolved "https://registry.yarnpkg.com/cfb/-/cfb-1.2.2.tgz#94e687628c700e5155436dac05f74e08df23bc44" integrity sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA== @@ -2929,13 +2921,10 @@ code-point-at@^1.0.0: resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" integrity sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA== -codepage@~1.14.0: - version "1.14.0" - resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.14.0.tgz#8cbe25481323559d7d307571b0fff91e7a1d2f99" - integrity sha512-iz3zJLhlrg37/gYRWgEPkaFTtzmnEv1h+r7NgZum2lFElYQPi0/5bnmuDfODHxfp0INEfnRqyfyeIJDbb7ahRw== - dependencies: - commander "~2.14.1" - exit-on-epipe "~1.0.1" +codepage@~1.15.0: + version "1.15.0" + resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.15.0.tgz#2e00519024b39424ec66eeb3ec07227e692618ab" + integrity sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA== collect-v8-coverage@^1.0.0: version "1.0.1" @@ -3006,16 +2995,6 @@ commander@^4.0.0: resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068" integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA== -commander@~2.14.1: - version "2.14.1" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.14.1.tgz#2235123e37af8ca3c65df45b026dbd357b01b9aa" - integrity sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw== - -commander@~2.17.1: - version "2.17.1" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf" - integrity sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg== - commondir@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" @@ -3175,7 +3154,7 @@ cpu-features@~0.0.4: buildcheck "0.0.3" nan "^2.15.0" -crc-32@~1.2.0: +crc-32@~1.2.0, crc-32@~1.2.1: version "1.2.2" resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.2.tgz#3cad35a934b8bf71f25ca524b6da51fb7eace2ff" integrity sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ== @@ -3370,30 +3349,11 @@ dbgate-plugin-xml@^5.0.0-alpha.1: resolved "https://registry.yarnpkg.com/dbgate-plugin-xml/-/dbgate-plugin-xml-5.0.9.tgz#c3abf6ed8cd1450c45058d35c9326458833ed27e" integrity sha512-P8Em1A6HhF0BfxEDDEUyzdgFeJHEC5vbg12frANpWHjO3V1HGdygsT2z1ukLK8FS5BLW/vcCdOFldXZGh+wWvg== -dbgate-query-splitter@^4.9.0, dbgate-query-splitter@^4.9.2, dbgate-query-splitter@^4.9.3: +dbgate-query-splitter@^4.9.0, dbgate-query-splitter@^4.9.3: version "4.9.3" resolved "https://registry.yarnpkg.com/dbgate-query-splitter/-/dbgate-query-splitter-4.9.3.tgz#f66396da9ae3cc8f775a282143bfca3441248aa2" integrity sha512-QMppAy3S6NGQMawNokmhbpZURvLCETyu/8yTfqWUHGdlK963fdSpmoX1A+9SjCDp62sX0vYntfD7uzd6jVSRcw== -dbgate-sqltree@^5.1.6: - version "5.1.6" - resolved "https://registry.yarnpkg.com/dbgate-sqltree/-/dbgate-sqltree-5.1.6.tgz#469fe6e06b5146afb9104114564e64aa46c13e7c" - integrity sha512-D2ffjeT5HsHBOeW0ORYkCLbDhyIvH6hrE1A/IG3kRZD1iX8Dy6c0h/BWNa4xOjz3pbJYkaB0ju8rIKJqi0BYog== - dependencies: - lodash "^4.17.21" - -dbgate-tools@^5.1.6: - version "5.1.6" - resolved "https://registry.yarnpkg.com/dbgate-tools/-/dbgate-tools-5.1.6.tgz#43c1e0575db550da5ba38627eaa19839df05d7f1" - integrity sha512-KTEqSnNzIdGfYJaIvEusch3/KnO0p376VRykJcK1+/+UtXf9Dk+azaWFjEoACBK3DlK1hmL1pEGfMh5FHDm7Qw== - dependencies: - dbgate-query-splitter "^4.9.2" - dbgate-sqltree "^5.1.6" - debug "^4.3.4" - json-stable-stringify "^1.0.1" - lodash "^4.17.21" - uuid "^3.4.0" - debug@2.6.9, debug@^2.2.0, debug@^2.3.3: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -4087,11 +4047,6 @@ execa@^5.0.0: signal-exit "^3.0.3" strip-final-newline "^2.0.0" -exit-on-epipe@~1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692" - integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw== - exit@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" @@ -4333,11 +4288,6 @@ fb-watchman@^2.0.0: dependencies: bser "2.1.1" -fflate@^0.3.8: - version "0.3.11" - resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.3.11.tgz#2c440d7180fdeb819e64898d8858af327b042a5d" - integrity sha512-Rr5QlUeGN1mbOHlaqcSYMKVpPbgLy0AWT/W0EHxA6NGI12yO1jpoui2zBBvU2G824ltM6Ut8BFgfHSBGfkmS0A== - figgy-pudding@^3.5.1: version "3.5.2" resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e" @@ -8896,11 +8846,6 @@ pretty-format@^28.1.3: ansi-styles "^5.0.0" react-is "^18.0.0" -printj@~1.1.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222" - integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ== - process-nextick-args@~1.0.6: version "1.0.7" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" @@ -11479,18 +11424,15 @@ ws@^7.4.3, ws@^7.4.6: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== -xlsx@^0.16.8: - version "0.16.9" - resolved "https://registry.yarnpkg.com/xlsx/-/xlsx-0.16.9.tgz#dacd5bb46bda6dd3743940c9c3dc1e2171826256" - integrity sha512-gxi1I3EasYvgCX1vN9pGyq920Ron4NO8PNfhuoA3Hpq6Y8f0ECXiy4OLrK4QZBnj1jx3QD+8Fq5YZ/3mPZ5iXw== +xlsx@^0.18.5: + version "0.18.5" + resolved "https://registry.yarnpkg.com/xlsx/-/xlsx-0.18.5.tgz#16711b9113c848076b8a177022799ad356eba7d0" + integrity sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ== dependencies: - adler-32 "~1.2.0" - cfb "^1.1.4" - codepage "~1.14.0" - commander "~2.17.1" - crc-32 "~1.2.0" - exit-on-epipe "~1.0.1" - fflate "^0.3.8" + adler-32 "~1.3.0" + cfb "~1.2.1" + codepage "~1.15.0" + crc-32 "~1.2.1" ssf "~0.11.2" wmf "~1.0.1" word "~0.3.0"