Merge branch 'develop'

integration-tests/__tests__/data-duplicator.spec.js

@@ -0,0 +1,94 @@
+const engines = require('../engines');
+const stream = require('stream');
+const { testWrapper } = require('../tools');
+const dataDuplicator = require('dbgate-api/src/shell/dataDuplicator');
+const { runCommandOnDriver } = require('dbgate-tools');
+
+describe('Data duplicator', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Insert simple data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't1',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+        })
+      );
+      runCommandOnDriver(conn, driver, dmp =>
+        dmp.createTable({
+          pureName: 't2',
+          columns: [
+            { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true },
+            { columnName: 'val', dataType: 'varchar(50)' },
+            { columnName: 'valfk', dataType: 'int', notNull: true },
+          ],
+          primaryKey: {
+            columns: [{ columnName: 'id' }],
+          },
+          foreignKeys: [{ refTableName: 't1', columns: [{ columnName: 'valfk', refColumnName: 'id' }] }],
+        })
+      );
+
+      const gett1 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1' },
+          { id: 2, val: 'v2' },
+          { id: 3, val: 'v3' },
+        ]);
+      const gett2 = () =>
+        stream.Readable.from([
+          { __isStreamHeader: true, __isDynamicStructure: true },
+          { id: 1, val: 'v1', valfk: 1 },
+          { id: 2, val: 'v2', valfk: 2 },
+          { id: 3, val: 'v3', valfk: 3 },
+        ]);
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            operation: 'copy',
+            openStream: gett1,
+          },
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+      });
+
+      await dataDuplicator({
+        systemConnection: conn,
+        driver,
+        items: [
+          {
+            name: 't1',
+            operation: 'copy',
+            openStream: gett1,
+          },
+          {
+            name: 't2',
+            operation: 'copy',
+            openStream: gett2,
+          },
+        ],
+      });
+
+      const res1 = await driver.query(conn, `select count(*) as cnt from t1`);
+      expect(res1.rows[0].cnt.toString()).toEqual('6');
+
+      const res2 = await driver.query(conn, `select count(*) as cnt from t2`);
+      expect(res2.rows[0].cnt.toString()).toEqual('6');
+    })
+  );
+});

integration-tests/engines.js

@@ -136,8 +136,8 @@ const filterLocal = [
   '-MySQL',
   '-MariaDB',
   '-PostgreSQL',
-  'SQL Server',
+  '-SQL Server',
-  '-SQLite',
+  'SQLite',
   '-CockroachDB',
 ];
 

integration-tests/package.json

@@ -13,6 +13,8 @@
     "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
 
     "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
+    "test:local:path": "cross-env DEVMODE=1 LOCALTEST=1 jest --runTestsByPath __tests__/data-duplicator.spec.js",
+    
     "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
     
     "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "5.2.3-beta.6",
+  "version": "5.2.3-beta.8",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",

packages/api/src/controllers/archive.js

@@ -3,13 +3,13 @@ const readline = require('readline');
 const path = require('path');
 const { archivedir, clearArchiveLinksCache, resolveArchiveFolder } = require('../utility/directories');
 const socket = require('../utility/socket');
-const { saveFreeTableData } = require('../utility/freeTableStorage');
 const loadFilesRecursive = require('../utility/loadFilesRecursive');
 const getJslFileName = require('../utility/getJslFileName');
 const { getLogger } = require('dbgate-tools');
 const uuidv1 = require('uuid/v1');
 const dbgateApi = require('../shell');
 const jsldata = require('./jsldata');
+const platformInfo = require('../utility/platformInfo');
 
 const logger = getLogger('archive');
 
@@ -137,8 +137,13 @@ module.exports = {
     });
     const writer = await dbgateApi.jsonLinesWriter({ fileName: tmpchangedFilePath });
     await dbgateApi.copyStream(reader, writer);
-    await fs.unlink(changedFilePath);
+    if (platformInfo.isWindows) {
-    await fs.rename(path.join(tmpchangedFilePath), path.join(changedFilePath));
+      await fs.copyFile(tmpchangedFilePath, changedFilePath);
+      await fs.unlink(tmpchangedFilePath);
+    } else {
+      await fs.unlink(changedFilePath);
+      await fs.rename(tmpchangedFilePath, changedFilePath);
+    }
     return true;
   },
 
@@ -162,34 +167,6 @@ module.exports = {
     return true;
   },
 
-  saveFreeTable_meta: true,
-  async saveFreeTable({ folder, file, data }) {
-    await saveFreeTableData(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), data);
-    socket.emitChanged(`archive-files-changed`, { folder });
-    return true;
-  },
-
-  loadFreeTable_meta: true,
-  async loadFreeTable({ folder, file }) {
-    return new Promise((resolve, reject) => {
-      const fileStream = fs.createReadStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));
-      const liner = readline.createInterface({
-        input: fileStream,
-      });
-      let structure = null;
-      const rows = [];
-      liner.on('line', line => {
-        const data = JSON.parse(line);
-        if (structure) rows.push(data);
-        else structure = data;
-      });
-      liner.on('close', () => {
-        resolve({ structure, rows });
-        fileStream.close();
-      });
-    });
-  },
-
   saveText_meta: true,
   async saveText({ folder, file, text }) {
     await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
@@ -198,10 +175,30 @@ module.exports = {
   },
 
   saveJslData_meta: true,
-  async saveJslData({ folder, file, jslid }) {
+  async saveJslData({ folder, file, jslid, changeSet }) {
     const source = getJslFileName(jslid);
     const target = path.join(resolveArchiveFolder(folder), `${file}.jsonl`);
-    await fs.copyFile(source, target);
+    if (changeSet) {
+      const reader = await dbgateApi.modifyJsonLinesReader({
+        fileName: source,
+        changeSet,
+      });
+      const writer = await dbgateApi.jsonLinesWriter({ fileName: target });
+      await dbgateApi.copyStream(reader, writer);
+    } else {
+      await fs.copyFile(source, target);
+      socket.emitChanged(`archive-files-changed`, { folder });
+    }
+    return true;
+  },
+
+  saveRows_meta: true,
+  async saveRows({ folder, file, rows }) {
+    const fileStream = fs.createWriteStream(path.join(resolveArchiveFolder(folder), `${file}.jsonl`));
+    for (const row of rows) {
+      await fileStream.write(JSON.stringify(row) + '\n');
+    }
+    await fileStream.close();
     socket.emitChanged(`archive-files-changed`, { folder });
     return true;
   },

packages/api/src/controllers/jsldata.js

@@ -4,7 +4,6 @@ const lineReader = require('line-reader');
 const _ = require('lodash');
 const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
-const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
 const requirePluginFunction = require('../utility/requirePluginFunction');
@@ -148,6 +147,12 @@ module.exports = {
     return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters, _.isEmpty(sort) ? null : sort);
   },
 
+  exists_meta: true,
+  async exists({ jslid }) {
+    const fileName = getJslFileName(jslid);
+    return fs.existsSync(fileName);
+  },
+
   getStats_meta: true,
   getStats({ jslid }) {
     const file = `${getJslFileName(jslid)}.stats`;
@@ -189,18 +194,22 @@ module.exports = {
     // }
   },
 
-  saveFreeTable_meta: true,
-  async saveFreeTable({ jslid, data }) {
-    saveFreeTableData(getJslFileName(jslid), data);
-    return true;
-  },
-
   saveText_meta: true,
   async saveText({ jslid, text }) {
     await fs.promises.writeFile(getJslFileName(jslid), text);
     return true;
   },
 
+  saveRows_meta: true,
+  async saveRows({ jslid, rows }) {
+    const fileStream = fs.createWriteStream(getJslFileName(jslid));
+    for (const row of rows) {
+      await fileStream.write(JSON.stringify(row) + '\n');
+    }
+    await fileStream.close();
+    return true;
+  },
+
   extractTimelineChart_meta: true,
   async extractTimelineChart({ jslid, timestampFunction, aggregateFunction, measures }) {
     const timestamp = requirePluginFunction(timestampFunction);

packages/api/src/controllers/runners.js

@@ -70,15 +70,20 @@ module.exports = {
     if (message) {
       const json = safeJsonParse(message.message);
 
-      if (json) logger.info(json);
+      if (json) logger.log(json);
       else logger.info(message.message);
 
-      socket.emit(`runner-info-${runid}`, {
+      const toEmit = {
         time: new Date(),
-        severity: 'info',
         ...message,
         message: json ? json.msg : message.message,
-      });
+      };
+
+      if (json && json.level >= 50) {
+        toEmit.severity = 'error';
+      }
+
+      socket.emit(`runner-info-${runid}`, toEmit);
     }
   },
 
@@ -125,8 +130,9 @@ module.exports = {
         },
       }
     );
-    const pipeDispatcher = severity => data =>
+    const pipeDispatcher = severity => data => {
-      this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+      return this.dispatchMessage(runid, { severity, message: data.toString().trim() });
+    };
 
     byline(subprocess.stdout).on('data', pipeDispatcher('info'));
     byline(subprocess.stderr).on('data', pipeDispatcher('error'));

packages/api/src/proc/databaseConnectionProcess.js

@@ -271,7 +271,7 @@ async function handleSqlPreview({ msgid, objects, options }) {
     process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
     if (generator.isUnhandledException) {
       setTimeout(() => {
-        getLogger.info('Exiting because of unhandled exception');
+        logger.error('Exiting because of unhandled exception');
         process.exit(0);
       }, 500);
     }

packages/api/src/shell/dataDuplicator.js

@@ -9,9 +9,18 @@ const copyStream = require('./copyStream');
 const jsonLinesReader = require('./jsonLinesReader');
 const { resolveArchiveFolder } = require('../utility/directories');
 
-async function dataDuplicator({ connection, archive, items, analysedStructure = null }) {
+async function dataDuplicator({
-  const driver = requireEngineDriver(connection);
+  connection,
-  const pool = await connectUtility(driver, connection, 'write');
+  archive,
+  items,
+  options,
+  analysedStructure = null,
+  driver,
+  systemConnection,
+}) {
+  if (!driver) driver = requireEngineDriver(connection);
+  const pool = systemConnection || (await connectUtility(driver, connection, 'write'));
+
   logger.info(`Connected.`);
 
   if (!analysedStructure) {
@@ -26,10 +35,13 @@ async function dataDuplicator({ connection, archive, items, analysedStructure =
       name: item.name,
       operation: item.operation,
       matchColumns: item.matchColumns,
-      openStream: () => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) }),
+      openStream:
+        item.openStream ||
+        (() => jsonLinesReader({ fileName: path.join(resolveArchiveFolder(archive), `${item.name}.jsonl`) })),
     })),
     stream,
-    copyStream
+    copyStream,
+    options
   );
 
   await dupl.run();

packages/api/src/shell/fakeObjectReader.js

@@ -1,18 +1,26 @@
 const stream = require('stream');
 
-async function fakeObjectReader({ delay = 0 } = {}) {
+async function fakeObjectReader({ delay = 0, dynamicData = null } = {}) {
   const pass = new stream.PassThrough({
     objectMode: true,
   });
   function doWrite() {
-    pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+    if (dynamicData) {
-    pass.write({ id: 1, country: 'Czechia' });
+      pass.write({ __isStreamHeader: true, __isDynamicStructure: true });
-    pass.write({ id: 2, country: 'Austria' });
+      for (const item of dynamicData) {
-    pass.write({ country: 'Germany', id: 3 });
+        pass.write(item);
-    pass.write({ country: 'Romania', id: 4 });
+      }
-    pass.write({ country: 'Great Britain', id: 5 });
+      pass.end();
-    pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+    } else {
-    pass.end();
+      pass.write({ columns: [{ columnName: 'id' }, { columnName: 'country' }], __isStreamHeader: true });
+      pass.write({ id: 1, country: 'Czechia' });
+      pass.write({ id: 2, country: 'Austria' });
+      pass.write({ country: 'Germany', id: 3 });
+      pass.write({ country: 'Romania', id: 4 });
+      pass.write({ country: 'Great Britain', id: 5 });
+      pass.write({ country: 'Bosna, Hecegovina', id: 6 });
+      pass.end();
+    }
   }
 
   if (delay) {

packages/api/src/shell/jsonLinesWriter.js

@@ -12,7 +12,9 @@ class StringifyStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      skip = (chunk.__isStreamHeader && !this.header) || (chunk.__isStreamHeader && chunk.__isDynamicStructure);
+      skip =
+        (chunk.__isStreamHeader && !this.header) ||
+        (chunk.__isStreamHeader && chunk.__isDynamicStructure && !chunk.__keepDynamicStreamHeader);
       this.wasHeader = true;
     }
     if (!skip) {

packages/api/src/shell/modifyJsonLinesReader.js

@@ -2,7 +2,7 @@ const fs = require('fs');
 const _ = require('lodash');
 const stream = require('stream');
 const byline = require('byline');
-const { getLogger } = require('dbgate-tools');
+const { getLogger, processJsonDataUpdateCommands, removeTablePairingId } = require('dbgate-tools');
 const logger = getLogger('modifyJsonLinesReader');
 const stableStringify = require('json-stable-stringify');
 
@@ -11,6 +11,7 @@ class ParseStream extends stream.Transform {
     super({ objectMode: true });
     this.limitRows = limitRows;
     this.changeSet = changeSet;
+    this.wasHeader = false;
     this.currentRowIndex = 0;
     if (mergeMode == 'merge') {
       if (mergedRows && mergeKey) {
@@ -28,12 +29,28 @@ class ParseStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let obj = JSON.parse(chunk);
     if (obj.__isStreamHeader) {
-      this.push(obj);
+      if (this.changeSet && this.changeSet.structure) {
+        this.push({
+          ...removeTablePairingId(this.changeSet.structure),
+          __isStreamHeader: true,
+        });
+      } else {
+        this.push(obj);
+      }
+      this.wasHeader = true;
       done();
       return;
     }
 
     if (this.changeSet) {
+      if (!this.wasHeader && this.changeSet.structure) {
+        this.push({
+          ...removeTablePairingId(this.changeSet.structure),
+          __isStreamHeader: true,
+        });
+        this.wasHeader = true;
+      }
+
       if (!this.limitRows || this.currentRowIndex < this.limitRows) {
         if (this.changeSet.deletes.find(x => x.existingRowIndex == this.currentRowIndex)) {
           obj = null;
@@ -41,13 +58,20 @@ class ParseStream extends stream.Transform {
 
         const update = this.changeSet.updates.find(x => x.existingRowIndex == this.currentRowIndex);
         if (update) {
-          obj = {
+          if (update.document) {
-            ...obj,
+            obj = update.document;
-            ...update.fields,
+          } else {
-          };
+            obj = {
+              ...obj,
+              ...update.fields,
+            };
+          }
         }
 
         if (obj) {
+          if (this.changeSet.dataUpdateCommands) {
+            obj = processJsonDataUpdateCommands(obj, this.changeSet.dataUpdateCommands);
+          }
           this.push(obj);
         }
         this.currentRowIndex += 1;

packages/api/src/shell/runScript.js

@@ -11,7 +11,7 @@ async function runScript(func) {
     await func();
     process.exit(0);
   } catch (err) {
-    logger.error('Error running script', err);
+    logger.error({ err }, `Error running script: ${err.message}`);
     process.exit(1);
   }
 }

packages/api/src/utility/JsonLinesDatastore.js

@@ -2,7 +2,6 @@ const fs = require('fs');
 const os = require('os');
 const rimraf = require('rimraf');
 const path = require('path');
-const lineReader = require('line-reader');
 const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
@@ -11,23 +10,7 @@ const requirePluginFunction = require('./requirePluginFunction');
 const esort = require('external-sorting');
 const uuidv1 = require('uuid/v1');
 const { jsldir } = require('./directories');
-
+const LineReader = require('./LineReader');
-function fetchNextLineFromReader(reader) {
-  return new Promise((resolve, reject) => {
-    if (!reader.hasNextLine()) {
-      resolve(null);
-      return;
-    }
-
-    reader.nextLine((err, line) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(line);
-      }
-    });
-  });
-}
 
 class JsonLinesDatastore {
   constructor(file, formatterFunction) {
@@ -74,7 +57,7 @@ class JsonLinesDatastore {
     await new Promise(resolve => rimraf(tempDir, resolve));
   }
 
-  _closeReader() {
+  async _closeReader() {
     // console.log('CLOSING READER', this.reader);
     if (!this.reader) return;
     const reader = this.reader;
@@ -84,7 +67,7 @@ class JsonLinesDatastore {
     // this.firstRowToBeReturned = null;
     this.currentFilter = null;
     this.currentSort = null;
-    return new Promise(resolve => reader.close(resolve));
+    await reader.close();
   }
 
   async notifyChanged(callback) {
@@ -100,12 +83,9 @@ class JsonLinesDatastore {
   async _openReader(fileName) {
     // console.log('OPENING READER', fileName);
     // console.log(fs.readFileSync(fileName, 'utf-8'));
-    return new Promise((resolve, reject) =>
+
-      lineReader.open(fileName, (err, reader) => {
+    const fileStream = fs.createReadStream(fileName);
-        if (err) reject(err);
+    return new LineReader(fileStream);
-        resolve(reader);
-      })
-    );
   }
 
   parseLine(line) {
@@ -120,7 +100,7 @@ class JsonLinesDatastore {
     //   return res;
     // }
     for (;;) {
-      const line = await fetchNextLineFromReader(this.reader);
+      const line = await this.reader.readLine();
       if (!line) {
         // EOF
         return null;
@@ -240,6 +220,7 @@ class JsonLinesDatastore {
       // console.log(JSON.stringify(this.currentFilter, undefined, 2));
       for (let i = 0; i < limit; i += 1) {
         const line = await this._readLine(true);
+        // console.log('READED LINE', i);
         if (line == null) break;
         res.push(line);
       }

packages/api/src/utility/LineReader.js

@@ -0,0 +1,88 @@
+const readline = require('readline');
+
+class Queue {
+  constructor() {
+    this.elements = {};
+    this.head = 0;
+    this.tail = 0;
+  }
+  enqueue(element) {
+    this.elements[this.tail] = element;
+    this.tail++;
+  }
+  dequeue() {
+    const item = this.elements[this.head];
+    delete this.elements[this.head];
+    this.head++;
+    return item;
+  }
+  peek() {
+    return this.elements[this.head];
+  }
+  getLength() {
+    return this.tail - this.head;
+  }
+  isEmpty() {
+    return this.getLength() === 0;
+  }
+}
+
+class LineReader {
+  constructor(input) {
+    this.input = input;
+    this.queue = new Queue();
+    this.resolve = null;
+    this.isEnded = false;
+    this.rl = readline.createInterface({
+      input,
+    });
+    this.input.pause();
+
+    this.rl.on('line', line => {
+      this.input.pause();
+      if (this.resolve) {
+        const resolve = this.resolve;
+        this.resolve = null;
+        resolve(line);
+        return;
+      }
+      this.queue.enqueue(line);
+    });
+
+    this.rl.on('close', () => {
+      if (this.resolve) {
+        const resolve = this.resolve;
+        this.resolve = null;
+        this.isEnded = true;
+        resolve(null);
+        return;
+      }
+      this.queue.enqueue(null);
+    });
+  }
+
+  readLine() {
+    if (this.isEnded) {
+      return Promise.resolve(null);
+    }
+
+    if (!this.queue.isEmpty()) {
+      const res = this.queue.dequeue();
+      if (res == null) this.isEnded = true;
+      return Promise.resolve(res);
+    }
+
+    this.input.resume();
+
+    return new Promise(resolve => {
+      this.resolve = resolve;
+    });
+  }
+
+  close() {
+    this.isEnded = true;
+    return new Promise(resolve => this.input.close(resolve));
+  }
+}
+
+module.exports = LineReader;

packages/api/src/utility/directories.js

@@ -42,18 +42,23 @@ function datadir() {
   return dir;
 }
 
-const dirFunc = (dirname, clean) => () => {
+const dirFunc =
-  const dir = path.join(datadir(), dirname);
+  (dirname, clean, subdirs = []) =>
-  ensureDirectory(dir, clean);
+  () => {
+    const dir = path.join(datadir(), dirname);
+    ensureDirectory(dir, clean);
+    for (const subdir of subdirs) {
+      ensureDirectory(path.join(dir, subdir), false);
+    }
 
-  return dir;
+    return dir;
-};
+  };
 
 const jsldir = dirFunc('jsl', true);
 const rundir = dirFunc('run', true);
 const uploadsdir = dirFunc('uploads', true);
 const pluginsdir = dirFunc('plugins');
-const archivedir = dirFunc('archive');
+const archivedir = dirFunc('archive', false, ['default']);
 const appdir = dirFunc('apps');
 const filesdir = dirFunc('files');
 const logsdir = dirFunc('logs', 3600 * 24 * 7);

packages/api/src/utility/freeTableStorage.js

@@ -1,15 +0,0 @@
-const fs = require('fs-extra');
-
-async function saveFreeTableData(file, data) {
-  const { structure, rows } = data;
-  const fileStream = fs.createWriteStream(file);
-  await fileStream.write(JSON.stringify({ __isStreamHeader: true, ...structure }) + '\n');
-  for (const row of rows) {
-    await fileStream.write(JSON.stringify(row) + '\n');
-  }
-  await fileStream.close();
-}
-
-module.exports = {
-  saveFreeTableData,
-};

packages/datalib/src/ChangeSet.ts

@@ -9,7 +9,8 @@ import {
   AllowIdentityInsert,
   Expression,
 } from 'dbgate-sqltree';
-import type { NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
+import type { NamedObjectInfo, DatabaseInfo, TableInfo } from 'dbgate-types';
+import { JsonDataObjectUpdateCommand } from 'dbgate-tools';
 
 export interface ChangeSetItem {
   pureName: string;
@@ -21,12 +22,18 @@ export interface ChangeSetItem {
   fields?: { [column: string]: string };
 }
 
-export interface ChangeSet {
+export interface ChangeSetItemFields {
   inserts: ChangeSetItem[];
   updates: ChangeSetItem[];
   deletes: ChangeSetItem[];
 }
 
+export interface ChangeSet extends ChangeSetItemFields {
+  structure?: TableInfo;
+  dataUpdateCommands?: JsonDataObjectUpdateCommand[];
+  setColumnMode?: 'fixed' | 'variable';
+}
+
 export function createChangeSet(): ChangeSet {
   return {
     inserts: [],
@@ -51,7 +58,7 @@ export interface ChangeSetFieldDefinition extends ChangeSetRowDefinition {
 export function findExistingChangeSetItem(
   changeSet: ChangeSet,
   definition: ChangeSetRowDefinition
-): [keyof ChangeSet, ChangeSetItem] {
+): [keyof ChangeSetItemFields, ChangeSetItem] {
   if (!changeSet || !definition) return ['updates', null];
   if (definition.insertedRowIndex != null) {
     return [
@@ -456,5 +463,12 @@ export function changeSetInsertDocuments(changeSet: ChangeSet, documents: any[],
 
 export function changeSetContainsChanges(changeSet: ChangeSet) {
   if (!changeSet) return false;
-  return changeSet.deletes.length > 0 || changeSet.updates.length > 0 || changeSet.inserts.length > 0;
+  return (
+    changeSet.deletes.length > 0 ||
+    changeSet.updates.length > 0 ||
+    changeSet.inserts.length > 0 ||
+    !!changeSet.structure ||
+    !!changeSet.setColumnMode ||
+    changeSet.dataUpdateCommands?.length > 0
+  );
 }

packages/datalib/src/DataDuplicator.ts

@@ -12,6 +12,11 @@ export interface DataDuplicatorItem {
   matchColumns: string[];
 }
 
+export interface DataDuplicatorOptions {
+  rollbackAfterFinish?: boolean;
+  skipRowsWithUnresolvedRefs?: boolean;
+}
+
 class DuplicatorReference {
   constructor(
     public base: DuplicatorItemHolder,
@@ -78,6 +83,13 @@ class DuplicatorItemHolder {
       if (ref) {
         // remap id
         res[key] = ref.ref.idMap[res[key]];
+        if (ref.isMandatory && res[key] == null) {
+          // mandatory refertence not matched
+          if (this.duplicator.options.skipRowsWithUnresolvedRefs) {
+            return null;
+          }
+          throw new Error(`Unresolved reference, base=${ref.base.name}, ref=${ref.ref.name}, ${key}=${chunk[key]}`);
+        }
       }
     }
 
@@ -91,6 +103,8 @@ class DuplicatorItemHolder {
     let inserted = 0;
     let mapped = 0;
     let missing = 0;
+    let skipped = 0;
+    let lastLogged = new Date();
 
     const writeStream = createAsyncWriteStream(this.duplicator.stream, {
       processItem: async chunk => {
@@ -99,18 +113,35 @@ class DuplicatorItemHolder {
         }
 
         const doCopy = async () => {
+          // console.log('chunk', this.name, JSON.stringify(chunk));
           const insertedObj = this.createInsertObject(chunk);
-          await runCommandOnDriver(pool, driver, dmp =>
+          // console.log('insertedObj', this.name, JSON.stringify(insertedObj));
-            dmp.putCmd(
+          if (insertedObj == null) {
+            skipped += 1;
+            return;
+          }
+          let res = await runQueryOnDriver(pool, driver, dmp => {
+            dmp.put(
               '^insert ^into %f (%,i) ^values (%,v)',
               this.table,
               Object.keys(insertedObj),
               Object.values(insertedObj)
-            )
+            );
-          );
+
+            if (
+              this.autoColumn &&
+              this.isReferenced &&
+              !this.duplicator.driver.dialect.requireStandaloneSelectForScopeIdentity
+            ) {
+              dmp.selectScopeIdentity(this.table);
+            }
+          });
           inserted += 1;
           if (this.autoColumn && this.isReferenced) {
-            const res = await runQueryOnDriver(pool, driver, dmp => dmp.selectScopeIdentity(this.table));
+            if (this.duplicator.driver.dialect.requireStandaloneSelectForScopeIdentity) {
+              res = await runQueryOnDriver(pool, driver, dmp => dmp.selectScopeIdentity(this.table));
+            }
+            // console.log('IDRES', JSON.stringify(res));
             const resId = Object.entries(res?.rows?.[0])?.[0]?.[1];
             if (resId != null) {
               this.idMap[chunk[this.autoColumn]] = resId;
@@ -146,6 +177,13 @@ class DuplicatorItemHolder {
             break;
           }
         }
+
+        if (new Date().getTime() - lastLogged.getTime() > 5000) {
+          logger.info(
+            `Duplicating ${this.item.name} in progress, inserted ${inserted} rows, mapped ${mapped} rows, missing ${missing} rows, skipped ${skipped} rows`
+          );
+          lastLogged = new Date();
+        }
         // this.idMap[oldId] = newId;
       },
     });
@@ -158,7 +196,7 @@ class DuplicatorItemHolder {
     //   },
     // });
 
-    return { inserted, mapped, missing };
+    return { inserted, mapped, missing, skipped };
   }
 }
 
@@ -172,7 +210,8 @@ export class DataDuplicator {
     public db: DatabaseInfo,
     public items: DataDuplicatorItem[],
     public stream,
-    public copyStream: (input, output) => Promise<void>
+    public copyStream: (input, output) => Promise<void>,
+    public options: DataDuplicatorOptions = {}
   ) {
     this.itemHolders = items.map(x => new DuplicatorItemHolder(x, this));
     this.itemHolders.forEach(x => x.initializeReferences());
@@ -212,13 +251,20 @@ export class DataDuplicator {
       for (const item of this.itemPlan) {
         const stats = await item.runImport();
         logger.info(
-          `Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows`
+          `Duplicated ${item.name}, inserted ${stats.inserted} rows, mapped ${stats.mapped} rows, missing ${stats.missing} rows, skipped ${stats.skipped} rows`
         );
       }
     } catch (err) {
-      logger.error({ err }, 'Failed duplicator job, rollbacking');
+      logger.error({ err }, `Failed duplicator job, rollbacking. ${err.message}`);
       await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction());
+      return;
+    }
+    if (this.options.rollbackAfterFinish) {
+      logger.info('Rollbacking transaction, nothing was changed');
+      await runCommandOnDriver(this.pool, this.driver, dmp => dmp.rollbackTransaction());
+    } else {
+      logger.info('Committing duplicator transaction');
+      await runCommandOnDriver(this.pool, this.driver, dmp => dmp.commitTransaction());
     }
-    await runCommandOnDriver(this.pool, this.driver, dmp => dmp.commitTransaction());
   }
 }

packages/datalib/src/FreeTableGridDisplay.ts

@@ -1,48 +0,0 @@
-import _ from 'lodash';
-import type { EngineDriver, ViewInfo, ColumnInfo } from 'dbgate-types';
-import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
-import { GridConfig, GridCache } from './GridConfig';
-import { FreeTableModel } from './FreeTableModel';
-import { analyseCollectionDisplayColumns } from '.';
-
-export class FreeTableGridDisplay extends GridDisplay {
-  constructor(
-    public model: FreeTableModel,
-    config: GridConfig,
-    setConfig: ChangeConfigFunc,
-    cache: GridCache,
-    setCache: ChangeCacheFunc
-  ) {
-    super(config, setConfig, cache, setCache);
-    this.columns = model?.structure?.__isDynamicStructure
-      ? analyseCollectionDisplayColumns(model?.rows, this)
-      : this.getDisplayColumns(model);
-    this.filterable = false;
-    this.sortable = false;
-  }
-
-  getDisplayColumns(model: FreeTableModel) {
-    return _.uniqBy(
-      model?.structure?.columns
-        ?.map(col => this.getDisplayColumn(col))
-        ?.map(col => ({
-          ...col,
-          isChecked: this.isColumnChecked(col),
-        })) || [],
-      col => col.uniqueName
-    );
-  }
-
-  getDisplayColumn(col: ColumnInfo) {
-    const uniquePath = [col.columnName];
-    const uniqueName = uniquePath.join('.');
-    return {
-      ...col,
-      pureName: 'data',
-      schemaName: '',
-      headerText: col.columnName,
-      uniqueName,
-      uniquePath,
-    };
-  }
-}

packages/datalib/src/FreeTableModel.ts

@@ -1,27 +0,0 @@
-import type { TableInfo } from 'dbgate-types';
-
-export interface FreeTableModel {
-  structure: TableInfo;
-  rows: any[];
-}
-
-export function createFreeTableModel() {
-  return {
-    structure: {
-      columns: [
-        {
-          columnName: 'col1',
-        },
-      ],
-      foreignKeys: [],
-    },
-    rows: [
-      {
-        col1: 'val1',
-      },
-      {
-        col1: 'val2',
-      },
-    ],
-  };
-}

packages/datalib/src/GridDisplay.ts

@@ -84,6 +84,7 @@ export abstract class GridDisplay {
     return this.baseTable || this.baseView;
   }
   changeSetKeyFields: string[] = null;
+  editableStructure: TableInfo = null;
   sortable = false;
   groupable = false;
   filterable = false;

packages/datalib/src/JslGridDisplay.ts

@@ -24,6 +24,7 @@ export class JslGridDisplay extends GridDisplay {
     this.isDynamicStructure = isDynamicStructure;
     this.filterTypeOverride = 'eval';
     this.editable = editable;
+    this.editableStructure = editable ? structure : null;
 
     if (structure?.columns) {
       this.columns = _.uniqBy(

packages/datalib/src/MacroDefinition.ts

@@ -11,7 +11,7 @@ export interface MacroDefinition {
   name: string;
   group: string;
   description?: string;
-  type: 'transformValue';
+  type: 'transformValue' | 'transformRow';
   code: string;
   args?: MacroArgument[];
 }

packages/datalib/src/index.ts

@@ -6,12 +6,8 @@ export * from './TableGridDisplay';
 export * from './ViewGridDisplay';
 export * from './JslGridDisplay';
 export * from './ChangeSet';
-export * from './FreeTableGridDisplay';
-export * from './FreeTableModel';
 export * from './MacroDefinition';
 export * from './runMacro';
-// export * from './FormViewDisplay';
-// export * from './TableFormViewDisplay';
 export * from './CollectionGridDisplay';
 export * from './deleteCascade';
 export * from './PerspectiveDisplay';

packages/datalib/src/runMacro.ts

@@ -1,10 +1,9 @@
-import { FreeTableModel } from './FreeTableModel';
 import _ from 'lodash';
 import uuidv1 from 'uuid/v1';
 import uuidv4 from 'uuid/v4';
 import moment from 'moment';
 import { MacroDefinition, MacroSelectedCell } from './MacroDefinition';
-import { ChangeSet, setChangeSetValue } from './ChangeSet';
+import { ChangeSet, setChangeSetValue, setChangeSetRowData } from './ChangeSet';
 import { GridDisplay } from './GridDisplay';
 
 const getMacroFunction = {
@@ -13,13 +12,8 @@ const getMacroFunction = {
     ${code}
 }
 `,
-  transformRows: code => `
+  transformRow: code => `
-(rows, args, modules, selectedCells, cols, columns) => {
+(row, args, modules, rowIndex, columns) => {
-  ${code}
-}
-`,
-  transformData: code => `
-(rows, args, modules, selectedCells, cols, columns) => {
   ${code}
 }
 `,
@@ -32,160 +26,6 @@ const modules = {
   moment,
 };
 
-function runTramsformValue(
-  func,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-) {
-  const selectedRows = _.groupBy(selectedCells, 'row');
-  const rows = data.rows.map((row, rowIndex) => {
-    const selectedRow = selectedRows[rowIndex];
-    if (selectedRow) {
-      const modifiedFields = [];
-      let res = null;
-      for (const cell of selectedRow) {
-        const { column } = cell;
-        const oldValue = row[column];
-        let newValue = oldValue;
-        try {
-          newValue = func(oldValue, macroArgs, modules, rowIndex, row, column);
-        } catch (err) {
-          errors.push(`Error processing column ${column} on row ${rowIndex}: ${err.message}`);
-        }
-        if (newValue != oldValue) {
-          if (res == null) {
-            res = { ...row };
-          }
-          res[column] = newValue;
-          if (preview) modifiedFields.push(column);
-        }
-      }
-      if (res) {
-        if (modifiedFields.length > 0) {
-          return {
-            ...res,
-            __modifiedFields: new Set(modifiedFields),
-          };
-        }
-        return res;
-      }
-      return row;
-    } else {
-      return row;
-    }
-  });
-
-  return {
-    structure: data.structure,
-    rows,
-  };
-}
-
-function removePreviewRowFlags(rows) {
-  rows = rows.filter(row => row.__rowStatus != 'deleted');
-  rows = rows.map(row => {
-    if (row.__rowStatus || row.__modifiedFields || row.__insertedFields || row.__deletedFields)
-      return _.omit(row, ['__rowStatus', '__modifiedFields', '__insertedFields', '__deletedFields']);
-    return row;
-  });
-  return rows;
-}
-
-function runTramsformRows(
-  func,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-) {
-  let rows = data.rows;
-  try {
-    rows = func(
-      data.rows,
-      macroArgs,
-      modules,
-      selectedCells,
-      data.structure.columns.map(x => x.columnName),
-      data.structure.columns
-    );
-    if (!preview) {
-      rows = removePreviewRowFlags(rows);
-    }
-  } catch (err) {
-    errors.push(`Error processing rows: ${err.message}`);
-  }
-  return {
-    structure: data.structure,
-    rows,
-  };
-}
-
-function runTramsformData(
-  func,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-) {
-  try {
-    let { rows, columns, cols } = func(
-      data.rows,
-      macroArgs,
-      modules,
-      selectedCells,
-      data.structure.columns.map(x => x.columnName),
-      data.structure.columns
-    );
-    if (cols && !columns) {
-      columns = cols.map(columnName => ({ columnName }));
-    }
-    columns = _.uniqBy(columns, 'columnName');
-    if (!preview) {
-      rows = removePreviewRowFlags(rows);
-    }
-    return {
-      structure: { columns },
-      rows,
-    };
-  } catch (err) {
-    errors.push(`Error processing data: ${err.message}`);
-  }
-  return data;
-}
-
-export function runMacro(
-  macro: MacroDefinition,
-  macroArgs: {},
-  data: FreeTableModel,
-  preview: boolean,
-  selectedCells: MacroSelectedCell[],
-  errors: string[] = []
-): FreeTableModel {
-  let func;
-  try {
-    func = eval(getMacroFunction[macro.type](macro.code));
-  } catch (err) {
-    errors.push(`Error compiling macro ${macro.name}: ${err.message}`);
-    return data;
-  }
-  if (macro.type == 'transformValue') {
-    return runTramsformValue(func, macroArgs, data, preview, selectedCells, errors);
-  }
-  if (macro.type == 'transformRows') {
-    return runTramsformRows(func, macroArgs, data, preview, selectedCells, errors);
-  }
-  if (macro.type == 'transformData') {
-    // @ts-ignore
-    return runTramsformData(func, macroArgs, data, preview, selectedCells, errors);
-  }
-  return data;
-}
-
 export function compileMacroFunction(macro: MacroDefinition, errors = []) {
   if (!macro) return null;
   let func;
@@ -198,7 +38,7 @@ export function compileMacroFunction(macro: MacroDefinition, errors = []) {
   }
 }
 
-export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, column, errors = []) {
+export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex: number, row, column: string, errors = []) {
   if (!compiledFunc) return value;
   try {
     const res = compiledFunc(value, macroArgs, modules, rowIndex, row, column);
@@ -209,31 +49,62 @@ export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, c
   }
 }
 
+export function runMacroOnRow(compiledFunc, macroArgs, rowIndex: number, row: any, columns: string[], errors = []) {
+  if (!compiledFunc) return row;
+  try {
+    const res = compiledFunc(row, macroArgs, modules, rowIndex, columns);
+    return res;
+  } catch (err) {
+    errors.push(`Error processing row ${rowIndex}: ${err.message}`);
+    return row;
+  }
+}
+
 export function runMacroOnChangeSet(
   macro: MacroDefinition,
   macroArgs: {},
   selectedCells: MacroSelectedCell[],
   changeSet: ChangeSet,
-  display: GridDisplay
+  display: GridDisplay,
+  useRowIndexInsteaOfCondition: boolean
 ): ChangeSet {
   const errors = [];
   const compiledMacroFunc = compileMacroFunction(macro, errors);
   if (!compiledMacroFunc) return null;
 
-  let res = changeSet;
+  if (macro.type == 'transformValue') {
-  for (const cell of selectedCells) {
+    let res = changeSet;
-    const definition = display.getChangeSetField(cell.rowData, cell.column, undefined);
+    for (const cell of selectedCells) {
-    const macroResult = runMacroOnValue(
+      const definition = display.getChangeSetField(
-      compiledMacroFunc,
+        cell.rowData,
-      macroArgs,
+        cell.column,
-      cell.value,
+        undefined,
-      cell.row,
+        useRowIndexInsteaOfCondition ? cell.row : undefined,
-      cell.rowData,
+        useRowIndexInsteaOfCondition
-      cell.column,
+      );
-      errors
+      const macroResult = runMacroOnValue(
-    );
+        compiledMacroFunc,
-    res = setChangeSetValue(res, definition, macroResult);
+        macroArgs,
+        cell.value,
+        cell.row,
+        cell.rowData,
+        cell.column,
+        errors
+      );
+      res = setChangeSetValue(res, definition, macroResult);
+    }
+    return res;
+  }
+  if (macro.type == 'transformRow') {
+    let res = changeSet;
+    const rowIndexes = _.uniq(selectedCells.map(x => x.row));
+    for (const index of rowIndexes) {
+      const rowData = selectedCells.find(x => x.row == index)?.rowData;
+      const columns = _.uniq(selectedCells.map(x => x.column));
+      const definition = display.getChangeSetRow(rowData, null, index, true);
+      const newRow = runMacroOnRow(compiledMacroFunc, macroArgs, index, rowData, columns);
+      res = setChangeSetRowData(res, definition, newRow);
+    }
+    return res;
   }
-
-  return res;
 }

packages/tools/src/DatabaseAnalyser.ts

@@ -235,7 +235,10 @@ export class DatabaseAnalyser {
     if (this.pool.feedback) {
       this.pool.feedback(obj);
     }
-  }
+    if (obj && obj.analysingMessage) {
+        logger.debug(obj.analysingMessage);
+    }
+  } 
 
   async getModifications() {
     const snapshot = await this._getFastSnapshot();

packages/tools/src/SqlDumper.ts

@@ -244,16 +244,7 @@ export class SqlDumper implements AlterProcessor {
       this.put('%i ', col.columnName);
       this.columnDefinition(col);
     });
-    if (table.primaryKey) {
+    this.createTablePrimaryKeyCore(table);
-      this.put(',&n');
-      if (table.primaryKey.constraintName) {
-        this.put('^constraint %i', table.primaryKey.constraintName);
-      }
-      this.put(
-        ' ^primary ^key (%,i)',
-        table.primaryKey.columns.map(x => x.columnName)
-      );
-    }
 
     (table.foreignKeys || []).forEach(fk => {
       this.put(',&n');
@@ -275,6 +266,19 @@ export class SqlDumper implements AlterProcessor {
     });
   }
 
+  createTablePrimaryKeyCore(table: TableInfo) {
+    if (table.primaryKey) {
+      this.put(',&n');
+      if (table.primaryKey.constraintName) {
+        this.put('^constraint %i', table.primaryKey.constraintName);
+      }
+      this.put(
+        ' ^primary ^key (%,i)',
+        table.primaryKey.columns.map(x => x.columnName)
+      );
+    }
+  }
+
   createForeignKeyFore(fk: ForeignKeyInfo) {
     if (fk.constraintName != null) this.put('^constraint %i ', fk.constraintName);
     this.put(

packages/tools/src/createAsyncWriteStream.ts

@@ -14,23 +14,12 @@ export function createAsyncWriteStream(stream, options: AsyncWriteStreamOptions)
   });
 
   writable._write = async (chunk, encoding, callback) => {
-    await options.processItem(chunk);
+    try {
-
+      await options.processItem(chunk);
-    // const { sql, id, newIdSql } = chunk;
+      callback(null);
-    // if (_isArray(sql)) {
+    } catch (err) {
-    //   for (const item of sql) await driver.query(pool, item, { discardResult: true });
+      callback(err);
-    // } else {
+    }
-    //   await driver.query(pool, sql, { discardResult: true });
-    // }
-    // if (newIdSql) {
-    //   const res = await driver.query(pool, newIdSql);
-    //   const resId = Object.entries(res?.rows?.[0])?.[0]?.[1];
-
-    //   if (options?.mapResultId) {
-    //     options?.mapResultId(id, resId as string);
-    //   }
-    // }
-    callback();
   };
 
   // writable._final = async callback => {

packages/tools/src/diffTools.ts

@@ -72,6 +72,34 @@ export function generateTablePairingId(table: TableInfo): TableInfo {
   return table;
 }
 
+export function removeTablePairingId(table: TableInfo): TableInfo {
+  if (!table) return table;
+  return {
+    ...table,
+    columns: table.columns?.map(col => ({
+      ...col,
+      pairingId: undefined,
+    })),
+    foreignKeys: table.foreignKeys?.map(cnt => ({
+      ...cnt,
+      pairingId: undefined,
+    })),
+    checks: table.checks?.map(cnt => ({
+      ...cnt,
+      pairingId: undefined,
+    })),
+    indexes: table.indexes?.map(cnt => ({
+      ...cnt,
+      pairingId: undefined,
+    })),
+    uniques: table.uniques?.map(cnt => ({
+      ...cnt,
+      pairingId: undefined,
+    })),
+    pairingId: undefined,
+  };
+}
+
 function generateObjectPairingId(obj) {
   if (obj.objectTypeField)
     return {
@@ -346,7 +374,13 @@ function createPairs(oldList, newList, additionalCondition = null) {
 function planTablePreload(plan: AlterPlan, oldTable: TableInfo, newTable: TableInfo) {
   const key = newTable.preloadedRowsKey || newTable.primaryKey?.columns?.map(x => x.columnName);
   if (newTable.preloadedRows?.length > 0 && key?.length > 0) {
-    plan.fillPreloadedRows(newTable, oldTable?.preloadedRows, newTable.preloadedRows, key, newTable.preloadedRowsInsertOnly);
+    plan.fillPreloadedRows(
+      newTable,
+      oldTable?.preloadedRows,
+      newTable.preloadedRows,
+      key,
+      newTable.preloadedRowsInsertOnly
+    );
   }
 }
 

packages/tools/src/driverBase.ts

@@ -23,6 +23,7 @@ const dialect = {
 export async function runCommandOnDriver(pool, driver: EngineDriver, cmd: (dmp: SqlDumper) => void): Promise<void> {
   const dmp = driver.createDumper();
   cmd(dmp as any);
+  // console.log('CMD:', dmp.s);
   await driver.query(pool, dmp.s, { discardResult: true });
 }
 
@@ -33,6 +34,7 @@ export async function runQueryOnDriver(
 ): Promise<QueryResult> {
   const dmp = driver.createDumper();
   cmd(dmp as any);
+  // console.log('QUERY:', dmp.s);
   return await driver.query(pool, dmp.s);
 }
 

packages/tools/src/schemaEditorTools.ts

@@ -10,6 +10,14 @@ import type {
   UniqueInfo,
 } from 'dbgate-types';
 import _ from 'lodash';
+import { parseSqlDefaultValue } from './stringTools';
+
+export interface JsonDataObjectUpdateCommand {
+  type: 'renameField' | 'deleteField' | 'setField' | 'setFieldIfNull';
+  oldField?: string;
+  newField?: string;
+  value?: any;
+}
 
 export interface EditorColumnInfo extends ColumnInfo {
   isPrimaryKey?: boolean;
@@ -23,6 +31,41 @@ export function fillEditorColumnInfo(column: ColumnInfo, table: TableInfo): Edit
   };
 }
 
+export function processJsonDataUpdateCommands(obj: any, commands: JsonDataObjectUpdateCommand[] = []) {
+  for (const cmd of commands) {
+    switch (cmd.type) {
+      case 'deleteField':
+        obj = {
+          ...obj,
+        };
+        delete obj[cmd.oldField];
+        break;
+      case 'renameField':
+        obj = {
+          ...obj,
+        };
+        obj[cmd.newField] = obj[cmd.oldField];
+        delete obj[cmd.oldField];
+        break;
+      case 'setField':
+        obj = {
+          ...obj,
+        };
+        obj[cmd.newField] = cmd.value;
+        break;
+      case 'setFieldIfNull':
+        obj = {
+          ...obj,
+        };
+        if (obj[cmd.newField] == null) {
+          obj[cmd.newField] = cmd.value;
+        }
+        break;
+    }
+  }
+  return obj;
+}
+
 function processPrimaryKey(table: TableInfo, oldColumn: EditorColumnInfo, newColumn: EditorColumnInfo): TableInfo {
   if (!oldColumn?.isPrimaryKey && newColumn?.isPrimaryKey) {
     let primaryKey = table?.primaryKey;
@@ -71,7 +114,11 @@ function processPrimaryKey(table: TableInfo, oldColumn: EditorColumnInfo, newCol
   return table;
 }
 
-export function editorAddColumn(table: TableInfo, column: EditorColumnInfo): TableInfo {
+function defineDataCommand(table: TableInfo, cmd: () => JsonDataObjectUpdateCommand) {
+  table['__addDataCommands'] = [...(table['__addDataCommands'] || []), cmd()];
+}
+
+export function editorAddColumn(table: TableInfo, column: EditorColumnInfo, addDataCommand?: boolean): TableInfo {
   let res = {
     ...table,
     columns: [...(table?.columns || []), { ...column, pairingId: uuidv1() }],
@@ -79,10 +126,18 @@ export function editorAddColumn(table: TableInfo, column: EditorColumnInfo): Tab
 
   res = processPrimaryKey(res, null, column);
 
+  if (addDataCommand && column.defaultValue) {
+    defineDataCommand(res, () => ({
+      type: 'setField',
+      newField: column.columnName,
+      value: parseSqlDefaultValue(column.defaultValue),
+    }));
+  }
+
   return res;
 }
 
-export function editorModifyColumn(table: TableInfo, column: EditorColumnInfo): TableInfo {
+export function editorModifyColumn(table: TableInfo, column: EditorColumnInfo, addDataCommand?: boolean): TableInfo {
   const oldColumn = table?.columns?.find(x => x.pairingId == column.pairingId);
 
   let res = {
@@ -91,10 +146,26 @@ export function editorModifyColumn(table: TableInfo, column: EditorColumnInfo):
   };
   res = processPrimaryKey(res, fillEditorColumnInfo(oldColumn, table), column);
 
+  if (addDataCommand && oldColumn.columnName != column.columnName) {
+    defineDataCommand(res, () => ({
+      type: 'renameField',
+      oldField: oldColumn.columnName,
+      newField: column.columnName,
+    }));
+  }
+
+  if (addDataCommand && !oldColumn.defaultValue && column.defaultValue) {
+    defineDataCommand(res, () => ({
+      type: 'setFieldIfNull',
+      newField: column.columnName,
+      value: parseSqlDefaultValue(column.defaultValue),
+    }));
+  }
+
   return res;
 }
 
-export function editorDeleteColumn(table: TableInfo, column: EditorColumnInfo): TableInfo {
+export function editorDeleteColumn(table: TableInfo, column: EditorColumnInfo, addDataCommand?: boolean): TableInfo {
   let res = {
     ...table,
     columns: table.columns.filter(col => col.pairingId != column.pairingId),
@@ -102,6 +173,13 @@ export function editorDeleteColumn(table: TableInfo, column: EditorColumnInfo):
 
   res = processPrimaryKey(res, column, null);
 
+  if (addDataCommand) {
+    defineDataCommand(res, () => ({
+      type: 'deleteField',
+      oldField: column.columnName,
+    }));
+  }
+
   return res;
 }
 

packages/tools/src/stringTools.ts

@@ -115,3 +115,15 @@ export function getAsImageSrc(obj) {
 
   return null;
 }
+
+export function parseSqlDefaultValue(value: string) {
+  if (!value) return undefined;
+  if (!_isString(value)) return undefined;
+  if (value.startsWith("'") && value.endsWith("'")) {
+    return value.slice(1, -1);
+  }
+  if (!isNaN(value as any) && !isNaN(parseFloat(value))) {
+    return parseFloat(value);
+  }
+  return undefined;
+}

packages/types/dialect.d.ts

@@ -11,6 +11,7 @@ export interface SqlDialect {
   anonymousPrimaryKey?: boolean;
   defaultSchemaName?: string;
   enableConstraintsPerTable?: boolean;
+  requireStandaloneSelectForScopeIdentity?: boolean;
 
   dropColumnDependencies?: string[];
   changeColumnDependencies?: string[];

packages/types/extensions.d.ts

@@ -37,8 +37,9 @@ export interface PluginDefinition {
 
 export interface QuickExportDefinition {
   label: string;
-  createWriter: (fileName: string) => { functionName: string; props: any };
+  createWriter: (fileName: string, dataName?: string) => { functionName: string; props: any };
   extension: string;
+  noFilenameDependency?: boolean;
 }
 
 export interface ExtensionsDirectory {

packages/web/src/TabRegister.svelte

@@ -1,16 +1,3 @@
-<script context="module" lang="ts">
-  function createTabComponent(selectedTab) {
-    const tabComponent = tabs[selectedTab.tabComponent]?.default;
-    if (tabComponent) {
-      return {
-        tabComponent,
-        props: selectedTab && selectedTab.props,
-      };
-    }
-    return null;
-  }
-</script>
-
 <script lang="ts">
   import _ from 'lodash';
   import { openedTabs } from './stores';
@@ -40,7 +27,7 @@
     if (selectedTab) {
       const { tabid } = selectedTab;
       if (tabid && !mountedTabs[tabid]) {
-        const newTab = createTabComponent(selectedTab);
+        const newTab = tabs[selectedTab.tabComponent]?.default;
         if (newTab) {
           mountedTabs = {
             ...mountedTabs,
@@ -50,12 +37,14 @@
       }
     }
   }
+
+  $: openedTabsByTabId = _.keyBy($openedTabs, x => x.tabid);
 </script>
 
 {#each _.keys(mountedTabs) as tabid (tabid)}
   <TabContent
-    tabComponent={mountedTabs[tabid].tabComponent}
+    tabComponent={mountedTabs[tabid]}
-    {...mountedTabs[tabid].props}
+    {...openedTabsByTabId[tabid]?.props}
     {tabid}
     tabVisible={tabid == (selectedTab && selectedTab.tabid)}
   />

packages/web/src/appobj/ArchiveFileAppObject.svelte

@@ -54,11 +54,9 @@
     'matview.sql': 'img view',
   };
 
-  function getArchiveIcon(archiveFilesAsDataSheets, data) {
+  function getArchiveIcon(data) {
     if (data.fileType == 'jsonl') {
-      return isArchiveFileMarkedAsDataSheet(archiveFilesAsDataSheets, data.folderName, data.fileName)
+      return 'img archive';
-        ? 'img free-table'
-        : 'img archive';
     }
     return ARCHIVE_ICONS[data.fileType];
   }
@@ -70,20 +68,14 @@
   import ImportExportModal from '../modals/ImportExportModal.svelte';
   import { showModal } from '../modals/modalTools';
 
-  import { archiveFilesAsDataSheets, currentArchive, extensions, getCurrentDatabase, getExtensions } from '../stores';
+  import { getExtensions } from '../stores';
 
   import createQuickExportMenu from '../utility/createQuickExportMenu';
   import { exportQuickExportFile } from '../utility/exportFileTools';
   import openNewTab from '../utility/openNewTab';
   import AppObjectCore from './AppObjectCore.svelte';
-  import getConnectionLabel from '../utility/getConnectionLabel';
   import InputTextModal from '../modals/InputTextModal.svelte';
   import ConfirmModal from '../modals/ConfirmModal.svelte';
-  import {
-    isArchiveFileMarkedAsDataSheet,
-    markArchiveFileAsDataSheet,
-    markArchiveFileAsReadonly,
-  } from '../utility/archiveTools';
   import { apiCall } from '../utility/api';
 
   export let data;
@@ -116,36 +108,12 @@
       },
     });
   };
-  const handleOpenRead = () => {
+  const handleOpenArchive = () => {
-    markArchiveFileAsReadonly(data.folderName, data.fileName);
     openArchive(data.fileName, data.folderName);
   };
-  const handleOpenWrite = () => {
-    markArchiveFileAsDataSheet(data.folderName, data.fileName);
-    openNewTab({
-      title: data.fileName,
-      icon: 'img free-table',
-      tabComponent: 'FreeTableTab',
-      props: {
-        initialArgs: {
-          functionName: 'archiveReader',
-          props: {
-            fileName: data.fileName,
-            folderName: data.folderName,
-          },
-        },
-        archiveFile: data.fileName,
-        archiveFolder: data.folderName,
-      },
-    });
-  };
   const handleClick = () => {
     if (data.fileType == 'jsonl') {
-      if (isArchiveFileMarkedAsDataSheet($archiveFilesAsDataSheets, data.folderName, data.fileName)) {
+      handleOpenArchive();
-        handleOpenWrite();
-      } else {
-        handleOpenRead();
-      }
     }
     if (data.fileType.endsWith('.sql')) {
       handleOpenSqlFile();
@@ -166,8 +134,7 @@
 
   function createMenu() {
     return [
-      data.fileType == 'jsonl' && { text: 'Open (readonly)', onClick: handleOpenRead },
+      data.fileType == 'jsonl' && { text: 'Open', onClick: handleOpenArchive },
-      data.fileType == 'jsonl' && { text: 'Open as data sheet', onClick: handleOpenWrite },
       data.fileType == 'jsonl' && { text: 'Open in text editor', onClick: handleOpenJsonLinesText },
       { text: 'Delete', onClick: handleDelete },
       { text: 'Rename', onClick: handleRename },
@@ -232,7 +199,7 @@
   {...$$restProps}
   {data}
   title={data.fileLabel}
-  icon={getArchiveIcon($archiveFilesAsDataSheets, data)}
+  icon={getArchiveIcon(data)}
   menu={createMenu}
   on:click={handleClick}
 />

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -102,10 +102,6 @@
         isImport: true,
         requiresWriteAccess: true,
       },
-      {
-        label: 'Open as data sheet',
-        isOpenFreeTable: true,
-      },
       {
         label: 'Open active chart',
         isActiveChart: true,
@@ -176,10 +172,6 @@
         isExport: true,
         functionName: 'tableReader',
       },
-      {
-        label: 'Open as data sheet',
-        isOpenFreeTable: true,
-      },
       {
         label: 'Open active chart',
         isActiveChart: true,
@@ -242,10 +234,6 @@
         isExport: true,
         functionName: 'tableReader',
       },
-      {
-        label: 'Open as data sheet',
-        isOpenFreeTable: true,
-      },
       {
         label: 'Open active chart',
         isActiveChart: true,
@@ -409,27 +397,7 @@
       return driver;
     };
 
-    if (menu.isOpenFreeTable) {
+    if (menu.isActiveChart) {
-      const coninfo = await getConnectionInfo(data);
-      openNewTab({
-        title: data.pureName,
-        icon: 'img free-table',
-        tabComponent: 'FreeTableTab',
-        props: {
-          initialArgs: {
-            functionName: 'tableReader',
-            props: {
-              connection: {
-                ...coninfo,
-                database: data.database,
-              },
-              schemaName: data.schemaName,
-              pureName: data.pureName,
-            },
-          },
-        },
-      });
-    } else if (menu.isActiveChart) {
       const driver = await getDriver();
       const dmp = driver.createDumper();
       dmp.put('^select * from %f', data);
@@ -734,6 +702,22 @@
             },
             {
               onClick: () => {
+                // openNewTab(
+                //   {
+                //     tabComponent: 'ImportExportTab',
+                //     title: 'Import/Export',
+                //     icon: 'img export',
+                //   },
+                //   {
+                //     editor: {
+                //       sourceStorageType: 'database',
+                //       sourceConnectionId: data.conid,
+                //       sourceDatabaseName: data.database,
+                //       sourceSchemaName: data.schemaName,
+                //       sourceList: [data.pureName],
+                //     },
+                //   }
+                // );
                 showModal(ImportExportModal, {
                   initialValues: {
                     sourceStorageType: 'database',

packages/web/src/commands/stdCommands.ts

@@ -58,7 +58,7 @@ registerCommand({
   category: 'Theme',
   name: 'Change',
   toolbarName: 'Change theme',
-  onClick: () => showModal(SettingsModal, { selectedTab: 1 }),
+  onClick: () => showModal(SettingsModal, { selectedTab: 2 }),
   // getSubCommands: () => get(extensions).themes.map(themeCommand),
 });
 
@@ -329,21 +329,6 @@ registerCommand({
   },
 });
 
-registerCommand({
-  id: 'new.freetable',
-  category: 'New',
-  icon: 'img markdown',
-  name: 'Data sheet',
-  menuName: 'New data sheet',
-  onClick: () => {
-    openNewTab({
-      title: 'Data #',
-      icon: 'img free-table',
-      tabComponent: 'FreeTableTab',
-    });
-  },
-});
-
 registerCommand({
   id: 'new.jsonl',
   category: 'New',

packages/web/src/datagrid/ChangeSetGrider.ts

@@ -1,4 +1,4 @@
-import type { ChangeSet, MacroDefinition, MacroSelectedCell } from 'dbgate-datalib';
+import { ChangeSet, MacroDefinition, MacroSelectedCell, runMacroOnRow } from 'dbgate-datalib';
 
 import {
   changeSetContainsChanges,
@@ -17,6 +17,7 @@ import {
 } from 'dbgate-datalib';
 import Grider from './Grider';
 import type { GriderRowStatus } from './Grider';
+import _ from 'lodash';
 
 function getRowFromItem(row, matchedChangeSetItem) {
   return matchedChangeSetItem.document
@@ -38,7 +39,7 @@ export default class ChangeSetGrider extends Grider {
   private rowStatusCache;
   private rowDefinitionsCache;
   private batchChangeSet: ChangeSet;
-  private _errors = null;
+  private _errors = [];
   private compiledMacroFunc;
 
   constructor(
@@ -89,7 +90,7 @@ export default class ChangeSetGrider extends Grider {
       this.useRowIndexInsteaOfCondition
     );
     const [matchedField, matchedChangeSetItem] = findExistingChangeSetItem(this.changeSet, rowDefinition);
-    const rowUpdated = matchedChangeSetItem
+    let rowUpdated = matchedChangeSetItem
       ? getRowFromItem(row, matchedChangeSetItem)
       : this.compiledMacroFunc
       ? { ...row }
@@ -105,18 +106,32 @@ export default class ChangeSetGrider extends Grider {
     };
 
     if (this.compiledMacroFunc) {
-      for (const cell of this.selectedCells) {
+      if (this.macro?.type == 'transformValue') {
-        if (cell.row != index) continue;
+        for (const cell of this.selectedCells) {
-        const newValue = runMacroOnValue(
+          if (cell.row != index) continue;
-          this.compiledMacroFunc,
+          const newValue = runMacroOnValue(
-          this.macroArgs,
+            this.compiledMacroFunc,
-          rowUpdated[cell.column],
+            this.macroArgs,
-          index,
+            rowUpdated[cell.column],
-          rowUpdated,
+            index,
-          cell.column,
+            rowUpdated,
-          this._errors
+            cell.column,
-        );
+            this._errors
-        rowUpdated[cell.column] = newValue;
+          );
+          rowUpdated[cell.column] = newValue;
+        }
+      }
+      if (this.macro?.type == 'transformRow') {
+        if (this.selectedCells.find(x => x.row == index)) {
+          rowUpdated = runMacroOnRow(
+            this.compiledMacroFunc,
+            this.macroArgs,
+            index,
+            rowUpdated,
+            _.uniq(this.selectedCells.map(x => x.column)),
+            this._errors
+          );
+        }
       }
     }
 

packages/web/src/datagrid/ColumnManager.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
-  import _, { indexOf, range } from 'lodash';
+  import _, { add, indexOf, range } from 'lodash';
-  import { GridDisplay } from 'dbgate-datalib';
+  import { ChangeSet, DisplayColumn, GridDisplay } from 'dbgate-datalib';
   import { filterName } from 'dbgate-tools';
   import CloseSearchButton from '../buttons/CloseSearchButton.svelte';
 
@@ -14,6 +14,9 @@
   import keycodes from '../utility/keycodes';
   import ColumnManagerRow from './ColumnManagerRow.svelte';
   import { copyTextToClipboard } from '../utility/clipboard';
+  import SelectField from '../forms/SelectField.svelte';
+  import ColumnEditorModal from '../tableeditor/ColumnEditorModal.svelte';
+  import { tick } from 'svelte';
 
   export let managerSize;
   export let display: GridDisplay;
@@ -21,6 +24,9 @@
   export let isDynamicStructure = false;
   export let conid;
   export let database;
+  export let allowChangeChangeSetStructure = false;
+  export let changeSetState: { value: ChangeSet } = null;
+  export let dispatchChangeSet = null;
 
   let filter;
   let domFocusField;
@@ -103,8 +109,67 @@
     selectedColumns = value;
     if (value.length > 0) currentColumnUniqueName = value[0];
   }
+
+  $: tableInfo = display?.editableStructure;
+  $: setTableInfo = updFunc => {
+    const structure = updFunc(display?.editableStructure);
+    let added = [];
+    if (structure['__addDataCommands']) {
+      added = structure['__addDataCommands'];
+      delete structure['__addDataCommands'];
+    }
+    dispatchChangeSet({
+      type: 'set',
+      value: {
+        ...changeSetState?.value,
+        dataUpdateCommands: [...(changeSetState?.value?.dataUpdateCommands || []), ...added],
+        structure,
+      },
+    });
+    tick().then(() => display.reload());
+  };
+
+  $: addDataCommand = allowChangeChangeSetStructure;
+
+  function handleAddColumn() {
+    showModal(ColumnEditorModal, {
+      setTableInfo,
+      tableInfo,
+      addDataCommand,
+      onAddNext: async () => {
+        await tick();
+        handleAddColumn();
+      },
+    });
+  }
 </script>
 
+{#if allowChangeChangeSetStructure}
+  <div class="selectwrap">
+    <SelectField
+      isNative
+      class="colmode"
+      value={isDynamicStructure ? 'variable' : 'fixed'}
+      options={[
+        { label: 'Fixed columns (like SQL)', value: 'fixed' },
+        { label: 'Variable columns (like MongoDB)', value: 'variable' },
+      ]}
+      on:change={e => {
+        dispatchChangeSet({
+          type: 'set',
+          value: {
+            ...changeSetState?.value,
+            structure: {
+              ...display?.editableStructure,
+              __isDynamicStructure: e.detail == 'variable',
+              // __keepDynamicStreamHeader: true,
+            },
+          },
+        });
+      }}
+    />
+  </div>
+{/if}
 <SearchBoxWrapper>
   <SearchInput placeholder="Search columns" bind:value={filter} />
   <CloseSearchButton bind:filter />
@@ -122,6 +187,9 @@
       }}>Add</InlineButton
     >
   {/if}
+  {#if allowChangeChangeSetStructure && !isDynamicStructure}
+    <InlineButton on:click={handleAddColumn}>Add</InlineButton>
+  {/if}
   <InlineButton on:click={() => display.hideAllColumns()}>Hide</InlineButton>
   <InlineButton on:click={() => display.showAllColumns()}>Show</InlineButton>
 </SearchBoxWrapper>
@@ -139,12 +207,19 @@
   />
 
   {#each items as column (column.uniqueName)}
+    {@const columnIndex = items.indexOf(column)}
     <ColumnManagerRow
       {display}
       {column}
       {isJsonView}
+      {isDynamicStructure}
       {conid}
       {database}
+      {tableInfo}
+      {setTableInfo}
+      columnInfo={tableInfo?.columns?.[columnIndex]}
+      {columnIndex}
+      {allowChangeChangeSetStructure}
       isSelected={selectedColumns.includes(column.uniqueName) || currentColumnUniqueName == column.uniqueName}
       on:click={() => {
         if (domFocusField) domFocusField.focus();
@@ -198,4 +273,14 @@
     left: -1000px;
     top: -1000px;
   }
+
+  .selectwrap :global(select) {
+    flex: 1;
+    padding: 3px 0px;
+    border: none;
+  }
+
+  .selectwrap {
+    border-bottom: 1px solid var(--theme-border);
+  }
 </style>

packages/web/src/datagrid/ColumnManagerRow.svelte

@@ -4,6 +4,9 @@
   import FontIcon from '../icons/FontIcon.svelte';
   import ColumnLabel from '../elements/ColumnLabel.svelte';
   import { createEventDispatcher } from 'svelte';
+  import { showModal } from '../modals/modalTools';
+  import ColumnEditorModal from '../tableeditor/ColumnEditorModal.svelte';
+  import { editorDeleteColumn } from 'dbgate-tools';
 
   export let column;
   export let display;
@@ -11,6 +14,28 @@
   export let isSelected = false;
   export let conid;
   export let database;
+  export let isDynamicStructure;
+
+  export let tableInfo;
+  export let setTableInfo;
+
+  export let columnInfo = null;
+  export let columnIndex = -1;
+
+  export let allowChangeChangeSetStructure = false;
+  $: addDataCommand = allowChangeChangeSetStructure;
+
+  function handleEditColumn() {
+    showModal(ColumnEditorModal, { columnInfo, tableInfo, setTableInfo, addDataCommand });
+  }
+
+  function exchange(array, i1, i2) {
+    const i1r = (i1 + array.length) % array.length;
+    const i2r = (i2 + array.length) % array.length;
+    const res = [...array];
+    [res[i1r], res[i2r]] = [res[i2r], res[i1r]];
+    return res;
+  }
 
   const dispatch = createEventDispatcher();
 </script>
@@ -29,32 +54,59 @@
   on:mousemove
   on:mouseup
 >
-  <span class="expandColumnIcon" style={`margin-right: ${5 + (column.uniquePath.length - 1) * 10}px`}>
+  <div>
-    <FontIcon
+    <span class="expandColumnIcon" style={`margin-right: ${5 + (column.uniquePath.length - 1) * 10}px`}>
-      icon={column.isExpandable ? plusExpandIcon(display.isExpandedColumn(column.uniqueName)) : 'icon invisible-box'}
+      <FontIcon
-      on:click={() => display.toggleExpandedColumn(column.uniqueName)}
+        icon={column.isExpandable ? plusExpandIcon(display.isExpandedColumn(column.uniqueName)) : 'icon invisible-box'}
-    />
+        on:click={() => display.toggleExpandedColumn(column.uniqueName)}
-  </span>
+      />
-  {#if isJsonView}
+    </span>
-    <FontIcon icon="img column" />
+    {#if isJsonView}
-  {:else}
+      <FontIcon icon="img column" />
-    <input
+    {:else}
-      type="checkbox"
+      <input
-      checked={column.isChecked}
+        type="checkbox"
-      on:click={e => {
+        checked={column.isChecked}
-        e.stopPropagation();
+        on:click={e => {
-      }}
+          e.stopPropagation();
-      on:mousedown={e => {
+        }}
-        e.stopPropagation();
+        on:mousedown={e => {
-      }}
+          e.stopPropagation();
-      on:change={() => {
+        }}
-        const newValue = !column.isChecked;
+        on:change={() => {
-        display.setColumnVisibility(column.uniquePath, newValue);
+          const newValue = !column.isChecked;
-        dispatch('setvisibility', newValue);
+          display.setColumnVisibility(column.uniquePath, newValue);
-      }}
+          dispatch('setvisibility', newValue);
-    />
+        }}
+      />
+    {/if}
+    <ColumnLabel {...column} showDataType {conid} {database} />
+  </div>
+
+  {#if allowChangeChangeSetStructure && !isDynamicStructure}
+    <div class="nowrap">
+      <span class="icon" on:click={handleEditColumn}>
+        <FontIcon icon="icon edit" />
+      </span>
+      <span class="icon" on:click={() => setTableInfo(info => editorDeleteColumn(info, columnInfo, addDataCommand))}>
+        <FontIcon icon="icon delete" />
+      </span>
+      <span
+        class="icon"
+        on:click={() =>
+          setTableInfo(info => ({ ...info, columns: exchange(info.columns, columnIndex, columnIndex - 1) }))}
+      >
+        <FontIcon icon="icon arrow-up" />
+      </span>
+      <span
+        class="icon"
+        on:click={() =>
+          setTableInfo(info => ({ ...info, columns: exchange(info.columns, columnIndex, columnIndex + 1) }))}
+      >
+        <FontIcon icon="icon arrow-down" />
+      </span>
+    </div>
   {/if}
-  <ColumnLabel {...column} showDataType {conid} {database} />
 </div>
 
 <style>
@@ -63,6 +115,8 @@
     margin-right: 5px;
     cursor: pointer;
     white-space: nowrap;
+    display: flex;
+    justify-content: space-between;
   }
   .row:hover {
     background: var(--theme-bg-hover);
@@ -71,4 +125,13 @@
   .row.isSelected {
     background: var(--theme-bg-selected);
   }
+
+  .icon {
+    position: relative;
+    /* top: 5px;
+    padding: 5px; */
+  }
+  .icon:hover {
+    background-color: var(--theme-bg-3);
+  }
 </style>

packages/web/src/datagrid/DataGrid.svelte

@@ -57,13 +57,12 @@
   import HorizontalSplitter from '../elements/HorizontalSplitter.svelte';
   import VerticalSplitter from '../elements/VerticalSplitter.svelte';
   import FormViewFilters from '../formview/FormViewFilters.svelte';
-  import MacroDetail from '../freetable/MacroDetail.svelte';
+  import MacroDetail from '../macro/MacroDetail.svelte';
-  import MacroManager from '../freetable/MacroManager.svelte';
+  import MacroManager from '../macro/MacroManager.svelte';
   import WidgetColumnBar from '../widgets/WidgetColumnBar.svelte';
   import WidgetColumnBarItem from '../widgets/WidgetColumnBarItem.svelte';
   import ColumnManager from './ColumnManager.svelte';
   import ReferenceManager from './ReferenceManager.svelte';
-  import FreeTableColumnEditor from '../freetable/FreeTableColumnEditor.svelte';
   import JsonViewFilters from '../jsonview/JsonViewFilters.svelte';
   import createActivator, { getActiveComponent } from '../utility/createActivator';
   import _ from 'lodash';
@@ -201,7 +200,6 @@
         height="40%"
         show={freeTableColumn && !isDynamicStructure}
       >
-        <FreeTableColumnEditor {...$$props} {managerSize} />
       </WidgetColumnBarItem>
 
       <WidgetColumnBarItem title="Filters" name="filters" height="30%" show={isFormView}>

packages/web/src/datagrid/DataGridCell.svelte

@@ -7,6 +7,7 @@
   import CellValue from './CellValue.svelte';
   import { showModal } from '../modals/modalTools';
   import EditCellDataModal from '../modals/EditCellDataModal.svelte';
+  import { openJsonLinesData } from '../utility/openJsonLinesData';
 
   export let rowIndex;
   export let col;
@@ -96,20 +97,7 @@
       icon="icon open-in-new"
       on:click={() => {
         if (_.every(jsonParsedValue || value, x => _.isPlainObject(x))) {
-          openNewTab(
+          openJsonLinesData(jsonParsedValue || value);
-            {
-              title: 'Data #',
-              icon: 'img free-table',
-              tabComponent: 'FreeTableTab',
-              props: {},
-            },
-            {
-              editor: {
-                rows: jsonParsedValue || value,
-                structure: { __isDynamicStructure: true, columns: [] },
-              },
-            }
-          );
         } else {
           openJsonDocument(jsonParsedValue || value, undefined, true);
         }

packages/web/src/datagrid/DataGridCore.svelte

@@ -161,7 +161,7 @@
   registerCommand({
     id: 'dataGrid.openJsonArrayInSheet',
     category: 'Data grid',
-    name: 'Open array as data sheet',
+    name: 'Open array as table',
     testEnabled: () => getCurrentDataGrid()?.openJsonArrayInSheetEnabled(),
     onClick: () => getCurrentDataGrid().openJsonArrayInSheet(),
   });
@@ -238,7 +238,7 @@
   registerCommand({
     id: 'dataGrid.openFreeTable',
     category: 'Data grid',
-    name: 'Edit selection as data sheet',
+    name: 'Edit selection as table',
     testEnabled: () => getCurrentDataGrid() != null,
     onClick: () => getCurrentDataGrid().openFreeTable(),
   });
@@ -398,6 +398,7 @@
   import EditCellDataModal, { shouldOpenMultilineDialog } from '../modals/EditCellDataModal.svelte';
   import { getDatabaseInfo, useDatabaseStatus } from '../utility/metadataLoaders';
   import { showSnackbarSuccess } from '../utility/snackbar';
+  import { openJsonLinesData } from '../utility/openJsonLinesData';
 
   export let onLoadNextData = undefined;
   export let grider = undefined;
@@ -645,15 +646,7 @@
   }
 
   export function openFreeTable() {
-    openNewTab(
+    openJsonLinesData(getSelectedFreeDataRows());
-      {
-        title: 'Data #',
-        icon: 'img free-table',
-        tabComponent: 'FreeTableTab',
-        props: {},
-      },
-      { editor: getSelectedFreeData() }
-    );
   }
 
   export function openChartFromSelection() {
@@ -784,20 +777,7 @@
   }
 
   export function openJsonArrayInSheet() {
-    openNewTab(
+    openJsonLinesData(getSelectedDataJson(true));
-      {
-        title: 'Data #',
-        icon: 'img free-table',
-        tabComponent: 'FreeTableTab',
-        props: {},
-      },
-      {
-        editor: {
-          rows: getSelectedDataJson(true),
-          structure: { __isDynamicStructure: true, columns: [] },
-        },
-      }
-    );
   }
 
   export function editJsonEnabled() {
@@ -1109,6 +1089,12 @@
     };
   };
 
+  const getSelectedFreeDataRows = () => {
+    const columns = getSelectedColumns();
+    const rows = getSelectedRowData().map(row => _.pickBy(row, (v, col) => columns.find(x => x.columnName == col)));
+    return rows;
+  };
+
   function getCellsPublished(cells) {
     const regular = cellsToRegularCells(cells);
     const res = regular

packages/web/src/datagrid/JslDataGrid.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
-  import { createGridCache, createGridConfig, JslGridDisplay } from 'dbgate-datalib';
+  import { createGridCache, createGridConfig, JslGridDisplay, runMacro, runMacroOnChangeSet } from 'dbgate-datalib';
+  import { generateTablePairingId, processJsonDataUpdateCommands } from 'dbgate-tools';
   import { writable } from 'svelte/store';
   import JslFormView from '../formview/JslFormView.svelte';
   import { apiOff, apiOn, useApiCall } from '../utility/api';
@@ -16,10 +17,13 @@
   export let changeSetStore = null;
   export let dispatchChangeSet = null;
 
+  export let allowChangeChangeSetStructure = false;
+  export let infoLoadCounter = 0;
+
   let loadedRows;
   let infoCounter = 0;
 
-  $: info = useApiCall('jsldata/get-info', { jslid, infoCounter }, {});
+  $: info = useApiCall('jsldata/get-info', { jslid, infoCounter, infoLoadCounter }, {});
 
   // $: columns = ($info && $info.columns) || [];
   const config = writable(createGridConfig());
@@ -29,6 +33,13 @@
     infoCounter += 1;
   }
 
+  function handleRunMacro(macro, params, cells) {
+    const newChangeSet = runMacroOnChangeSet(macro, params, cells, changeSetState?.value, display, true);
+    if (newChangeSet) {
+      dispatchChangeSet({ type: 'set', value: newChangeSet });
+    }
+  }
+
   $: effect = useEffect(() => onJslId(jslid));
   function onJslId(jslidVal) {
     if (jslidVal && listenInitializeFile) {
@@ -42,15 +53,20 @@
   }
   $: $effect;
 
+  $: infoWithPairingId = generateTablePairingId($info);
+  $: infoUsed = (allowChangeChangeSetStructure && changeSetState?.value?.structure) || infoWithPairingId;
+
+  // $: console.log('infoUsed', infoUsed);
+
   $: display = new JslGridDisplay(
     jslid,
-    $info,
+    infoUsed,
     $config,
     config.update,
     $cache,
     cache.update,
     loadedRows,
-    $info?.__isDynamicStructure,
+    infoUsed?.__isDynamicStructure,
     supportsReload,
     !!changeSetState
   );
@@ -66,10 +82,18 @@
     gridCoreComponent={JslDataGridCore}
     formViewComponent={JslFormView}
     bind:loadedRows
-    isDynamicStructure={$info?.__isDynamicStructure}
+    isDynamicStructure={!!infoUsed?.__isDynamicStructure}
     useEvalFilters
+    showMacros={!!dispatchChangeSet}
+    expandMacros={!!dispatchChangeSet}
+    onRunMacro={handleRunMacro}
+    macroCondition={infoUsed?.__isDynamicStructure ? null : macro => macro.type == 'transformValue'}
     {changeSetState}
     {changeSetStore}
     {dispatchChangeSet}
+    {allowChangeChangeSetStructure}
+    preprocessLoadedRow={changeSetState?.value?.dataUpdateCommands
+      ? row => processJsonDataUpdateCommands(row, changeSetState?.value?.dataUpdateCommands)
+      : null}
   />
 {/key}

packages/web/src/datagrid/JslDataGridCore.svelte

@@ -67,6 +67,9 @@
   export let changeSetState;
   export let dispatchChangeSet;
 
+  export let macroPreview;
+  export let macroValues;
+  export let selectedCellsPublished = () => [];
   export const activator = createActivator('JslDataGridCore', false);
 
   export let loadedRows = [];
@@ -97,16 +100,37 @@
   }
   $: $effect;
 
-  $: grider = new ChangeSetGrider(
+  let grider;
-    loadedRows,
+
-    changeSetState,
+  $: {
-    dispatchChangeSet,
+    if (macroPreview) {
-    display,
+      grider = new ChangeSetGrider(
-    undefined,
+        loadedRows,
-    undefined,
+        changeSetState,
-    undefined,
+        dispatchChangeSet,
-    true
+        display,
-  );
+        macroPreview,
+        macroValues,
+        selectedCellsPublished(),
+        true
+      );
+    }
+  }
+
+  $: {
+    if (!macroPreview) {
+      grider = new ChangeSetGrider(
+        loadedRows,
+        changeSetState,
+        dispatchChangeSet,
+        display,
+        undefined,
+        undefined,
+        undefined,
+        true
+      );
+    }
+  }
 
   // $: grider = new RowsArrayGrider(loadedRows);
 
@@ -168,6 +192,7 @@
   bind:this={domGrid}
   {...$$props}
   bind:loadedRows
+  bind:selectedCellsPublished
   {loadDataPage}
   {dataPageAvailable}
   {loadRowCount}

packages/web/src/datagrid/LoadingDataGridCore.svelte

@@ -15,6 +15,8 @@
   export let selectedCellsPublished;
   export let rowCountLoaded = null;
 
+  export let preprocessLoadedRow = null;
+
   // export let griderFactory;
 
   export let loadedRows = [];
@@ -65,7 +67,8 @@
       errorMessage = nextRows.errorMessage;
     } else {
       if (allRowCount == null) handleLoadRowCount();
-      loadedRows = [...loadedRows, ...nextRows];
+
+      loadedRows = [...loadedRows, ...(preprocessLoadedRow ? nextRows.map(preprocessLoadedRow) : nextRows)];
       isLoadedAll = nextRows.length === 0;
       //   const loadedInfo = {
       //     loadedRows: [...loadedRows, ...nextRows],

packages/web/src/datagrid/TableDataGrid.svelte

@@ -140,7 +140,7 @@
   };
 
   function handleRunMacro(macro, params, cells) {
-    const newChangeSet = runMacroOnChangeSet(macro, params, cells, changeSetState?.value, display);
+    const newChangeSet = runMacroOnChangeSet(macro, params, cells, changeSetState?.value, display, false);
     if (newChangeSet) {
       dispatchChangeSet({ type: 'set', value: newChangeSet });
     }

packages/web/src/elements/Link.svelte

@@ -1,15 +1,18 @@
 <script lang="ts">
   import { openWebLink } from '../utility/exportFileTools';
+  import contextMenu from '../utility/contextMenu';
 
   export let href = undefined;
   export let onClick = undefined;
+  export let menu = '__no_menu';
 </script>
 
 <a
-  on:click={(e) => {
+  on:click={e => {
     if (onClick) onClick(e);
     else openWebLink(href);
   }}
+  use:contextMenu={menu}
 >
   <slot />
 </a>

packages/web/src/freetable/ColumnManagerRow.svelte

@@ -1,50 +0,0 @@
-<script lang="ts">
-  import FontIcon from '../icons/FontIcon.svelte';
-
-  export let column;
-  export let onEdit;
-  export let onRemove;
-  export let onUp;
-  export let onDown;
-</script>
-
-<div class="row">
-  <div class="name">{column.columnName}</div>
-  <div class="nowrap">
-    <span class="icon" on:click={onEdit}>
-      <FontIcon icon="icon edit" />
-    </span>
-    <span class="icon" on:click={onRemove}>
-      <FontIcon icon="icon delete" />
-    </span>
-    <span class="icon" on:click={onUp}>
-      <FontIcon icon="icon arrow-up" />
-    </span>
-    <span class="icon" on:click={onDown}>
-      <FontIcon icon="icon arrow-down" />
-    </span>
-  </div>
-</div>
-
-<style>
-  .row {
-    display: flex;
-    justify-content: space-between;
-    cursor: pointer;
-  }
-  .row:hover {
-    background-color: var(--theme-bg-selected);
-  }
-  .name {
-    white-space: nowrap;
-    margin: 5px;
-  }
-  .icon {
-    position: relative;
-    top: 5px;
-    padding: 5px;
-  }
-  .icon:hover {
-    background-color: var(--theme-bg-3);
-  }
-</style>

packages/web/src/freetable/ColumnNameEditor.svelte

@@ -1,55 +0,0 @@
-<script lang="ts">
-  import { onMount } from 'svelte';
-  import keycodes from '../utility/keycodes';
-
-  export let onEnter;
-  export let onBlur = undefined;
-  export let focusOnCreate = false;
-  export let blurOnEnter = false;
-  export let existingNames;
-  export let defaultValue = '';
-
-  let domEditor;
-  let value = defaultValue || '';
-  $: isError = value && existingNames && existingNames.includes(value);
-
-  const handleKeyDown = event => {
-    if (value && event.keyCode == keycodes.enter && !isError) {
-      onEnter(value);
-      value = '';
-      if (blurOnEnter) domEditor.blur();
-    }
-    if (event.keyCode == keycodes.escape) {
-      value = '';
-      domEditor.blur();
-    }
-  };
-  const handleBlur = () => {
-    if (value && !isError) {
-      onEnter(value);
-      value = '';
-    }
-    if (onBlur) onBlur();
-  };
-  if (focusOnCreate) onMount(() => domEditor.focus());
-</script>
-
-<input
-  type="text"
-  {...$$restProps}
-  bind:value
-  bind:this={domEditor}
-  on:keydown={handleKeyDown}
-  on:blur={handleBlur}
-  class:isError
-/>
-
-<style>
-  input {
-    width: calc(100% - 10px);
-  }
-
-  input.isError {
-    background: var(--theme-bg-red);
-  }
-</style>

packages/web/src/freetable/FreeTableColumnEditor.svelte

@@ -1,82 +0,0 @@
-<script context="module">
-  function dispatchChangeColumns(props, func, rowFunc = null) {
-    const { modelState, dispatchModel } = props;
-    const model = modelState.value;
-
-    dispatchModel({
-      type: 'set',
-      value: {
-        rows: rowFunc ? model.rows.map(rowFunc) : model.rows,
-        structure: {
-          ...model.structure,
-          columns: func(model.structure?.columns),
-        },
-      },
-    });
-  }
-
-  function exchange(array, i1, i2) {
-    const i1r = (i1 + array.length) % array.length;
-    const i2r = (i2 + array.length) % array.length;
-    const res = [...array];
-    [res[i1r], res[i2r]] = [res[i2r], res[i1r]];
-    return res;
-  }
-</script>
-
-<script>
-  import _ from 'lodash';
-  import ManagerInnerContainer from '../elements/ManagerInnerContainer.svelte';
-  import ColumnManagerRow from './ColumnManagerRow.svelte';
-  import ColumnNameEditor from './ColumnNameEditor.svelte';
-
-  export let modelState;
-  export let dispatchModel;
-  export let managerSize;
-
-  let editingColumn = null;
-
-  $: structure = modelState.value.structure;
-</script>
-
-<ManagerInnerContainer width={managerSize}>
-  {#each structure?.columns || [] as column, index}
-    {#if index == editingColumn}
-      <ColumnNameEditor
-        defaultValue={column.columnName}
-        onEnter={columnName => {
-          dispatchChangeColumns(
-            $$props,
-            cols => cols.map((col, i) => (index == i ? { columnName } : col)),
-            row => _.mapKeys(row, (v, k) => (k == column.columnName ? columnName : k))
-          );
-        }}
-        onBlur={() => (editingColumn = null)}
-        focusOnCreate
-        blurOnEnter
-        existingNames={structure?.columns.map(x => x.columnName)}
-      />
-    {:else}
-      <ColumnManagerRow
-        {column}
-        onEdit={() => (editingColumn = index)}
-        onRemove={() => {
-          dispatchChangeColumns($$props, cols => cols.filter((c, i) => i != index));
-        }}
-        onUp={() => {
-          dispatchChangeColumns($$props, cols => exchange(cols, index, index - 1));
-        }}
-        onDown={() => {
-          dispatchChangeColumns($$props, cols => exchange(cols, index, index + 1));
-        }}
-      />
-    {/if}
-  {/each}
-  <ColumnNameEditor
-    onEnter={columnName => {
-      dispatchChangeColumns($$props, cols => [...cols, { columnName }]);
-    }}
-    placeholder="New column"
-    existingNames={(structure?.columns || []).map(x => x.columnName)}
-  />
-</ManagerInnerContainer>

packages/web/src/freetable/FreeTableGridCore.svelte

@@ -1,84 +0,0 @@
-<script context="module" lang="ts">
-  const getCurrentEditor = () => getActiveComponent('FreeTableGridCore');
-
-  registerCommand({
-    id: 'freeTableGrid.export',
-    category: 'Data grid',
-    icon: 'icon export',
-    name: 'Export',
-    keyText: 'CtrlOrCommand+E',
-    testEnabled: () => getCurrentEditor() != null,
-    onClick: () => getCurrentEditor().exportGrid(),
-  });
-</script>
-
-<script lang="ts">
-  import { createGridCache, FreeTableGridDisplay } from 'dbgate-datalib';
-  import { writable } from 'svelte/store';
-  import uuidv1 from 'uuid/v1';
-  import { registerQuickExportHandler } from '../buttons/ToolStripExportButton.svelte';
-  import registerCommand from '../commands/registerCommand';
-
-  import DataGridCore from '../datagrid/DataGridCore.svelte';
-  import ImportExportModal from '../modals/ImportExportModal.svelte';
-  import { showModal } from '../modals/modalTools';
-  import { apiCall } from '../utility/api';
-  import { registerMenu } from '../utility/contextMenu';
-  import createActivator, { getActiveComponent } from '../utility/createActivator';
-  import createQuickExportMenu from '../utility/createQuickExportMenu';
-  import { exportQuickExportFile } from '../utility/exportFileTools';
-  import FreeTableGrider from './FreeTableGrider';
-  import MacroPreviewGrider from './MacroPreviewGrider';
-
-  export let macroPreview;
-  export let modelState;
-  export let dispatchModel;
-  export let macroValues;
-  export let config;
-  export let setConfig;
-  export let selectedCellsPublished;
-
-  export const activator = createActivator('FreeTableGridCore', false);
-
-  const cache = writable(createGridCache());
-
-  $: grider = macroPreview
-    ? new MacroPreviewGrider(modelState.value, macroPreview, macroValues, selectedCellsPublished())
-    : new FreeTableGrider(modelState, dispatchModel);
-  $: display = new FreeTableGridDisplay(grider.model || modelState.value, config, setConfig, $cache, cache.update);
-
-  export async function exportGrid() {
-    const jslid = uuidv1();
-    await apiCall('jsldata/save-free-table', { jslid, data: modelState.value });
-    const initialValues: any = {};
-    initialValues.sourceStorageType = 'jsldata';
-    initialValues.sourceJslId = jslid;
-    initialValues.sourceList = ['editor-data'];
-    initialValues[`columns_editor-data`] = display.getExportColumnMap();
-    showModal(ImportExportModal, { initialValues: initialValues });
-  }
-
-  const quickExportHandler = fmt => async () => {
-    const jslid = uuidv1();
-    await apiCall('jsldata/save-free-table', { jslid, data: modelState.value });
-    exportQuickExportFile(
-      'editor-data',
-      {
-        functionName: 'jslDataReader',
-        props: {
-          jslid,
-        },
-      },
-      fmt,
-      display.getExportColumnMap()
-    );
-  };
-  registerQuickExportHandler(quickExportHandler);
-
-  registerMenu(() => ({
-    ...createQuickExportMenu(quickExportHandler, { command: 'freeTableGrid.export' }),
-    tag: 'export',
-  }));
-</script>
-
-<DataGridCore {...$$props} {grider} {display} frameSelection={!!macroPreview} bind:selectedCellsPublished />

packages/web/src/freetable/FreeTableGrider.ts

@@ -1,104 +0,0 @@
-import type { FreeTableModel } from 'dbgate-datalib';
-import Grider from '../datagrid/Grider';
-
-export default class FreeTableGrider extends Grider {
-  public model: FreeTableModel;
-  private batchModel: FreeTableModel;
-
-  constructor(public modelState, public dispatchModel) {
-    super();
-    this.model = modelState && modelState.value;
-  }
-  getRowData(index: any) {
-    return this.model.rows?.[index];
-  }
-  get rowCount() {
-    return this.model.rows?.length;
-  }
-  get currentModel(): FreeTableModel {
-    return this.batchModel || this.model;
-  }
-  set currentModel(value) {
-    if (this.batchModel) this.batchModel = value;
-    else this.dispatchModel({ type: 'set', value });
-  }
-  setCellValue(index: number, uniqueName: string, value: any) {
-    const model = this.currentModel;
-    if (model.rows[index]) {
-      this.currentModel = {
-        ...model,
-        rows: model.rows.map((row, i) => (index == i ? { ...row, [uniqueName]: value } : row)),
-      };
-    }
-  }
-  setRowData(index: number, document: any) {
-    const model = this.currentModel;
-    if (model.rows[index]) {
-      this.currentModel = {
-        ...model,
-        rows: model.rows.map((row, i) => (index == i ? document : row)),
-      };
-    }
-  }
-  get editable() {
-    return true;
-  }
-  get canInsert() {
-    return true;
-  }
-  get allowSave() {
-    return true;
-  }
-  insertRow(): number {
-    const model = this.currentModel;
-    this.currentModel = {
-      ...model,
-      rows: [...model.rows, {}],
-    };
-    return this.currentModel.rows.length - 1;
-  }
-  insertDocuments(documents: any[]): number {
-    const model = this.currentModel;
-    this.currentModel = {
-      ...model,
-      rows: [...model.rows, ...documents],
-    };
-    return this.currentModel.rows.length - documents.length;
-  }
-
-  deleteRow(index: number) {
-    const model = this.currentModel;
-    this.currentModel = {
-      ...model,
-      rows: model.rows.filter((row, i) => index != i),
-    };
-  }
-  beginUpdate() {
-    this.batchModel = this.model;
-  }
-  endUpdate() {
-    if (this.model != this.batchModel) {
-      this.dispatchModel({ type: 'set', value: this.batchModel });
-      this.batchModel = null;
-    }
-  }
-
-  // static factory({ modelState, dispatchModel }): FreeTableGrider {
-  //   return new FreeTableGrider(modelState, dispatchModel);
-  // }
-  // static factoryDeps({ modelState, dispatchModel }) {
-  //   return [modelState, dispatchModel];
-  // }
-  undo() {
-    this.dispatchModel({ type: 'undo' });
-  }
-  redo() {
-    this.dispatchModel({ type: 'redo' });
-  }
-  get canUndo() {
-    return this.modelState.canUndo;
-  }
-  get canRedo() {
-    return this.modelState.canRedo;
-  }
-}

packages/web/src/icons/FontIcon.svelte

@@ -228,6 +228,8 @@
     'img keydb': 'mdi mdi-key color-icon-blue',
 
     'img duplicator': 'mdi mdi-content-duplicate color-icon-green',
+    'img import': 'mdi mdi-database-import color-icon-green',
+    'img export': 'mdi mdi-database-export color-icon-green',
   };
 </script>
 

packages/web/src/impexp/ImportExportConfigurator.svelte

@@ -22,7 +22,7 @@
     for (const file of getAsArray(files)) {
       const format = findFileFormat(extensions, storage);
       if (format) {
-        await (format.addFileToSourceList || addFileToSourceListDefault)(file, newSources, newValues);
+        await (format.addFileToSourceList || addFileToSourceListDefault)(file, newSources, newValues, apiCall);
       }
     }
     newValues['sourceList'] = [...(values.sourceList || []).filter(x => !newSources.includes(x)), ...newSources];
@@ -58,6 +58,7 @@
   import { showModal } from '../modals/modalTools';
   import { findFileFormat } from '../plugins/fileformats';
   import { extensions } from '../stores';
+  import { apiCall } from '../utility/api';
   import getAsArray from '../utility/getAsArray';
   import { useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders';
   import { setUploadListener } from '../utility/uploadFiles';

packages/web/src/jsontree/JSONTree.svelte

@@ -5,6 +5,7 @@
   import openNewTab from '../utility/openNewTab';
   import _ from 'lodash';
   import { copyTextToClipboard } from '../utility/clipboard';
+  import { openJsonLinesData } from '../utility/openJsonLinesData';
 
   setContext('json-tree-context-key', {});
 
@@ -49,22 +50,9 @@
 
     if (value && _.isArray(value)) {
       res.push({
-        text: 'Open as data sheet',
+        text: 'Open as table',
         onClick: () => {
-          openNewTab(
+          openJsonLinesData(value);
-            {
-              title: 'Data #',
-              icon: 'img free-table',
-              tabComponent: 'FreeTableTab',
-              props: {},
-            },
-            {
-              editor: {
-                rows: value,
-                structure: { __isDynamicStructure: true, columns: [] },
-              },
-            }
-          );
         },
       });
     }

packages/web/src/macro/macros.js

@@ -203,65 +203,17 @@ return !!value;
     ],
     code: `return modules.moment().format(args.format)`,
   },
-  {
-    title: 'Duplicate rows',
-    name: 'duplicateRows',
-    group: 'Tools',
-    description: 'Duplicate selected rows',
-    type: 'transformRows',
-    code: `
-const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
-const selectedRows = modules.lodash.groupBy(selectedCells, 'row');
-const maxIndex = modules.lodash.max(selectedRowIndexes);
-return [
-  ...rows.slice(0, maxIndex + 1),
-  ...selectedRowIndexes.map(index => ({
-    ...modules.lodash.pick(rows[index], selectedRows[index].map(x => x.column)),
-    __rowStatus: 'inserted',
-  })),
-  ...rows.slice(maxIndex + 1),
-]
-    `,
-  },
-  {
-    title: 'Delete empty rows',
-    name: 'deleteEmptyRows',
-    group: 'Tools',
-    description: 'Delete empty rows - rows with all values null or empty string',
-    type: 'transformRows',
-    code: `
-return rows.map(row => {
-  if (cols.find(col => row[col])) return row;
-  return {
-    ...row,
-    __rowStatus: 'deleted',
-  };
-})
-`,
-  },
   {
     title: 'Duplicate columns',
     name: 'duplicateColumns',
     group: 'Tools',
     description: 'Duplicate selected columns',
-    type: 'transformData',
+    type: 'transformRow',
     code: `
-const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column));
+      return {
-const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
+        ...row,
-const addedColumnNames = selectedColumnNames.map(col => (args.prefix || '') + col + (args.postfix || ''));
+        ...modules.lodash.fromPairs(columns.map(col=>[(args.prefix || '') + col + (args.postfix || ''), row[col]]))
-const resultRows = rows.map((row, rowIndex) => ({
+      }
-  ...row,
-  ...(selectedRowIndexes.includes(rowIndex) ? modules.lodash.fromPairs(selectedColumnNames.map(col => [(args.prefix || '') + col + (args.postfix || ''), row[col]])) : {}),
-  __insertedFields: addedColumnNames,
-}));
-const resultCols = [
-  ...cols,
-  ...addedColumnNames,
-];
-return {
-  rows: resultRows,
-  cols: resultCols,
-}
     `,
     args: [
       {
@@ -282,42 +234,27 @@ return {
     name: 'splitColumns',
     group: 'Tools',
     description: 'Split selected columns',
-    type: 'transformData',
+    type: 'transformRow',
     code: `
-const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column));
+       const res = {...row};
-const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
+       for(const col of columns) {
-
+          const value = row[col];
-const addedColumnNames = new Set();
+          if (modules.lodash.isString(value)) {
-
+            const splitted = value.split(args.delimiter);      
-const resultRows = modules.lodash.cloneDeep(rows);
+            splitted.forEach((splitValue, valueIndex) => {
-resultRows.forEach((row, rowIndex) => {
+              const name = col + '_' + (valueIndex + 1).toString();
-  for(const cell of selectedCells) {
+              res[name] = splitValue;
-    if (cell.row == rowIndex && modules.lodash.isString(cell.value)) {
+            });
-      const splitted = cell.value.split(args.delimiter);
+          }
-      splitted.forEach((value, valueIndex) => {
+       }
-        const name = cell.column + '_' + (valueIndex + 1).toString();
+       return res;
-        row[name] = value;
-        addedColumnNames.add(name);
-      });
-    }
-  }
-});
-
-const resultCols = [
-  ...cols,
-  ...addedColumnNames,
-];
-return {
-  rows: resultRows,
-  cols: resultCols,
-}
     `,
     args: [
       {
         type: 'text',
         label: 'Delimiter',
         name: 'delimiter',
-        default: ','
+        default: ',',
       },
     ],
   },
@@ -342,53 +279,34 @@ return {
     name: 'extractDateFields',
     group: 'Tools',
     description: 'Extract yaear, month, day and other date/time fields from selection and adds it as new columns',
-    type: 'transformData',
+    type: 'transformRow',
     code: `
-const selectedColumnNames = modules.lodash.uniq(selectedCells.map(x => x.column));
+      let mom = null;
-const selectedRowIndexes = modules.lodash.uniq(selectedCells.map(x => x.row));
+      for(const col of columns) {
-const addedColumnNames = modules.lodash.compact([args.year, args.month, args.day, args.hour, args.minute, args.second]);
+        const m = modules.moment(row[col]);
-const selectedRows = modules.lodash.groupBy(selectedCells, 'row');
+        if (m.isValid()) {
-const resultRows = rows.map((row, rowIndex) => {
+          mom = m;
-  if (!selectedRowIndexes.includes(rowIndex)) return {
+          break;
-    ...row,
+        }
-    __insertedFields: addedColumnNames,
+      }
-  };
-  let mom = null;
-  for(const cell of selectedRows[rowIndex]) {
-    const m = modules.moment(row[cell.column]);
-    if (m.isValid()) {
-      mom = m;
-      break;
-    }
-  }
-  if (!mom) return {
-    ...row,
-    __insertedFields: addedColumnNames,
-  };
 
-  const fields = {
+      if (!mom) return row;
-    [args.year]: mom.year(),
-    [args.month]: mom.month() + 1,
-    [args.day]: mom.day(),
-    [args.hour]: mom.hour(),
-    [args.minute]: mom.minute(),
-    [args.second]: mom.second(),
-  };
 
-  return {
+      const addedColumnNames = modules.lodash.compact([args.year, args.month, args.day, args.hour, args.minute, args.second]);
-    ...row,
+
-    ...modules.lodash.pick(fields, addedColumnNames),
+      const fields = {
-    __insertedFields: addedColumnNames,
+        [args.year]: mom.year(),
-  }
+        [args.month]: mom.month() + 1,
-});
+        [args.day]: mom.day(),
-const resultCols = [
+        [args.hour]: mom.hour(),
-  ...cols,
+        [args.minute]: mom.minute(),
-  ...addedColumnNames,
+        [args.second]: mom.second(),
-];
+      };
-return {
+    
-  rows: resultRows,
+      return {
-  cols: resultCols,
+        ...row,
-}
+        ...modules.lodash.pick(fields, addedColumnNames),
+      };
     `,
     args: [
       {

packages/web/src/plugins/PluginsProvider.svelte

@@ -1,8 +1,4 @@
 <script lang="ts" context="module">
-  const dbgateEnv = {
-    apiCall,
-  };
-
   async function loadPlugins(pluginsDict, installedPlugins) {
     window['DBGATE_TOOLS'] = dbgateTools;
 
@@ -20,7 +16,6 @@
         const module = eval(`${resp}; plugin`);
         console.log('Loaded plugin', module);
         const moduleContent = module.__esModule ? module.default : module;
-        if (moduleContent.initialize) moduleContent.initialize(dbgateEnv);
         newPlugins[installed.name] = moduleContent;
       }
     }

packages/web/src/stores.ts

@@ -84,7 +84,6 @@ export const pinnedDatabases = writableWithStorage([], 'pinnedDatabases');
 export const pinnedTables = writableWithStorage([], 'pinnedTables');
 export const commandsSettings = writable({});
 export const allResultsInOneTabDefault = writableWithStorage(false, 'allResultsInOneTabDefault');
-export const archiveFilesAsDataSheets = writableWithStorage([], 'archiveFilesAsDataSheets');
 export const commandsCustomized = derived([commands, commandsSettings], ([$commands, $commandsSettings]) =>
   _.mapValues($commands, (v, k) => ({
     // @ts-ignore
@@ -252,3 +251,9 @@ export function subscribeApiDependendStores() {
     }
   });
 }
+
+let currentArchiveValue = null;
+currentArchive.subscribe(value => {
+  currentArchiveValue = value;
+});
+export const getCurrentArchive = () => currentArchiveValue;

packages/web/src/tableeditor/ColumnEditorModal.svelte

@@ -13,10 +13,12 @@
   import { editorAddColumn, editorDeleteColumn, editorModifyColumn, fillEditorColumnInfo } from 'dbgate-tools';
 
   export let columnInfo;
-  export let setTableInfo;
+  export let setTableInfo = null;
-  export let tableInfo;
+  export let tableInfo = null;
   export let onAddNext;
-  export let driver;
+  export let driver = null;
+
+  export let addDataCommand = false;
 </script>
 
 <FormProvider initialValues={fillEditorColumnInfo(columnInfo || {}, tableInfo)}>
@@ -31,7 +33,10 @@
     <FormCheckboxField name="notNull" label="NOT NULL" />
     <FormCheckboxField name="isPrimaryKey" label="Is Primary Key" />
     <FormCheckboxField name="autoIncrement" label="Is Autoincrement" />
-    <FormTextField name="defaultValue" label="Default value. Please use valid SQL expression, eg. 'Hello World' for string value, '' for empty string" />
+    <FormTextField
+      name="defaultValue"
+      label="Default value. Please use valid SQL expression, eg. 'Hello World' for string value, '' for empty string"
+    />
     <FormTextField name="computedExpression" label="Computed expression" />
     {#if driver?.dialect?.columnProperties?.isUnsigned}
       <FormCheckboxField name="isUnsigned" label="Unsigned" />
@@ -52,9 +57,9 @@
         on:click={e => {
           closeCurrentModal();
           if (columnInfo) {
-            setTableInfo(tbl => editorModifyColumn(tbl, e.detail));
+            setTableInfo(tbl => editorModifyColumn(tbl, e.detail, addDataCommand));
           } else {
-            setTableInfo(tbl => editorAddColumn(tbl, e.detail));
+            setTableInfo(tbl => editorAddColumn(tbl, e.detail, addDataCommand));
             if (onAddNext) onAddNext();
           }
         }}
@@ -65,7 +70,7 @@
           value="Save"
           on:click={e => {
             closeCurrentModal();
-            setTableInfo(tbl => editorAddColumn(tbl, e.detail));
+            setTableInfo(tbl => editorAddColumn(tbl, e.detail, addDataCommand));
           }}
         />
       {/if}
@@ -77,7 +82,7 @@
           value="Remove"
           on:click={() => {
             closeCurrentModal();
-            setTableInfo(tbl => editorDeleteColumn(tbl, columnInfo));
+            setTableInfo(tbl => editorDeleteColumn(tbl, columnInfo, addDataCommand));
           }}
         />
       {/if}

packages/web/src/tabs/ArchiveFileTab.svelte

@@ -14,13 +14,25 @@
     testEnabled: () => getCurrentEditor()?.canSave(),
     onClick: () => getCurrentEditor().save(),
   });
+
+  registerCommand({
+    id: 'archiveFile.saveAs',
+    category: 'Archive file',
+    name: 'Save as',
+    icon: 'icon save',
+    isRelatedToTab: true,
+    testEnabled: () => getCurrentEditor() != null,
+    onClick: () => getCurrentEditor().saveAs(),
+  });
 </script>
 
 <script lang="ts">
   import { changeSetContainsChanges, createChangeSet } from 'dbgate-datalib';
-  import { tick } from 'svelte';
+  import localforage from 'localforage';
+  import { onMount, tick } from 'svelte';
 
   import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte';
+  import ToolStripCommandSplitButton from '../buttons/ToolStripCommandSplitButton.svelte';
 
   import ToolStripContainer from '../buttons/ToolStripContainer.svelte';
   import ToolStripExportButton, { createQuickExportHandlerRef } from '../buttons/ToolStripExportButton.svelte';
@@ -29,9 +41,11 @@
   import runCommand from '../commands/runCommand';
 
   import JslDataGrid from '../datagrid/JslDataGrid.svelte';
+  import { showModal } from '../modals/modalTools';
+  import SaveArchiveModal from '../modals/SaveArchiveModal.svelte';
   import useEditorData from '../query/useEditorData';
   import { apiCall } from '../utility/api';
-  import { markTabSaved, markTabUnsaved } from '../utility/common';
+  import { changeTab, markTabSaved, markTabUnsaved, sleep } from '../utility/common';
   import createActivator, { getActiveComponent } from '../utility/createActivator';
   import createUndoReducer from '../utility/createUndoReducer';
 
@@ -42,6 +56,8 @@
   export let jslid = undefined;
 
   export let tabid;
+  let infoLoadCounter = 0;
+  let jslidChecked = false;
 
   const quickExportHandlerRef = createQuickExportHandlerRef();
 
@@ -67,34 +83,96 @@
     }
   }
 
-  export async function save() {
+  export function saveAs() {
-    await apiCall('archive/modify-file', {
+    showModal(SaveArchiveModal, {
       folder: archiveFolder,
       file: archiveFile,
-      changeSet: $changeSetStore.value,
+      onSave: doSaveAs,
     });
+  }
+
+  const doSaveAs = async (folder, file) => {
+    await apiCall('archive/save-jsl-data', {
+      folder,
+      file,
+      jslid: jslid || `archive://${archiveFolder}/${archiveFile}`,
+      changeSet: changeSetContainsChanges($changeSetStore?.value) ? $changeSetStore.value : null,
+    });
+    changeTab(tabid, tab => ({
+      ...tab,
+      title: file,
+      props: { archiveFile: file, archiveFolder: folder },
+      archiveFile: file,
+      archiveFolder: folder,
+    }));
+
+    if (changeSetContainsChanges($changeSetStore?.value)) {
+      await sleep(100);
+      afterSaveChangeSet();
+    }
+  };
+
+  async function afterSaveChangeSet() {
+    const structureChanged = !!$changeSetStore.value?.structure;
     dispatchChangeSet({ type: 'reset', value: createChangeSet() });
+    if (structureChanged) {
+      infoLoadCounter += 1;
+    }
     await tick();
     runCommand('dataGrid.refresh');
   }
 
-  export function canSave() {
+  export async function save() {
-    return changeSetContainsChanges($changeSetStore?.value);
+    if (jslid) {
+      saveAs();
+    } else {
+      await apiCall('archive/modify-file', {
+        folder: archiveFolder,
+        file: archiveFile,
+        changeSet: $changeSetStore.value,
+      });
+      await afterSaveChangeSet();
+    }
   }
+
+  export function canSave() {
+    return jslid || changeSetContainsChanges($changeSetStore?.value);
+  }
+
+  async function checkJslid() {
+    if (jslid) {
+      if (!(await apiCall('jsldata/exists', { jslid }))) {
+        const rows = await localforage.getItem(`tabdata_rows_${tabid}`);
+        if (rows) {
+          await apiCall('jsldata/save-rows', { jslid, rows });
+        }
+      }
+    }
+    jslidChecked = true;
+  }
+
+  onMount(() => {
+    checkJslid();
+  });
 </script>
 
 <ToolStripContainer>
-  <JslDataGrid
+  {#if jslidChecked || !jslid}
-    jslid={jslid || `archive://${archiveFolder}/${archiveFile}`}
+    <JslDataGrid
-    supportsReload
+      jslid={jslid || `archive://${archiveFolder}/${archiveFile}`}
-    changeSetState={$changeSetStore}
+      supportsReload
-    focusOnVisible
+      allowChangeChangeSetStructure
-    {changeSetStore}
+      changeSetState={$changeSetStore}
-    {dispatchChangeSet}
+      focusOnVisible
-  />
+      {changeSetStore}
+      {dispatchChangeSet}
+      {infoLoadCounter}
+    />
+  {/if}
   <svelte:fragment slot="toolstrip">
     <ToolStripCommandButton command="dataGrid.refresh" />
     <ToolStripExportButton command="jslTableGrid.export" {quickExportHandlerRef} />
     <ToolStripCommandButton command="archiveFile.save" />
+    <ToolStripCommandButton command="archiveFile.saveAs" />
   </svelte:fragment>
 </ToolStripContainer>

packages/web/src/tabs/CollectionDataTab.svelte

@@ -154,7 +154,7 @@
   }
 
   function handleRunMacro(macro, params, cells) {
-    const newChangeSet = runMacroOnChangeSet(macro, params, cells, $changeSetStore?.value, display);
+    const newChangeSet = runMacroOnChangeSet(macro, params, cells, $changeSetStore?.value, display, false);
     if (newChangeSet) {
       dispatchChangeSet({ type: 'set', value: newChangeSet });
     }

packages/web/src/tabs/DataDuplicatorTab.svelte

@@ -12,6 +12,16 @@
     testEnabled: () => getCurrentEditor()?.canRun(),
     onClick: () => getCurrentEditor().run(),
   });
+  registerCommand({
+    id: 'dataDuplicator.kill',
+    category: 'Data duplicator',
+    icon: 'icon close',
+    name: 'Kill',
+    toolbar: true,
+    isRelatedToTab: true,
+    testEnabled: () => getCurrentEditor()?.canKill(),
+    onClick: () => getCurrentEditor().kill(),
+  });
 </script>
 
 <script lang="ts">
@@ -22,11 +32,14 @@
   import ToolStripContainer from '../buttons/ToolStripContainer.svelte';
   import invalidateCommands from '../commands/invalidateCommands';
   import registerCommand from '../commands/registerCommand';
+  import Link from '../elements/Link.svelte';
   import ObjectConfigurationControl from '../elements/ObjectConfigurationControl.svelte';
   import TableControl from '../elements/TableControl.svelte';
   import VerticalSplitter from '../elements/VerticalSplitter.svelte';
   import CheckboxField from '../forms/CheckboxField.svelte';
+  import FormFieldTemplateLarge from '../forms/FormFieldTemplateLarge.svelte';
   import SelectField from '../forms/SelectField.svelte';
+  import FontIcon from '../icons/FontIcon.svelte';
   import { extractShellConnection } from '../impexp/createImpExpScript';
   import SocketMessageView from '../query/SocketMessageView.svelte';
   import useEditorData from '../query/useEditorData';
@@ -35,7 +48,11 @@
   import { changeTab } from '../utility/common';
   import createActivator, { getActiveComponent } from '../utility/createActivator';
   import { useArchiveFiles, useArchiveFolders, useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders';
+  import openNewTab from '../utility/openNewTab';
   import useEffect from '../utility/useEffect';
+  import useTimerLabel from '../utility/useTimerLabel';
+  import appObjectTypes from '../appobj';
+  import RowHeaderCell from '../datagrid/RowHeaderCell.svelte';
 
   export let conid;
   export let database;
@@ -47,16 +64,20 @@
 
   export const activator = createActivator('DataDuplicatorTab', true);
 
+  const timerLabel = useTimerLabel();
+
   $: connection = useConnectionInfo({ conid });
   $: dbinfo = useDatabaseInfo({ conid, database });
 
   $: archiveFolders = useArchiveFolders();
   $: archiveFiles = useArchiveFiles({ folder: $editorState?.value?.archiveFolder });
 
-  $: pairedNames = _.intersectionBy(
+  $: pairedNames = _.sortBy(
-    $dbinfo?.tables?.map(x => x.pureName),
+    _.intersectionBy(
-    $archiveFiles?.map(x => x.name),
+      $dbinfo?.tables?.map(x => x.pureName),
-    (x: string) => _.toUpper(x)
+      $archiveFiles?.map(x => x.name),
+      (x: string) => _.toUpper(x)
+    )
   );
 
   $: {
@@ -100,6 +121,10 @@
           operation: row.operation,
           matchColumns: _.compact([row.matchColumn1]),
         })),
+      options: {
+        rollbackAfterFinish: !!$editorState.value?.rollbackAfterFinish,
+        skipRowsWithUnresolvedRefs: !!$editorState.value?.skipRowsWithUnresolvedRefs,
+      },
     });
     return script.getScript();
   }
@@ -117,6 +142,7 @@
     const resp = await apiCall('runners/start', { script });
     runid = resp.runid;
     runnerId = runid;
+    timerLabel.start();
   }
 
   $: effect = useEffect(() => registerRunnerDone(runnerId));
@@ -136,8 +162,20 @@
 
   const handleRunnerDone = () => {
     busy = false;
+    timerLabel.stop();
   };
 
+  export function canKill() {
+    return busy;
+  }
+
+  export function kill() {
+    apiCall('runners/cancel', {
+      runid: runnerId,
+    });
+    timerLabel.stop();
+  }
+
   // $: console.log('$archiveFiles', $archiveFiles);
   // $: console.log('$editorState', $editorState.value);
 
@@ -154,43 +192,115 @@
       isChecked,
       operation,
       matchColumn1,
-      file: `${name}.jsonl`,
+      file: name,
       table: tableInfo?.schemaName ? `${tableInfo?.schemaName}.${tableInfo?.pureName}` : tableInfo?.pureName,
+      schemaName: tableInfo?.schemaName,
+      pureName: tableInfo?.pureName,
+      tableInfo,
     };
   });
 
   // $: console.log('$archiveFolders', $archiveFolders);
+
+  const changeCheckStatus = isChecked => () => {
+    setEditorData(old => {
+      const tables = { ...old?.tables };
+      for (const table of pairedNames) {
+        tables[table] = {
+          ...old?.tables?.[table],
+          isChecked,
+        };
+      }
+      return {
+        ...old,
+        tables,
+      };
+    });
+  };
 </script>
 
 <ToolStripContainer>
-  <VerticalSplitter>
+  <VerticalSplitter initialValue="70%">
     <svelte:fragment slot="1">
       <div class="wrapper">
         <ObjectConfigurationControl title="Configuration">
-          <div class="bold m-2">Source archive</div>
+          <FormFieldTemplateLarge label="Source archive" type="combo">
-          <SelectField
+            <SelectField
-            isNative
+              isNative
-            value={$editorState.value?.archiveFolder}
+              value={$editorState.value?.archiveFolder}
-            on:change={e => {
+              on:change={e => {
-              setEditorData(old => ({
+                setEditorData(old => ({
-                ...old,
+                  ...old,
-                archiveFolder: e.detail,
+                  archiveFolder: e.detail,
-              }));
+                }));
+              }}
+              options={$archiveFolders?.map(x => ({
+                label: x.name,
+                value: x.name,
+              })) || []}
+            />
+          </FormFieldTemplateLarge>
+
+          <FormFieldTemplateLarge
+            label="Dry run - no changes (rollback when finished)"
+            type="checkbox"
+            labelProps={{
+              onClick: () => {
+                setEditorData(old => ({
+                  ...old,
+                  rollbackAfterFinish: !$editorState.value?.rollbackAfterFinish,
+                }));
+              },
             }}
-            options={$archiveFolders?.map(x => ({
+          >
-              label: x.name,
+            <CheckboxField
-              value: x.name,
+              checked={$editorState.value?.rollbackAfterFinish}
-            })) || []}
+              on:change={e => {
-          />
+                setEditorData(old => ({
+                  ...old,
+                  rollbackAfterFinish: e.target.checked,
+                }));
+              }}
+            />
+          </FormFieldTemplateLarge>
+
+          <FormFieldTemplateLarge
+            label="Skip rows with unresolved mandatory references"
+            type="checkbox"
+            labelProps={{
+              onClick: () => {
+                setEditorData(old => ({
+                  ...old,
+                  skipRowsWithUnresolvedRefs: !$editorState.value?.skipRowsWithUnresolvedRefs,
+                }));
+              },
+            }}
+          >
+            <CheckboxField
+              checked={$editorState.value?.skipRowsWithUnresolvedRefs}
+              on:change={e => {
+                setEditorData(old => ({
+                  ...old,
+                  skipRowsWithUnresolvedRefs: e.target.checked,
+                }));
+              }}
+            />
+          </FormFieldTemplateLarge>
         </ObjectConfigurationControl>
 
         <ObjectConfigurationControl title="Imported files">
+          <div class="mb-2">
+            <Link onClick={changeCheckStatus(true)}>Check all</Link>
+            |
+            <Link onClick={changeCheckStatus(false)}>Uncheck all</Link>
+          </div>
+
           <TableControl
             rows={tableRows}
             columns={[
               { header: '', fieldName: 'isChecked', slot: 1 },
-              { header: 'Source file', fieldName: 'file' },
+              { header: 'Source file', fieldName: 'file', slot: 4 },
-              { header: 'Target table', fieldName: 'table' },
+              { header: 'Target table', fieldName: 'table', slot: 5 },
               { header: 'Operation', fieldName: 'operation', slot: 2 },
               { header: 'Match column', fieldName: 'matchColumn1', slot: 3 },
             ]}
@@ -236,6 +346,41 @@
                 />
               {/if}
             </svelte:fragment>
+            <svelte:fragment slot="4" let:row>
+              <Link
+                onClick={() => {
+                  openNewTab({
+                    title: row.file,
+                    icon: 'img archive',
+                    tooltip: `${$editorState.value?.archiveFolder}\n${row.file}`,
+                    tabComponent: 'ArchiveFileTab',
+                    props: {
+                      archiveFile: row.file,
+                      archiveFolder: $editorState.value?.archiveFolder,
+                    },
+                  });
+                }}><FontIcon icon="img archive" /> {row.file}</Link
+              >
+            </svelte:fragment>
+            <svelte:fragment slot="5" let:row>
+              <Link
+                menu={appObjectTypes.DatabaseObjectAppObject.createAppObjectMenu({ ...row.tableInfo, conid, database })}
+                onClick={() => {
+                  openNewTab({
+                    title: row.pureName,
+                    icon: 'img table',
+                    tabComponent: 'TableDataTab',
+                    props: {
+                      schemaName: row.schemaName,
+                      pureName: row.pureName,
+                      conid,
+                      database,
+                      objectTypeField: 'tables',
+                    },
+                  });
+                }}><FontIcon icon="img table" /> {row.table}</Link
+              >
+            </svelte:fragment>
           </TableControl>
         </ObjectConfigurationControl>
       </div>
@@ -247,6 +392,7 @@
 
   <svelte:fragment slot="toolstrip">
     <ToolStripCommandButton command="dataDuplicator.run" />
+    <ToolStripCommandButton command="dataDuplicator.kill" />
   </svelte:fragment>
 </ToolStripContainer>
 

packages/web/src/tabs/FreeTableTab.svelte

@@ -1,169 +0,0 @@
-<script lang="ts" context="module">
-  const getCurrentEditor = () => getActiveComponent('FreeTableTab');
-
-  registerCommand({
-    id: 'freeTable.save',
-    group: 'save',
-    category: 'Table data',
-    name: 'Save',
-    // keyText: 'CtrlOrCommand+S',
-    toolbar: true,
-    isRelatedToTab: true,
-    icon: 'icon save',
-    testEnabled: () => getCurrentEditor() != null,
-    onClick: () => getCurrentEditor().save(),
-  });
-
-  registerCommand({
-    id: 'freeTable.toggleDynamicStructure',
-    category: 'Table data',
-    name: 'Toggle dynamic structure',
-    testEnabled: () => getCurrentEditor() != null,
-    onClick: () => getCurrentEditor().toggleDynamicStructure(),
-  });
-</script>
-
-<script lang="ts">
-  import {
-    analyseCollectionDisplayColumns,
-    createFreeTableModel,
-    FreeTableGridDisplay,
-    runMacro,
-  } from 'dbgate-datalib';
-  import { setContext } from 'svelte';
-  import { writable } from 'svelte/store';
-  import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte';
-  import ToolStripContainer from '../buttons/ToolStripContainer.svelte';
-  import ToolStripExportButton, { createQuickExportHandlerRef } from '../buttons/ToolStripExportButton.svelte';
-  import registerCommand from '../commands/registerCommand';
-  import DataGrid from '../datagrid/DataGrid.svelte';
-  import ErrorInfo from '../elements/ErrorInfo.svelte';
-  import LoadingInfo from '../elements/LoadingInfo.svelte';
-
-  import FreeTableGridCore from '../freetable/FreeTableGridCore.svelte';
-  import { showModal } from '../modals/modalTools';
-  import SaveArchiveModal from '../modals/SaveArchiveModal.svelte';
-  import useEditorData from '../query/useEditorData';
-  import { apiCall } from '../utility/api';
-  import { markArchiveFileAsDataSheet } from '../utility/archiveTools';
-  import { changeTab } from '../utility/common';
-  import { registerMenu } from '../utility/contextMenu';
-  import createActivator, { getActiveComponent } from '../utility/createActivator';
-  import createUndoReducer from '../utility/createUndoReducer';
-  import { getLocalStorage, setLocalStorage } from '../utility/storageCache';
-  import useGridConfig from '../utility/useGridConfig';
-
-  export let tabid;
-  export let initialArgs;
-  export let archiveFolder;
-  export let archiveFile;
-
-  export const activator = createActivator('FreeTableTab', true);
-
-  const config = useGridConfig(tabid);
-  const [modelState, dispatchModel] = createUndoReducer(createFreeTableModel());
-
-  const { setEditorData, editorState } = useEditorData({
-    tabid,
-    loadFromArgs: initialArgs && initialArgs.functionName ? () => apiCall('runners/load-reader', initialArgs) : null,
-    onInitialData: value => {
-      dispatchModel({ type: 'reset', value });
-    },
-  });
-
-  $: isLoading = $editorState.isLoading;
-  $: errorMessage = $editorState.errorMessage;
-
-  $: setEditorData($modelState.value);
-
-  export function save() {
-    showModal(SaveArchiveModal, {
-      folder: archiveFolder,
-      file: archiveFile,
-      onSave: doSave,
-    });
-  }
-
-  const doSave = async (folder, file) => {
-    await apiCall('archive/save-free-table', { folder, file, data: $modelState.value });
-    changeTab(tabid, tab => ({
-      ...tab,
-      title: file,
-      props: { archiveFile: file, archiveFolder: folder },
-      archiveFile: file,
-      archiveFolder: folder,
-    }));
-    archiveFile = file;
-    archiveFolder = folder;
-    markArchiveFileAsDataSheet(folder, file);
-  };
-
-  function handleRunMacro(macro, params, cells) {
-    const newModel = runMacro(macro, params, $modelState.value, false, cells);
-    dispatchModel({ type: 'set', value: newModel });
-  }
-
-  const collapsedLeftColumnStore = writable(getLocalStorage('freeTable_collapsedLeftColumn', false));
-  setContext('collapsedLeftColumnStore', collapsedLeftColumnStore);
-  $: setLocalStorage('freeTable_collapsedLeftColumn', $collapsedLeftColumnStore);
-
-  export function toggleDynamicStructure() {
-    let structure = $modelState.value.structure;
-    structure = { ...structure, __isDynamicStructure: !structure.__isDynamicStructure };
-    if (!structure.__isDynamicStructure) {
-      const columns = analyseCollectionDisplayColumns($modelState.value.rows, display);
-      structure = {
-        ...structure,
-        columns: columns
-          .filter(col => col.uniquePath.length == 1)
-          .map(col => ({
-            columnName: col.uniqueName,
-          })),
-      };
-    }
-    dispatchModel({
-      type: 'set',
-      value: {
-        ...$modelState.value,
-        structure,
-      },
-    });
-  }
-
-  registerMenu(
-    { command: 'freeTable.save', tag: 'save' },
-    { command: 'freeTable.toggleDynamicStructure', tag: 'export' }
-  );
-
-  // display is overridden in FreeTableGridCore, this is because of column manager
-  $: display = new FreeTableGridDisplay($modelState.value, $config, config.update, null, null);
-
-  const quickExportHandlerRef = createQuickExportHandlerRef();
-</script>
-
-{#if isLoading}
-  <LoadingInfo wrapper message="Loading data" />
-{:else if errorMessage}
-  <ErrorInfo message={errorMessage} />
-{:else}
-  <ToolStripContainer>
-    <DataGrid
-      config={$config}
-      setConfig={config.update}
-      modelState={$modelState}
-      {dispatchModel}
-      focusOnVisible
-      gridCoreComponent={FreeTableGridCore}
-      freeTableColumn
-      showMacros
-      expandMacros
-      onRunMacro={handleRunMacro}
-      isDynamicStructure={$modelState.value?.structure?.__isDynamicStructure}
-      {display}
-    />
-    <svelte:fragment slot="toolstrip">
-      <ToolStripCommandButton command="freeTable.save" />
-      <ToolStripExportButton command="freeTableGrid.export" {quickExportHandlerRef} />
-    </svelte:fragment>
-  </ToolStripContainer>
-{/if}

packages/web/src/tabs/ImportExportTab.svelte

@@ -0,0 +1,197 @@
+<script lang="ts">
+  import moment from 'moment';
+  import { writable } from 'svelte/store';
+  import HorizontalSplitter from '../elements/HorizontalSplitter.svelte';
+  import LargeButton from '../buttons/LargeButton.svelte';
+  import LoadingInfo from '../elements/LoadingInfo.svelte';
+  import VerticalSplitter from '../elements/VerticalSplitter.svelte';
+
+  import FormProvider from '../forms/FormProvider.svelte';
+  import FormTextField from '../forms/FormTextField.svelte';
+  import LargeFormButton from '../forms/LargeFormButton.svelte';
+  import FontIcon from '../icons/FontIcon.svelte';
+  import createImpExpScript from '../impexp/createImpExpScript';
+  import ImportExportConfigurator from '../impexp/ImportExportConfigurator.svelte';
+  import PreviewDataGrid from '../impexp/PreviewDataGrid.svelte';
+  import { getDefaultFileFormat } from '../plugins/fileformats';
+  import RunnerOutputFiles from '../query/RunnerOutputFiles.svelte';
+  import SocketMessageView from '../query/SocketMessageView.svelte';
+  import { currentArchive, currentDatabase, extensions, visibleWidgetSideBar, selectedWidget } from '../stores';
+  import { apiCall, apiOff, apiOn } from '../utility/api';
+  import createRef from '../utility/createRef';
+  import openNewTab from '../utility/openNewTab';
+  import useEffect from '../utility/useEffect';
+  import WidgetColumnBar from '../widgets/WidgetColumnBar.svelte';
+  import WidgetColumnBarItem from '../widgets/WidgetColumnBarItem.svelte';
+  import useEditorData from '../query/useEditorData';
+
+  let busy = false;
+  let executeNumber = 0;
+  let runnerId = null;
+
+  const previewReaderStore = writable(null);
+
+  export let tabid;
+  export let initialValues;
+  export let uploadedFile = undefined;
+  export let openedFile = undefined;
+  export let importToCurrentTarget = false;
+
+  const refreshArchiveFolderRef = createRef(null);
+
+  const { editorState, editorValue, setEditorData } = useEditorData({
+    tabid,
+  });
+
+  function detectCurrentTarget() {
+    if (!importToCurrentTarget) return {};
+
+    if ($currentDatabase && $selectedWidget != 'archive') {
+      return {
+        targetStorageType: 'database',
+        targetConnectionId: $currentDatabase?.connection?._id,
+        targetDatabaseName: $currentDatabase?.name,
+      };
+    }
+
+    if ($currentArchive == 'default') {
+      return {
+        targetStorageType: 'archive',
+        targetArchiveFolder: `import-${moment().format('YYYY-MM-DD-hh-mm-ss')}`,
+      };
+    } else {
+      return {
+        targetStorageType: 'archive',
+        targetArchiveFolder: $currentArchive,
+      };
+    }
+  }
+
+  $: effect = useEffect(() => registerRunnerDone(runnerId));
+
+  function registerRunnerDone(rid) {
+    if (rid) {
+      apiOn(`runner-done-${rid}`, handleRunnerDone);
+      return () => {
+        apiOff(`runner-done-${rid}`, handleRunnerDone);
+      };
+    } else {
+      return () => {};
+    }
+  }
+
+  $: $effect;
+
+  const handleRunnerDone = () => {
+    busy = false;
+    if (refreshArchiveFolderRef.get()) {
+      apiCall('archive/refresh-folders', {});
+      apiCall('archive/refresh-files', { folder: refreshArchiveFolderRef.get() });
+      $currentArchive = refreshArchiveFolderRef.get();
+      $selectedWidget = 'archive';
+      $visibleWidgetSideBar = true;
+    }
+  };
+
+  const handleGenerateScript = async e => {
+    const code = await createImpExpScript($extensions, e.detail, undefined, true);
+    openNewTab(
+      {
+        title: 'Shell #',
+        icon: 'img shell',
+        tabComponent: 'ShellTab',
+      },
+      { editor: code }
+    );
+  };
+
+  const handleExecute = async e => {
+    if (busy) return;
+    const values = e.detail;
+    busy = true;
+    const script = await createImpExpScript($extensions, values);
+    executeNumber += 1;
+    let runid = runnerId;
+    const resp = await apiCall('runners/start', { script });
+    runid = resp.runid;
+    runnerId = runid;
+
+    if (values.targetStorageType == 'archive') {
+      refreshArchiveFolderRef.set(values.targetArchiveFolder);
+    } else {
+      refreshArchiveFolderRef.set(null);
+    }
+  };
+
+  const handleCancel = () => {
+    apiCall('runners/cancel', {
+      runid: runnerId,
+    });
+  };
+</script>
+
+<FormProvider
+  initialValues={{
+    sourceStorageType: 'database',
+    targetStorageType: getDefaultFileFormat($extensions).storageType,
+    targetArchiveFolder: $currentArchive,
+    sourceArchiveFolder: $currentArchive,
+    ...detectCurrentTarget(),
+    ...initialValues,
+  }}
+>
+  <HorizontalSplitter initialValue="70%">
+    <div class="content" slot="1">
+      <ImportExportConfigurator {uploadedFile} {openedFile} {previewReaderStore} />
+
+      {#if busy}
+        <LoadingInfo wrapper message="Processing import/export ..." />
+      {/if}
+    </div>
+
+    <svelte:fragment slot="2">
+      <WidgetColumnBar>
+        <WidgetColumnBarItem title="Output files" name="output" height="20%">
+          <RunnerOutputFiles {runnerId} {executeNumber} />
+        </WidgetColumnBarItem>
+        <WidgetColumnBarItem title="Messages" name="messages">
+          <SocketMessageView
+            eventName={runnerId ? `runner-info-${runnerId}` : null}
+            {executeNumber}
+            showNoMessagesAlert
+          />
+        </WidgetColumnBarItem>
+        <WidgetColumnBarItem title="Preview" name="preview" skip={!$previewReaderStore}>
+          <PreviewDataGrid reader={$previewReaderStore} />
+        </WidgetColumnBarItem>
+        <WidgetColumnBarItem title="Advanced configuration" name="config" collapsed>
+          <FormTextField label="Schedule" name="schedule" />
+          <FormTextField label="Start variable index" name="startVariableIndex" />
+        </WidgetColumnBarItem>
+      </WidgetColumnBar>
+    </svelte:fragment>
+  </HorizontalSplitter>
+
+  <!-- <svelte:fragment slot="footer">
+        <div class="flex m-2">
+          {#if busy}
+            <LargeButton icon="icon stop" on:click={handleCancel}>Stop</LargeButton>
+          {:else}
+            <LargeFormButton on:click={handleExecute} icon="icon run">Run</LargeFormButton>
+          {/if}
+          <LargeFormButton icon="img sql-file" on:click={handleGenerateScript}>Generate script</LargeFormButton>
+  
+          <LargeButton on:click={closeCurrentModal} icon="icon close">Close</LargeButton>
+        </div>
+      </svelte:fragment> -->
+</FormProvider>
+
+<style>
+  .content {
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+    overflow-y: auto;
+    overflow-x: hidden;
+  }
+</style>

packages/web/src/tabs/index.js

@@ -5,7 +5,6 @@ import * as TableStructureTab from './TableStructureTab.svelte';
 import * as QueryTab from './QueryTab.svelte';
 import * as ShellTab from './ShellTab.svelte';
 import * as ArchiveFileTab from './ArchiveFileTab.svelte';
-import * as FreeTableTab from './FreeTableTab.svelte';
 import * as PluginTab from './PluginTab.svelte';
 import * as ChartTab from './ChartTab.svelte';
 import * as MarkdownEditorTab from './MarkdownEditorTab.svelte';
@@ -29,6 +28,7 @@ import * as PerspectiveTab from './PerspectiveTab.svelte';
 import * as ServerSummaryTab from './ServerSummaryTab.svelte';
 import * as ProfilerTab from './ProfilerTab.svelte';
 import * as DataDuplicatorTab from './DataDuplicatorTab.svelte';
+import * as ImportExportTab from './ImportExportTab.svelte';
 
 export default {
   TableDataTab,
@@ -38,7 +38,6 @@ export default {
   QueryTab,
   ShellTab,
   ArchiveFileTab,
-  FreeTableTab,
   PluginTab,
   ChartTab,
   MarkdownEditorTab,
@@ -62,4 +61,5 @@ export default {
   ServerSummaryTab,
   ProfilerTab,
   DataDuplicatorTab,
+  ImportExportTab,
 };

packages/web/src/utility/archiveTools.ts

@@ -1,15 +0,0 @@
-import { archiveFilesAsDataSheets } from '../stores';
-
-export function markArchiveFileAsDataSheet(folder, file) {
-  archiveFilesAsDataSheets.update(ar =>
-    ar.find(x => x.folder == folder && x.file == file) ? ar : [...ar, { folder, file }]
-  );
-}
-
-export function markArchiveFileAsReadonly(folder, file) {
-  archiveFilesAsDataSheets.update(ar => ar.filter(x => x.folder != folder || x.file != file));
-}
-
-export function isArchiveFileMarkedAsDataSheet(store, folder, file) {
-  return !!store.find(x => x.folder == folder && x.file == file);
-}

packages/web/src/utility/createQuickExportMenu.ts

@@ -1,5 +1,5 @@
 import type { QuickExportDefinition } from 'dbgate-types';
-import { getExtensions } from '../stores';
+import { currentArchive, getCurrentArchive, getExtensions } from '../stores';
 
 export function createQuickExportMenuItems(handler: (fmt: QuickExportDefinition) => Function, advancedExportMenuItem) {
   const extensions = getExtensions();
@@ -9,6 +9,22 @@ export function createQuickExportMenuItems(handler: (fmt: QuickExportDefinition)
       onClick: handler(fmt),
     })),
     { divider: true },
+    {
+      text: 'Current archive',
+      onClick: handler({
+        extension: 'jsonl',
+        label: 'Current archive',
+        noFilenameDependency: true,
+        createWriter: (fileName, dataName) => ({
+          functionName: 'archiveWriter',
+          props: {
+            fileName: dataName,
+            folderName: getCurrentArchive(),
+          },
+        }),
+      }),
+    },
+    { divider: true },
     {
       text: 'More...',
       ...advancedExportMenuItem,

packages/web/src/utility/exportFileTools.ts

@@ -7,6 +7,7 @@ import { normalizeExportColumnMap } from '../impexp/createImpExpScript';
 import { getCurrentConfig } from '../stores';
 import { showModal } from '../modals/modalTools';
 import RunScriptModal from '../modals/RunScriptModal.svelte';
+import { QuickExportDefinition } from 'dbgate-types';
 
 export async function importSqlDump(inputFile, connection) {
   const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson();
@@ -117,35 +118,53 @@ export async function saveExportedFile(filters, defaultPath, extension, dataName
   });
 }
 
-export async function exportQuickExportFile(dataName, reader, format, columnMap = null) {
+function generateQuickExportScript(
-  await saveExportedFile(
+  reader,
-    [{ name: format.label, extensions: [format.extension] }],
+  format: QuickExportDefinition,
-    `${dataName}.${format.extension}`,
+  filePath: string,
-    format.extension,
+  dataName: string,
-    dataName,
+  columnMap
-    filePath => {
+) {
-      const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson();
+  const script = getCurrentConfig().allowShellScripting ? new ScriptWriter() : new ScriptWriterJson();
 
-      const sourceVar = script.allocVariable();
+  const sourceVar = script.allocVariable();
-      script.assign(sourceVar, reader.functionName, reader.props);
+  script.assign(sourceVar, reader.functionName, reader.props);
 
-      const targetVar = script.allocVariable();
+  const targetVar = script.allocVariable();
-      const writer = format.createWriter(filePath, dataName);
+  const writer = format.createWriter(filePath, dataName);
-      script.assign(targetVar, writer.functionName, writer.props);
+  script.assign(targetVar, writer.functionName, writer.props);
 
-      const colmap = normalizeExportColumnMap(columnMap);
+  const colmap = normalizeExportColumnMap(columnMap);
-      let colmapVar = null;
+  let colmapVar = null;
-      if (colmap) {
+  if (colmap) {
-        colmapVar = script.allocVariable();
+    colmapVar = script.allocVariable();
-        script.assignValue(colmapVar, colmap);
+    script.assignValue(colmapVar, colmap);
-      }
+  }
 
-      script.copyStream(sourceVar, targetVar, colmapVar);
+  script.copyStream(sourceVar, targetVar, colmapVar);
-      script.endLine();
+  script.endLine();
 
-      return script.getScript();
+  return script.getScript();
-    }
+}
-  );
+
+export async function exportQuickExportFile(dataName, reader, format: QuickExportDefinition, columnMap = null) {
+  if (format.noFilenameDependency) {
+    const script = generateQuickExportScript(reader, format, null, dataName, columnMap);
+    runImportExportScript({
+      script,
+      runningMessage: `Exporting ${dataName}`,
+      canceledMessage: `Export ${dataName} canceled`,
+      finishedMessage: `Export ${dataName} finished`,
+    });
+  } else {
+    await saveExportedFile(
+      [{ name: format.label, extensions: [format.extension] }],
+      `${dataName}.${format.extension}`,
+      format.extension,
+      dataName,
+      filePath => generateQuickExportScript(reader, format, filePath, dataName, columnMap)
+    );
+  }
 }
 
 // export async function exportSqlDump(connection, databaseName) {

packages/web/src/utility/openJsonLinesData.ts

@@ -0,0 +1,22 @@
+import uuidv1 from 'uuid/v1';
+import { apiCall } from './api';
+import openNewTab from './openNewTab';
+
+export async function openJsonLinesData(rows) {
+  const jslid = uuidv1();
+
+  // await apiCall('jsldata/save-rows', { jslid, rows });
+  openNewTab(
+    {
+      tabComponent: 'ArchiveFileTab',
+      icon: 'img archive',
+      title: 'Data #',
+      props: {
+        jslid,
+      },
+    },
+    {
+      rows,
+    }
+  );
+}

packages/web/src/utility/openNewTab.ts

@@ -65,7 +65,7 @@ export default async function openNewTab(newTab, initialData = undefined, option
   const tabid = uuidv1();
   if (initialData) {
     for (const key of _.keys(initialData)) {
-      if (key == 'editor') {
+      if (key == 'editor' || key == 'rows') {
         await localforage.setItem(`tabdata_${key}_${tabid}`, initialData[key]);
       } else {
         localStorage.setItem(`tabdata_${key}_${tabid}`, JSON.stringify(initialData[key]));

packages/web/src/widgets/AppFilesList.svelte

@@ -33,7 +33,6 @@
   import newQuery from '../query/newQuery';
   import { currentApplication } from '../stores';
   import { apiCall } from '../utility/api';
-  import { markArchiveFileAsDataSheet } from '../utility/archiveTools';
   import { useAppFiles, useArchiveFolders } from '../utility/metadataLoaders';
   import openNewTab from '../utility/openNewTab';
   import WidgetsInnerContainer from './WidgetsInnerContainer.svelte';

packages/web/src/widgets/ArchiveFilesList.svelte

@@ -29,7 +29,6 @@
   import { showModal } from '../modals/modalTools';
   import { currentArchive } from '../stores';
   import { apiCall } from '../utility/api';
-  import { markArchiveFileAsDataSheet } from '../utility/archiveTools';
   import { useArchiveFiles, useArchiveFolders } from '../utility/metadataLoaders';
   import openNewTab from '../utility/openNewTab';
   import WidgetsInnerContainer from './WidgetsInnerContainer.svelte';
@@ -43,31 +42,26 @@
     apiCall('archive/refresh-files', { folder });
   };
 
-  function handleNewDataSheet() {
+  function handleNewJsonLines() {
     showModal(InputTextModal, {
       value: '',
       label: 'New file name',
-      header: 'Create new data sheet',
+      header: 'Create new JSON lines',
       onConfirm: async file => {
-        await apiCall('archive/save-free-table', {
+        await apiCall('archive/save-rows', {
           folder: $currentArchive,
           file,
-          data: createFreeTableModel(),
+          rows: [
+            { id: 1, value: 'val1' },
+            { id: 1, value: 'val2' },
+          ],
         });
-        markArchiveFileAsDataSheet($currentArchive, file);
 
         openNewTab({
           title: file,
-          icon: 'img free-table',
+          icon: 'img archive',
-          tabComponent: 'FreeTableTab',
+          tabComponent: 'ArchiveFileTab',
           props: {
-            initialArgs: {
-              functionName: 'archiveReader',
-              props: {
-                fileName: file,
-                folderName: $currentArchive,
-              },
-            },
             archiveFile: file,
             archiveFolder: $currentArchive,
           },
@@ -77,7 +71,7 @@
   }
 
   function createAddMenu() {
-    return [{ text: 'New data sheet', onClick: handleNewDataSheet }];
+    return [{ text: 'New NDJSON file', onClick: handleNewJsonLines }];
   }
 </script>
 

packages/web/src/widgets/StatusBar.svelte

@@ -6,7 +6,14 @@
 
   import FontIcon from '../icons/FontIcon.svelte';
 
-  import { activeTabId, currentDatabase, currentThemeDefinition, visibleCommandPalette } from '../stores';
+  import {
+    activeTabId,
+    currentArchive,
+    currentDatabase,
+    currentThemeDefinition,
+    selectedWidget,
+    visibleCommandPalette,
+  } from '../stores';
   import getConnectionLabel from '../utility/getConnectionLabel';
   import { useConnectionList, useDatabaseServerVersion, useDatabaseStatus } from '../utility/metadataLoaders';
   import { findCommand } from '../commands/runCommand';
@@ -140,6 +147,18 @@
         </div>
       </div>
     {/if}
+    {#if $currentArchive}
+      <div
+        class="item flex clickable"
+        title="Current archive"
+        on:click={() => {
+          $selectedWidget = 'archive';
+        }}
+      >
+        <FontIcon icon="icon archive" padRight />
+        {$currentArchive}
+      </div>
+    {/if}
   </div>
   <div class="container">
     {#each contextItems || [] as item}

plugins/dbgate-plugin-excel/package.json

@@ -33,7 +33,7 @@
   },
   "devDependencies": {
     "lodash": "^4.17.21",
-    "xlsx": "^0.16.8",
+    "xlsx": "^0.18.5",
     "dbgate-plugin-tools": "^1.0.7",
     "webpack": "^4.42.0",
     "webpack-cli": "^3.3.11"

plugins/dbgate-plugin-excel/src/frontend/index.js

@@ -1,9 +1,3 @@
-let dbgateEnv;
-
-function initialize(dbgateEnv) {
-  dbgateEnv = dbgateEnv;
-}
-
 const fileFormat = {
   packageName: 'dbgate-plugin-excel',
   // file format identifier
@@ -17,8 +11,8 @@ const fileFormat = {
   // function name from backend, which contains writer factory, postfixed by package name
   writerFunc: 'writer@dbgate-plugin-excel',
 
-  addFileToSourceList: async ({ fileName }, newSources, newValues) => {
+  addFileToSourceList: async ({ fileName }, newSources, newValues, apiCall) => {
-    const resp = await dbgateEnv.apiCall('plugins/command', {
+    const resp = await apiCall('plugins/command', {
       command: 'analyse',
       packageName: 'dbgate-plugin-excel',
       args: {
@@ -85,5 +79,4 @@ export default {
       }),
     },
   ],
-  initialize,
 };

plugins/dbgate-plugin-mysql/src/frontend/drivers.js

@@ -42,6 +42,7 @@ const dialect = {
   dropCheck: true,
 
   dropReferencesWhenDropTable: false,
+  requireStandaloneSelectForScopeIdentity: true,
 
   columnProperties: {
     columnComment: true,

plugins/dbgate-plugin-oracle/package.json

@@ -32,7 +32,7 @@
   "devDependencies": {
     "dbgate-plugin-tools": "^1.0.8",
     "dbgate-query-splitter": "^4.9.0",
-    "dbgate-tools": "^5.1.6",
+    "dbgate-tools": "^5.0.0-alpha.1",
     "lodash": "^4.17.21",
     "webpack": "^4.42.0",
     "webpack-cli": "^3.3.11"

plugins/dbgate-plugin-oracle/src/backend/Analyser.js

@@ -104,7 +104,7 @@ class Analyser extends DatabaseAnalyser {
     const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']);
     this.feedback({ analysingMessage: 'Finalizing DB structure' });
 
-    const columnColumnsMapped = fkColumns.rows.map(x => ({
+    const fkColumnsMapped = fkColumns.rows.map(x => ({
       pureName: x.pure_name,
       schemaName: x.schema_name,
       constraintSchema: x.constraint_schema,
@@ -124,6 +124,9 @@ class Analyser extends DatabaseAnalyser {
       columnName: x.column_name,
     }));
 
+    const columnGroup = col => `${col.schema_name}||${col.pure_name}`;
+    const columnsGrouped = _.groupBy(columns.rows, columnGroup);
+
     const res = {
       tables: tables.rows.map(table => {
         const newTable = {
@@ -134,11 +137,11 @@ class Analyser extends DatabaseAnalyser {
         };
         return {
           ...newTable,
-          columns: columns.rows
+          columns: (columnsGrouped[columnGroup(table)] || []).map(col =>
-            .filter(col => col.pure_name == table.pure_name && col.schema_name == table.schema_name)
+            getColumnInfo(col, newTable, geometryColumns, geographyColumns)
-            .map(col => getColumnInfo(col, newTable, geometryColumns, geographyColumns)),
+          ),
           primaryKey: DatabaseAnalyser.extractPrimaryKeys(newTable, pkColumnsMapped),
-          foreignKeys: DatabaseAnalyser.extractForeignKeys(newTable, columnColumnsMapped),
+          foreignKeys: DatabaseAnalyser.extractForeignKeys(newTable, fkColumnsMapped),
           indexes: _.uniqBy(
             indexes.rows.filter(
               idx =>
@@ -176,9 +179,7 @@ class Analyser extends DatabaseAnalyser {
         schemaName: view.schema_name,
         contentHash: view.hash_code,
         createSql: `CREATE VIEW "${view.schema_name}"."${view.pure_name}"\nAS\n${view.create_sql}`,
-        columns: columns.rows
+        columns: (columnsGrouped[columnGroup(view)] || []).map(col => getColumnInfo(col)),
-          .filter(col => col.pure_name == view.pure_name && col.schema_name == view.schema_name)
-          .map(col => getColumnInfo(col)),
       })),
       matviews: matviews
         ? matviews.rows.map(matview => ({
@@ -212,6 +213,9 @@ class Analyser extends DatabaseAnalyser {
         })),
     };
 
+    // this.feedback({ analysingMessage: 'Debug sleep' });
+    // await new Promise(resolve => setTimeout(resolve, 90 * 1000));
+
     this.feedback({ analysingMessage: null });
 
     return res;

plugins/dbgate-plugin-oracle/src/backend/drivers.js

@@ -52,52 +52,10 @@ const drivers = driverBases.map(driverBase => ({
     authType,
     socketPath,
   }) {
-    let options = null;
+    client = await oracledb.getConnection({
-
+      user,
-    if (engine == 'redshift@dbgate-plugin-oracle') {
+      password,
-      let url = databaseUrl;
+      connectString: useDatabaseUrl ? databaseUrl : port ? `${server}:${port}` : server,
-      if (url && url.startsWith('jdbc:redshift://')) {
-        url = url.substring('jdbc:redshift://'.length);
-      }
-      if (user && password) {
-        url = `oracle://${user}:${password}@${url}`;
-      } else if (user) {
-        url = `oracle://${user}@${url}`;
-      } else {
-        url = `oracle://${url}`;
-      }
-
-      options = {
-        connectionString: url,
-      };
-    } else {
-      options = useDatabaseUrl
-        ? {
-            connectionString: databaseUrl,
-          }
-        : {
-            host: authType == 'socket' ? socketPath || driverBase.defaultSocketPath : server,
-            port: authType == 'socket' ? null : port,
-            user,
-            password,
-            database: database || 'oracle',
-            ssl,
-          };
-    }
-
-    // console.log('OPTIONS', options);
-/*
-    const client = new pg.Client(options);
-    await client.connect();
-
-    if (isReadOnly) {
-      await this.query(client, 'SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY');
-    }
-*/
-  client = await oracledb.getConnection( {
-      user          : options.user,
-      password      : options.password,
-      connectString : options.host
     });
     return client;
   },
@@ -105,28 +63,25 @@ const drivers = driverBases.map(driverBase => ({
     return pool.end();
   },
   async query(client, sql) {
-      //console.log('query sql', sql);
+    //console.log('query sql', sql);
     if (sql == null) {
       return {
         rows: [],
         columns: [],
       };
     }
-try {
+    try {
       //console.log('sql3', sql);
-    const res = await client.execute(sql);
+      const res = await client.execute(sql);
-    //console.log('res', res);
+      //console.log('res', res);
-    const columns = extractOracleColumns(res.metaData);
+      const columns = extractOracleColumns(res.metaData);
-     //console.log('columns', columns);
+      //console.log('columns', columns);
-    return { rows: (res.rows || []).map(row => zipDataRow(row, columns)), columns };
+      return { rows: (res.rows || []).map(row => zipDataRow(row, columns)), columns };
-}
+    } catch (err) {
-catch(err) {
+      console.log('Error query', err, sql);
-  console.log('Error query', err, sql);
+    } finally {
-}
+      //console.log('finally', sql);
-finally {
+    }
-  //console.log('finally', sql);
-}
-
   },
   stream(client, sql, options) {
     /*
@@ -137,8 +92,8 @@ finally {
 */
     // console.log('queryStream', sql);
     const query = client.queryStream(sql);
-   // const consumeStream = new Promise((resolve, reject) => {
+    // const consumeStream = new Promise((resolve, reject) => {
-      let rowcount = 0;
+    let rowcount = 0;
     let wasHeader = false;
 
     query.on('metadata', row => {
@@ -200,13 +155,12 @@ finally {
       });
       options.done();
     });
-     query.on('close', function() {
+    query.on('close', function () {
-        //console.log("stream 'close' event");
+      //console.log("stream 'close' event");
-        // The underlying ResultSet has been closed, so the connection can now
+      // The underlying ResultSet has been closed, so the connection can now
-        // be closed, if desired.  Note: do not close connections on 'end'.
+      // be closed, if desired.  Note: do not close connections on 'end'.
-        //resolve(rowcount);
+      //resolve(rowcount);
-        ;
+    });
-      });
     //});
 
     //const numrows = await consumeStream;
@@ -215,7 +169,7 @@ finally {
   },
   async getVersion(client) {
     //const { rows } = await this.query(client, "SELECT banner as version FROM v$version WHERE banner LIKE 'Oracle%'");
-    const { rows } = await this.query(client, "SELECT version as \"version\" FROM v$instance");
+    const { rows } = await this.query(client, 'SELECT version as "version" FROM v$instance');
     const { version } = rows[0];
 
     const isCockroach = false; //version.toLowerCase().includes('cockroachdb');
@@ -245,7 +199,7 @@ finally {
     };
   },
   async readQuery(client, sql, structure) {
-/*
+    /*
     const query = new pg.Query({
       text: sql,
       rowMode: 'array',
@@ -267,10 +221,10 @@ finally {
       if (!wasHeader) {
         columns = extractOracleColumns(row);
         if (columns && columns.length > 0) {
-        pass.write({
+          pass.write({
-          __isStreamHeader: true,
+            __isStreamHeader: true,
-          ...(structure || { columns }),
+            ...(structure || { columns }),
-        });
+          });
         }
         wasHeader = true;
       }
@@ -301,7 +255,7 @@ finally {
     return createBulkInsertStreamBase(this, stream, pool, name, options);
   },
   async listDatabases(client) {
-    const { rows } = await this.query(client, 'SELECT instance_name AS \"name\" FROM v$instance');
+    const { rows } = await this.query(client, 'SELECT instance_name AS "name" FROM v$instance');
     return rows;
   },
 
@@ -319,7 +273,7 @@ finally {
   },
 }));
 
-drivers.initialize = (dbgateEnv) => {
+drivers.initialize = dbgateEnv => {
   if (dbgateEnv.nativeModules && dbgateEnv.nativeModules.oracledb) {
     oracledb = dbgateEnv.nativeModules.oracledb();
   }

plugins/dbgate-plugin-oracle/src/frontend/drivers.js

@@ -118,11 +118,7 @@ const oracleDriverBase = {
       return ['databaseUrl', 'isReadOnly'].includes(field);
     }
 
-    return (
+    return ['user', 'password', 'defaultDatabase', 'singleDatabase', 'isReadOnly', 'server', 'port'].includes(field);
-      ['authType', 'user', 'password', 'defaultDatabase', 'singleDatabase', 'isReadOnly'].includes(field) ||
-      (values.authType == 'socket' && ['socketPath'].includes(field)) ||
-      (values.authType != 'socket' && ['server', 'port'].includes(field))
-    );
   },
 
   beforeConnectionSave: connection => {
@@ -166,17 +162,13 @@ $$ LANGUAGE plpgsql;`,
       },
     ];
   },
-
-  authTypeLabel: 'Connection mode',
-  defaultAuthTypeName: 'hostPort',
-  defaultSocketPath: '/var/run/oracledb',
 };
 
 /** @type {import('dbgate-types').EngineDriver} */
 const oracleDriver = {
   ...oracleDriverBase,
   engine: 'oracle@dbgate-plugin-oracle',
-  title: 'OracleDB',
+  title: 'OracleDB (Experimental)',
   defaultPort: 1521,
   dialect: {
     ...dialect,
@@ -196,7 +188,8 @@ const oracleDriver = {
     }
     return dialect;
   },
+
+  showConnectionTab: (field) => field == 'sshTunnel',
 };
 
-
 module.exports = [oracleDriver];

plugins/dbgate-plugin-postgres/src/frontend/drivers.js

@@ -35,6 +35,7 @@ const dialect = {
   dropCheck: true,
 
   dropReferencesWhenDropTable: true,
+  requireStandaloneSelectForScopeIdentity: true,
 
   predefinedDataTypes: [
     'bigint',

plugins/dbgate-plugin-sqlite/src/frontend/Dumper.js

@@ -18,7 +18,23 @@ class Dumper extends SqlDumper {
   }
 
   selectScopeIdentity() {
-    this.put('^select last_insert_rowid()')
+    this.put('^select last_insert_rowid()');
+  }
+
+  columnDefinition(column, flags) {
+    if (column.dataType && column.dataType.toLowerCase().includes('int') && column.notNull && column.autoIncrement) {
+      this.put('^integer ^primary ^key ^autoincrement');
+      return;
+    }
+    super.columnDefinition(column, flags);
+  }
+
+  createTablePrimaryKeyCore(table) {
+    const column = table.columns.find((x) => x.autoIncrement);
+    if (column && column.dataType && column.dataType.toLowerCase().includes('int') && column.notNull) {
+      return;
+    }
+    super.createTablePrimaryKeyCore(table);
   }
 }
 

plugins/dbgate-plugin-sqlite/src/frontend/driver.js

@@ -22,6 +22,7 @@ const dialect = {
     return `[${s}]`;
   },
   anonymousPrimaryKey: true,
+  requireStandaloneSelectForScopeIdentity: true,
 
   createColumn: true,
   dropColumn: true,

yarn.lock

@@ -1787,14 +1787,6 @@ activedirectory2@^2.1.0:
     ldapjs "^2.1.0"
     merge-options "^2.0.0"
 
-adler-32@~1.2.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.2.0.tgz#6a3e6bf0a63900ba15652808cb15c6813d1a5f25"
-  integrity sha512-/vUqU/UY4MVeFsg+SsK6c+/05RZXIHZMGJA+PX5JyWI0ZRcBpupnRuPLU/NXXoFwMYCPCoxIfElM2eS+DUXCqQ==
-  dependencies:
-    exit-on-epipe "~1.0.1"
-    printj "~1.1.0"
-
 adler-32@~1.3.0:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.3.1.tgz#1dbf0b36dda0012189a32b3679061932df1821e2"
@@ -2732,7 +2724,7 @@ caseless@~0.12.0:
   resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
   integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==
 
-cfb@^1.1.4:
+cfb@~1.2.1:
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/cfb/-/cfb-1.2.2.tgz#94e687628c700e5155436dac05f74e08df23bc44"
   integrity sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==
@@ -2929,13 +2921,10 @@ code-point-at@^1.0.0:
   resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
   integrity sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==
 
-codepage@~1.14.0:
+codepage@~1.15.0:
-  version "1.14.0"
+  version "1.15.0"
-  resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.14.0.tgz#8cbe25481323559d7d307571b0fff91e7a1d2f99"
+  resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.15.0.tgz#2e00519024b39424ec66eeb3ec07227e692618ab"
-  integrity sha512-iz3zJLhlrg37/gYRWgEPkaFTtzmnEv1h+r7NgZum2lFElYQPi0/5bnmuDfODHxfp0INEfnRqyfyeIJDbb7ahRw==
+  integrity sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==
-  dependencies:
-    commander "~2.14.1"
-    exit-on-epipe "~1.0.1"
 
 collect-v8-coverage@^1.0.0:
   version "1.0.1"
@@ -3006,16 +2995,6 @@ commander@^4.0.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068"
   integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==
 
-commander@~2.14.1:
-  version "2.14.1"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-2.14.1.tgz#2235123e37af8ca3c65df45b026dbd357b01b9aa"
-  integrity sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw==
-
-commander@~2.17.1:
-  version "2.17.1"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf"
-  integrity sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==
-
 commondir@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
@@ -3175,7 +3154,7 @@ cpu-features@~0.0.4:
     buildcheck "0.0.3"
     nan "^2.15.0"
 
-crc-32@~1.2.0:
+crc-32@~1.2.0, crc-32@~1.2.1:
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.2.tgz#3cad35a934b8bf71f25ca524b6da51fb7eace2ff"
   integrity sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==
@@ -3370,30 +3349,11 @@ dbgate-plugin-xml@^5.0.0-alpha.1:
   resolved "https://registry.yarnpkg.com/dbgate-plugin-xml/-/dbgate-plugin-xml-5.0.9.tgz#c3abf6ed8cd1450c45058d35c9326458833ed27e"
   integrity sha512-P8Em1A6HhF0BfxEDDEUyzdgFeJHEC5vbg12frANpWHjO3V1HGdygsT2z1ukLK8FS5BLW/vcCdOFldXZGh+wWvg==
 
-dbgate-query-splitter@^4.9.0, dbgate-query-splitter@^4.9.2, dbgate-query-splitter@^4.9.3:
+dbgate-query-splitter@^4.9.0, dbgate-query-splitter@^4.9.3:
   version "4.9.3"
   resolved "https://registry.yarnpkg.com/dbgate-query-splitter/-/dbgate-query-splitter-4.9.3.tgz#f66396da9ae3cc8f775a282143bfca3441248aa2"
   integrity sha512-QMppAy3S6NGQMawNokmhbpZURvLCETyu/8yTfqWUHGdlK963fdSpmoX1A+9SjCDp62sX0vYntfD7uzd6jVSRcw==
 
-dbgate-sqltree@^5.1.6:
-  version "5.1.6"
-  resolved "https://registry.yarnpkg.com/dbgate-sqltree/-/dbgate-sqltree-5.1.6.tgz#469fe6e06b5146afb9104114564e64aa46c13e7c"
-  integrity sha512-D2ffjeT5HsHBOeW0ORYkCLbDhyIvH6hrE1A/IG3kRZD1iX8Dy6c0h/BWNa4xOjz3pbJYkaB0ju8rIKJqi0BYog==
-  dependencies:
-    lodash "^4.17.21"
-
-dbgate-tools@^5.1.6:
-  version "5.1.6"
-  resolved "https://registry.yarnpkg.com/dbgate-tools/-/dbgate-tools-5.1.6.tgz#43c1e0575db550da5ba38627eaa19839df05d7f1"
-  integrity sha512-KTEqSnNzIdGfYJaIvEusch3/KnO0p376VRykJcK1+/+UtXf9Dk+azaWFjEoACBK3DlK1hmL1pEGfMh5FHDm7Qw==
-  dependencies:
-    dbgate-query-splitter "^4.9.2"
-    dbgate-sqltree "^5.1.6"
-    debug "^4.3.4"
-    json-stable-stringify "^1.0.1"
-    lodash "^4.17.21"
-    uuid "^3.4.0"
-
 debug@2.6.9, debug@^2.2.0, debug@^2.3.3:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -4087,11 +4047,6 @@ execa@^5.0.0:
     signal-exit "^3.0.3"
     strip-final-newline "^2.0.0"
 
-exit-on-epipe@~1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692"
-  integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==
-
 exit@^0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c"
@@ -4333,11 +4288,6 @@ fb-watchman@^2.0.0:
   dependencies:
     bser "2.1.1"
 
-fflate@^0.3.8:
-  version "0.3.11"
-  resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.3.11.tgz#2c440d7180fdeb819e64898d8858af327b042a5d"
-  integrity sha512-Rr5QlUeGN1mbOHlaqcSYMKVpPbgLy0AWT/W0EHxA6NGI12yO1jpoui2zBBvU2G824ltM6Ut8BFgfHSBGfkmS0A==
-
 figgy-pudding@^3.5.1:
   version "3.5.2"
   resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e"
@@ -8896,11 +8846,6 @@ pretty-format@^28.1.3:
     ansi-styles "^5.0.0"
     react-is "^18.0.0"
 
-printj@~1.1.0:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222"
-  integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==
-
 process-nextick-args@~1.0.6:
   version "1.0.7"
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3"
@@ -11479,18 +11424,15 @@ ws@^7.4.3, ws@^7.4.6:
   resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591"
   integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==
 
-xlsx@^0.16.8:
+xlsx@^0.18.5:
-  version "0.16.9"
+  version "0.18.5"
-  resolved "https://registry.yarnpkg.com/xlsx/-/xlsx-0.16.9.tgz#dacd5bb46bda6dd3743940c9c3dc1e2171826256"
+  resolved "https://registry.yarnpkg.com/xlsx/-/xlsx-0.18.5.tgz#16711b9113c848076b8a177022799ad356eba7d0"
-  integrity sha512-gxi1I3EasYvgCX1vN9pGyq920Ron4NO8PNfhuoA3Hpq6Y8f0ECXiy4OLrK4QZBnj1jx3QD+8Fq5YZ/3mPZ5iXw==
+  integrity sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==
   dependencies:
-    adler-32 "~1.2.0"
+    adler-32 "~1.3.0"
-    cfb "^1.1.4"
+    cfb "~1.2.1"
-    codepage "~1.14.0"
+    codepage "~1.15.0"
-    commander "~2.17.1"
+    crc-32 "~1.2.1"
-    crc-32 "~1.2.0"
-    exit-on-epipe "~1.0.1"
-    fflate "^0.3.8"
     ssf "~0.11.2"
     wmf "~1.0.1"
     word "~0.3.0"