fixes

packages/api/src/controllers/apps.js

@@ -40,17 +40,11 @@ module.exports = {
         }));
     }
 
-    function refsType() {
+    return [
-      return files
+      ...fileType('.command.sql', 'command.sql'),
-        .filter(name => name == 'virtual-references.json')
+      ...fileType('.query.sql', 'query.sql'),
-        .map(name => ({
+      ...fileType('.config.json', 'config.json'),
-          name: 'virtual-references.json',
+    ];
-          label: 'virtual-references.json',
-          type: 'vfk.json',
-        }));
-    }
-
-    return [...refsType(), ...fileType('.command.sql', 'command.sql'), ...fileType('.query.sql', 'query.sql')];
   },
 
   async emitChangedDbApp(folder) {
@@ -187,7 +181,7 @@ module.exports = {
 
     try {
       res.virtualReferences = JSON.parse(
-        await fs.readFile(path.join(dir, 'virtual-references.json'), { encoding: 'utf-8' })
+        await fs.readFile(path.join(dir, 'virtual-references.config.json'), { encoding: 'utf-8' })
       );
     } catch (err) {
       res.virtualReferences = [];
@@ -198,7 +192,7 @@ module.exports = {
 
   saveVfk_meta: true,
   async saveVfk({ appFolder, schemaName, pureName, refSchemaName, refTableName, columns }) {
-    const file = path.join(appdir(), appFolder, 'virtual-references.json');
+    const file = path.join(appdir(), appFolder, 'virtual-references.config.json');
 
     let json;
     try {

packages/api/src/utility/JsonLinesDatabase.js

@@ -0,0 +1,118 @@
+const fs = require('fs-extra');
+const uuidv1 = require('uuid/v1');
+
+// const lineReader = require('line-reader');
+// const { fetchNextLineFromReader } = require('./JsonLinesDatastore');
+
+export default class JsonLinesDatabase {
+  constructor(filename) {
+    this.filename = filename;
+    this.data = [];
+    this.loaded = false;
+  }
+
+  async _read() {
+    this.data = [];
+    if (!(await fs.exists(this.filename))) return;
+    try {
+      const text = await fs.readFile(this.filename, { encoding: 'utf-8' });
+      this.data = text
+        .split('\n')
+        .filter(x => x.trim())
+        .map(x => JSON.parse(x));
+    } catch (err) {
+      console.error(`Error loading file ${this.filename}`, err);
+    }
+  }
+
+  async _write() {
+    await fs.writeFile(this.filename, this.data.map(x => JSON.stringify(x)).join('\n'));
+  }
+
+  async _ensureLoaded() {
+    if (!this.loaded) {
+      this._read();
+      this.loaded = true;
+    }
+  }
+
+  async insert(obj) {
+    if (obj._id && (await this.get(obj._id))) {
+      throw new Error(`Cannot insert duplicate ID ${obj._id} into ${this.filename}`);
+    }
+    const elem = obj._id
+      ? obj
+      : {
+          ...obj,
+          _id: uuidv1(),
+        };
+    this.data.push(elem);
+    await this._write();
+    return elem;
+  }
+
+  async get(id) {
+    return this.data.find(x => x._id == id);
+  }
+
+  async find(cond) {
+    if (cond) {
+      return this.data.filter(x => {
+        for (const key of Object.keys(cond)) {
+          if (x[key] != cond[key]) return false;
+        }
+        return true;
+      });
+    } else {
+      return this.data;
+    }
+  }
+
+  async update(obj) {
+    this.data = this.data.map(x => (x._id == obj._id ? obj : x));
+    await this._write();
+  }
+
+  async patch(id, values) {
+    this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));
+    await this._write();
+  }
+
+  async remove(id) {
+    this.data = this.data.filter(x => x._id!=id);
+    await this._write();
+  }
+
+  //   async _openReader() {
+  //     return new Promise((resolve, reject) =>
+  //       lineReader.open(this.filename, (err, reader) => {
+  //         if (err) reject(err);
+  //         resolve(reader);
+  //       })
+  //     );
+  //   }
+
+  //   async _read() {
+  //     this.data = [];
+  //     if (!(await fs.exists(this.filename))) return;
+  //     try {
+  //       const reader = await this._openReader();
+  //       for (;;) {
+  //         const line = await fetchNextLineFromReader(reader);
+  //         if (!line) break;
+  //         this.data.push(JSON.parse(line));
+  //       }
+  //     } catch (err) {
+  //       console.error(`Error loading file ${this.filename}`, err);
+  //     }
+  //   }
+
+  //   async _write() {
+  //     const fw = fs.createWriteStream(this.filename);
+  //     for (const obj of this.data) {
+  //       await fw.write(JSON.stringify(obj));
+  //       await fw.write('\n');
+  //     }
+  //     await fw.end();
+  //   }
+}

packages/api/src/utility/JsonLinesDatastore.js

@@ -4,7 +4,7 @@ const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
 
-async function fetchNextLine(reader) {
+export async function fetchNextLineFromReader(reader) {
   return new Promise((resolve, reject) => {
     if (!reader.hasNextLine()) {
       resolve(null);
@@ -62,7 +62,7 @@ class JsonLinesDatastore {
 
   async _readLine(parse) {
     for (;;) {
-      const line = await fetchNextLine(this.reader);
+      const line = await fetchNextLineFromReader(this.reader);
       if (!line) {
         // EOF
         return null;

packages/tools/src/createBulkInsertStreamBase.ts

@@ -29,18 +29,18 @@ export function createBulkInsertStreamBase(driver, stream, pool, name, options):
     // console.log('ANALYSING', name, structure);
     if (structure && options.dropIfExists) {
       console.log(`Dropping table ${fullNameQuoted}`);
-      await driver.query(pool, `DROP TABLE ${fullNameQuoted}`);
+      await driver.script(pool, `DROP TABLE ${fullNameQuoted}`);
     }
     if (options.createIfNotExists && (!structure || options.dropIfExists)) {
       console.log(`Creating table ${fullNameQuoted}`);
       const dmp = driver.createDumper();
       dmp.createTable(prepareTableForImport({ ...writable.structure, ...name }));
       console.log(dmp.s);
-      await driver.query(pool, dmp.s);
+      await driver.script(pool, dmp.s);
       structure = await driver.analyseSingleTable(pool, name);
     }
     if (options.truncate) {
-      await driver.query(pool, `TRUNCATE TABLE ${fullNameQuoted}`);
+      await driver.script(pool, `TRUNCATE TABLE ${fullNameQuoted}`);
     }
 
     writable.columnNames = _intersection(

packages/tools/src/tableTransforms.ts

@@ -4,6 +4,9 @@ import _cloneDeep from 'lodash/cloneDeep';
 export function prepareTableForImport(table: TableInfo): TableInfo {
   const res = _cloneDeep(table);
   res.foreignKeys = [];
+  res.indexes = [];
+  res.uniques = [];
+  res.checks = [];
   if (res.primaryKey) res.primaryKey.constraintName = null;
   return res;
 }

packages/types/appdefs.d.ts

@@ -19,10 +19,19 @@ interface VirtualReferenceDefinition {
   }[];
 }
 
-interface ApplicationDefinition {
+interface ColumnDescriptionDefinition {
+  pureName: string;
+  schemaName?: string;
+  expresssion?: string;
+  columns: string[];
+  delimiter: string;
+}
+
+export interface ApplicationDefinition {
   name: string;
 
   queries: ApplicationQuery[];
   commands: ApplicationCommand[];
   virtualReferences: VirtualReferenceDefinition[];
+  columnDescriptions: ColumnDescriptionDefinition[];
 }

packages/web/src/appobj/AppFileAppObject.svelte

@@ -3,11 +3,9 @@
     const connProps: any = {};
     let tooltip = undefined;
 
-    const savedFile = fileType == 'vfk.json' ? fileName : fileName + '.' + fileType;
-
     const resp = await apiCall('files/load', {
       folder: 'app:' + folderName,
-      file: savedFile,
+      file: fileName + '.' + fileType,
       format: 'text',
     });
 
@@ -18,7 +16,7 @@
         tabComponent,
         tooltip,
         props: {
-          savedFile,
+          savedFile:fileName + '.' + fileType,
           savedFolder: 'app:' + folderName,
           savedFormat: 'text',
           appFolder: folderName,
@@ -32,7 +30,7 @@
   export const extractKey = data => data.fileName;
   export const createMatcher = ({ fileName }) => filter => filterName(filter, fileName);
   const APP_ICONS = {
-    'vfk.json': 'img json',
+    'config.json': 'img json',
     'command.sql': 'img app-command',
     'query.sql': 'img app-query',
   };