From c89e3adb38ead486309cf6b901eb4fece4e46b8d Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 29 Jan 2022 11:30:10 +0100 Subject: [PATCH] fixes --- packages/api/src/controllers/apps.js | 20 ++- packages/api/src/utility/JsonLinesDatabase.js | 118 ++++++++++++++++++ .../api/src/utility/JsonLinesDatastore.js | 4 +- .../tools/src/createBulkInsertStreamBase.ts | 6 +- packages/tools/src/tableTransforms.ts | 3 + packages/types/appdefs.d.ts | 11 +- .../web/src/appobj/AppFileAppObject.svelte | 8 +- 7 files changed, 146 insertions(+), 24 deletions(-) create mode 100644 packages/api/src/utility/JsonLinesDatabase.js diff --git a/packages/api/src/controllers/apps.js b/packages/api/src/controllers/apps.js index e07be5c50..1c2b7f0de 100644 --- a/packages/api/src/controllers/apps.js +++ b/packages/api/src/controllers/apps.js @@ -40,17 +40,11 @@ module.exports = { })); } - function refsType() { - return files - .filter(name => name == 'virtual-references.json') - .map(name => ({ - name: 'virtual-references.json', - label: 'virtual-references.json', - type: 'vfk.json', - })); - } - - return [...refsType(), ...fileType('.command.sql', 'command.sql'), ...fileType('.query.sql', 'query.sql')]; + return [ + ...fileType('.command.sql', 'command.sql'), + ...fileType('.query.sql', 'query.sql'), + ...fileType('.config.json', 'config.json'), + ]; }, async emitChangedDbApp(folder) { @@ -187,7 +181,7 @@ module.exports = { try { res.virtualReferences = JSON.parse( - await fs.readFile(path.join(dir, 'virtual-references.json'), { encoding: 'utf-8' }) + await fs.readFile(path.join(dir, 'virtual-references.config.json'), { encoding: 'utf-8' }) ); } catch (err) { res.virtualReferences = []; @@ -198,7 +192,7 @@ module.exports = { saveVfk_meta: true, async saveVfk({ appFolder, schemaName, pureName, refSchemaName, refTableName, columns }) { - const file = path.join(appdir(), appFolder, 'virtual-references.json'); + const file = path.join(appdir(), appFolder, 'virtual-references.config.json'); let json; try { diff --git a/packages/api/src/utility/JsonLinesDatabase.js b/packages/api/src/utility/JsonLinesDatabase.js new file mode 100644 index 000000000..9edcde79e --- /dev/null +++ b/packages/api/src/utility/JsonLinesDatabase.js @@ -0,0 +1,118 @@ +const fs = require('fs-extra'); +const uuidv1 = require('uuid/v1'); + +// const lineReader = require('line-reader'); +// const { fetchNextLineFromReader } = require('./JsonLinesDatastore'); + +export default class JsonLinesDatabase { + constructor(filename) { + this.filename = filename; + this.data = []; + this.loaded = false; + } + + async _read() { + this.data = []; + if (!(await fs.exists(this.filename))) return; + try { + const text = await fs.readFile(this.filename, { encoding: 'utf-8' }); + this.data = text + .split('\n') + .filter(x => x.trim()) + .map(x => JSON.parse(x)); + } catch (err) { + console.error(`Error loading file ${this.filename}`, err); + } + } + + async _write() { + await fs.writeFile(this.filename, this.data.map(x => JSON.stringify(x)).join('\n')); + } + + async _ensureLoaded() { + if (!this.loaded) { + this._read(); + this.loaded = true; + } + } + + async insert(obj) { + if (obj._id && (await this.get(obj._id))) { + throw new Error(`Cannot insert duplicate ID ${obj._id} into ${this.filename}`); + } + const elem = obj._id + ? obj + : { + ...obj, + _id: uuidv1(), + }; + this.data.push(elem); + await this._write(); + return elem; + } + + async get(id) { + return this.data.find(x => x._id == id); + } + + async find(cond) { + if (cond) { + return this.data.filter(x => { + for (const key of Object.keys(cond)) { + if (x[key] != cond[key]) return false; + } + return true; + }); + } else { + return this.data; + } + } + + async update(obj) { + this.data = this.data.map(x => (x._id == obj._id ? obj : x)); + await this._write(); + } + + async patch(id, values) { + this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x)); + await this._write(); + } + + async remove(id) { + this.data = this.data.filter(x => x._id!=id); + await this._write(); + } + + // async _openReader() { + // return new Promise((resolve, reject) => + // lineReader.open(this.filename, (err, reader) => { + // if (err) reject(err); + // resolve(reader); + // }) + // ); + // } + + // async _read() { + // this.data = []; + // if (!(await fs.exists(this.filename))) return; + // try { + // const reader = await this._openReader(); + // for (;;) { + // const line = await fetchNextLineFromReader(reader); + // if (!line) break; + // this.data.push(JSON.parse(line)); + // } + // } catch (err) { + // console.error(`Error loading file ${this.filename}`, err); + // } + // } + + // async _write() { + // const fw = fs.createWriteStream(this.filename); + // for (const obj of this.data) { + // await fw.write(JSON.stringify(obj)); + // await fw.write('\n'); + // } + // await fw.end(); + // } +} diff --git a/packages/api/src/utility/JsonLinesDatastore.js b/packages/api/src/utility/JsonLinesDatastore.js index b9965ae88..32245c271 100644 --- a/packages/api/src/utility/JsonLinesDatastore.js +++ b/packages/api/src/utility/JsonLinesDatastore.js @@ -4,7 +4,7 @@ const lock = new AsyncLock(); const stableStringify = require('json-stable-stringify'); const { evaluateCondition } = require('dbgate-sqltree'); -async function fetchNextLine(reader) { +export async function fetchNextLineFromReader(reader) { return new Promise((resolve, reject) => { if (!reader.hasNextLine()) { resolve(null); @@ -62,7 +62,7 @@ class JsonLinesDatastore { async _readLine(parse) { for (;;) { - const line = await fetchNextLine(this.reader); + const line = await fetchNextLineFromReader(this.reader); if (!line) { // EOF return null; diff --git a/packages/tools/src/createBulkInsertStreamBase.ts b/packages/tools/src/createBulkInsertStreamBase.ts index eed503a48..068912c3d 100644 --- a/packages/tools/src/createBulkInsertStreamBase.ts +++ b/packages/tools/src/createBulkInsertStreamBase.ts @@ -29,18 +29,18 @@ export function createBulkInsertStreamBase(driver, stream, pool, name, options): // console.log('ANALYSING', name, structure); if (structure && options.dropIfExists) { console.log(`Dropping table ${fullNameQuoted}`); - await driver.query(pool, `DROP TABLE ${fullNameQuoted}`); + await driver.script(pool, `DROP TABLE ${fullNameQuoted}`); } if (options.createIfNotExists && (!structure || options.dropIfExists)) { console.log(`Creating table ${fullNameQuoted}`); const dmp = driver.createDumper(); dmp.createTable(prepareTableForImport({ ...writable.structure, ...name })); console.log(dmp.s); - await driver.query(pool, dmp.s); + await driver.script(pool, dmp.s); structure = await driver.analyseSingleTable(pool, name); } if (options.truncate) { - await driver.query(pool, `TRUNCATE TABLE ${fullNameQuoted}`); + await driver.script(pool, `TRUNCATE TABLE ${fullNameQuoted}`); } writable.columnNames = _intersection( diff --git a/packages/tools/src/tableTransforms.ts b/packages/tools/src/tableTransforms.ts index 994161b7a..403426cff 100644 --- a/packages/tools/src/tableTransforms.ts +++ b/packages/tools/src/tableTransforms.ts @@ -4,6 +4,9 @@ import _cloneDeep from 'lodash/cloneDeep'; export function prepareTableForImport(table: TableInfo): TableInfo { const res = _cloneDeep(table); res.foreignKeys = []; + res.indexes = []; + res.uniques = []; + res.checks = []; if (res.primaryKey) res.primaryKey.constraintName = null; return res; } diff --git a/packages/types/appdefs.d.ts b/packages/types/appdefs.d.ts index 4d45f5a51..6fdd2822a 100644 --- a/packages/types/appdefs.d.ts +++ b/packages/types/appdefs.d.ts @@ -19,10 +19,19 @@ interface VirtualReferenceDefinition { }[]; } -interface ApplicationDefinition { +interface ColumnDescriptionDefinition { + pureName: string; + schemaName?: string; + expresssion?: string; + columns: string[]; + delimiter: string; +} + +export interface ApplicationDefinition { name: string; queries: ApplicationQuery[]; commands: ApplicationCommand[]; virtualReferences: VirtualReferenceDefinition[]; + columnDescriptions: ColumnDescriptionDefinition[]; } diff --git a/packages/web/src/appobj/AppFileAppObject.svelte b/packages/web/src/appobj/AppFileAppObject.svelte index 1c962b71d..7a70039f4 100644 --- a/packages/web/src/appobj/AppFileAppObject.svelte +++ b/packages/web/src/appobj/AppFileAppObject.svelte @@ -3,11 +3,9 @@ const connProps: any = {}; let tooltip = undefined; - const savedFile = fileType == 'vfk.json' ? fileName : fileName + '.' + fileType; - const resp = await apiCall('files/load', { folder: 'app:' + folderName, - file: savedFile, + file: fileName + '.' + fileType, format: 'text', }); @@ -18,7 +16,7 @@ tabComponent, tooltip, props: { - savedFile, + savedFile:fileName + '.' + fileType, savedFolder: 'app:' + folderName, savedFormat: 'text', appFolder: folderName, @@ -32,7 +30,7 @@ export const extractKey = data => data.fileName; export const createMatcher = ({ fileName }) => filter => filterName(filter, fileName); const APP_ICONS = { - 'vfk.json': 'img json', + 'config.json': 'img json', 'command.sql': 'img app-command', 'query.sql': 'img app-query', };