dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-19 10:46:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

jsl data refactor

Browse Source
This commit is contained in:
Jan Prochazka
2020-10-22 08:23:53 +02:00
parent c4914429ce
commit da1617729b
5 changed files with 318 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
packages/api/src/controllers/jsldata.js
View File

@@ -1,6 +1,9 @@
const fs = require('fs');
const lineReader = require('line-reader');
const { off } = require('process');
const DatastoreProxy = require('../utility/DatastoreProxy');
const getJslFileName = require('../utility/getJslFileName');
const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
const socket = require('../utility/socket');
function readFirstLine(file) {
@@ -19,76 +22,85 @@ function readFirstLine(file) {
});
}
module.exports = {
openedReaders: {},
datastores: {},
closeReader(jslid) {
// console.log('CLOSING READER');
if (!this.openedReaders[jslid]) return Promise.resolve();
return new Promise((resolve, reject) => {
this.openedReaders[jslid].reader.close((err) => {
if (err) reject(err);
delete this.openedReaders[jslid];
resolve();
});
});
},
// closeReader(jslid) {
// // console.log('CLOSING READER');
// if (!this.openedReaders[jslid]) return Promise.resolve();
// return new Promise((resolve, reject) => {
// this.openedReaders[jslid].reader.close((err) => {
// if (err) reject(err);
// delete this.openedReaders[jslid];
// resolve();
// });
// });
// },
readLine(readerInfo) {
return new Promise((resolve, reject) => {
const { reader } = readerInfo;
if (!reader.hasNextLine()) {
resolve(null);
return;
}
reader.nextLine((err, line) => {
if (readerInfo.readedSchemaRow) readerInfo.readedDataRowCount += 1;
else readerInfo.readedSchemaRow = true;
if (err) reject(err);
resolve(line);
});
});
},
// readLine(readerInfo) {
// return new Promise((resolve, reject) => {
// const { reader } = readerInfo;
// if (!reader.hasNextLine()) {
// resolve(null);
// return;
// }
// reader.nextLine((err, line) => {
// if (readerInfo.readedSchemaRow) readerInfo.readedDataRowCount += 1;
// else readerInfo.readedSchemaRow = true;
// if (err) reject(err);
// resolve(line);
// });
// });
// },
openReader(jslid) {
// console.log('OPENING READER');
// console.log(
// 'OPENING READER, LINES=',
// fs.readFileSync(path.join(jsldir(), `${jslid}.jsonl`), 'utf-8').split('\n').length
// );
const file = getJslFileName(jslid);
return new Promise((resolve, reject) =>
lineReader.open(file, (err, reader) => {
if (err) reject(err);
const readerInfo = {
reader,
readedDataRowCount: 0,
readedSchemaRow: false,
isReading: true,
};
this.openedReaders[jslid] = readerInfo;
resolve(readerInfo);
})
);
},
// openReader(jslid) {
// // console.log('OPENING READER');
// // console.log(
// // 'OPENING READER, LINES=',
// // fs.readFileSync(path.join(jsldir(), `${jslid}.jsonl`), 'utf-8').split('\n').length
// // );
// const file = getJslFileName(jslid);
// return new Promise((resolve, reject) =>
// lineReader.open(file, (err, reader) => {
// if (err) reject(err);
// const readerInfo = {
// reader,
// readedDataRowCount: 0,
// readedSchemaRow: false,
// isReading: true,
// };
// this.openedReaders[jslid] = readerInfo;
// resolve(readerInfo);
// })
// );
// },
async ensureReader(jslid, offset) {
if (this.openedReaders[jslid] && this.openedReaders[jslid].readedDataRowCount > offset) {
await this.closeReader(jslid);
// async ensureReader(jslid, offset) {
// if (this.openedReaders[jslid] && this.openedReaders[jslid].readedDataRowCount > offset) {
// await this.closeReader(jslid);
// }
// let readerInfo = this.openedReaders[jslid];
// if (!this.openedReaders[jslid]) {
// readerInfo = await this.openReader(jslid);
// }
// readerInfo.isReading = true;
// if (!readerInfo.readedSchemaRow) {
// await this.readLine(readerInfo); // skip structure
// }
// while (readerInfo.readedDataRowCount < offset) {
// await this.readLine(readerInfo);
// }
// return readerInfo;
// },
async ensureDatastore(jslid) {
let datastore = this.datastores[jslid];
if (!datastore) {
datastore = new JsonLinesDatastore(getJslFileName(jslid));
// datastore = new DatastoreProxy(getJslFileName(jslid));
this.datastores[jslid] = datastore;
}
let readerInfo = this.openedReaders[jslid];
if (!this.openedReaders[jslid]) {
readerInfo = await this.openReader(jslid);
}
readerInfo.isReading = true;
if (!readerInfo.readedSchemaRow) {
await this.readLine(readerInfo); // skip structure
}
while (readerInfo.readedDataRowCount < offset) {
await this.readLine(readerInfo);
}
return readerInfo;
return datastore;
},
getInfo_meta: 'get',
@@ -101,20 +113,8 @@ module.exports = {
getRows_meta: 'get',
async getRows({ jslid, offset, limit }) {
const readerInfo = await this.ensureReader(jslid, offset);
const res = [];
for (let i = 0; i < limit; i += 1) {
const line = await this.readLine(readerInfo);
if (line == null) break;
res.push(JSON.parse(line));
}
readerInfo.isReading = false;
if (readerInfo.closeAfterReadAndSendStats) {
await this.closeReader(jslid);
socket.emit(`jsldata-stats-${jslid}`, readerInfo.closeAfterReadAndSendStats);
readerInfo.closeAfterReadAndSendStats = null;
}
return res;
const datastore = await this.ensureDatastore(jslid);
return datastore.getRows(offset, limit);
},
getStats_meta: 'get',
@@ -126,12 +126,16 @@ module.exports = {
async notifyChangedStats(stats) {
console.log('SENDING STATS', JSON.stringify(stats));
const readerInfo = this.openedReaders[stats.jslid];
if (readerInfo && readerInfo.isReading) {
readerInfo.closeAfterReadAndSendStats = stats;
} else {
await this.closeReader(stats.jslid);
socket.emit(`jsldata-stats-${stats.jslid}`, stats);
}
const datastore = this.datastores[stats.jslid];
if (datastore) await datastore.notifyChanged();
socket.emit(`jsldata-stats-${stats.jslid}`, stats);
// const readerInfo = this.openedReaders[stats.jslid];
// if (readerInfo && readerInfo.isReading) {
// readerInfo.closeAfterReadAndSendStats = stats;
// } else {
// await this.closeReader(stats.jslid);
// socket.emit(`jsldata-stats-${stats.jslid}`, stats);
// }
},
};