diff --git a/packages/api/package.json b/packages/api/package.json index 980079095..13c7aec6e 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -36,6 +36,7 @@ "express": "^4.17.1", "express-basic-auth": "^1.2.0", "express-fileupload": "^1.2.0", + "external-sorting": "^1.3.1", "fs-extra": "^9.1.0", "fs-reverse": "^0.0.3", "get-port": "^5.1.1", @@ -52,6 +53,7 @@ "on-finished": "^2.4.1", "pinomin": "^1.0.1", "portfinder": "^1.0.28", + "rimraf": "^4.1.2", "simple-encryptor": "^4.0.0", "ssh2": "^1.11.0", "tar": "^6.0.5", diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index 4778e2f59..e22a8ad52 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -135,9 +135,9 @@ module.exports = { }, getRows_meta: true, - async getRows({ jslid, offset, limit, filters, formatterFunction }) { + async getRows({ jslid, offset, limit, filters, sort, formatterFunction }) { const datastore = await this.ensureDatastore(jslid, formatterFunction); - return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters); + return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters, _.isEmpty(sort) ? null : sort); }, getStats_meta: true, diff --git a/packages/api/src/utility/JsonLinesDatastore.js b/packages/api/src/utility/JsonLinesDatastore.js index a5e12da47..3749936c0 100644 --- a/packages/api/src/utility/JsonLinesDatastore.js +++ b/packages/api/src/utility/JsonLinesDatastore.js @@ -1,9 +1,16 @@ +const fs = require('fs'); +const os = require('os'); +const rimraf = require('rimraf'); +const path = require('path'); const lineReader = require('line-reader'); const AsyncLock = require('async-lock'); const lock = new AsyncLock(); const stableStringify = require('json-stable-stringify'); const { evaluateCondition } = require('dbgate-sqltree'); const requirePluginFunction = require('./requirePluginFunction'); +const esort = require('external-sorting'); +const uuidv1 = require('uuid/v1'); +const { jsldir } = require('./directories'); function fetchNextLineFromReader(reader) { return new Promise((resolve, reject) => { @@ -32,7 +39,39 @@ class JsonLinesDatastore { // this.firstRowToBeReturned = null; this.notifyChangedCallback = null; this.currentFilter = null; + this.currentSort = null; this.rowFormatter = requirePluginFunction(formatterFunction); + this.sortedFiles = {}; + } + + static async sortFile(infile, outfile, sort) { + const tempDir = path.join(os.tmpdir(), uuidv1()); + fs.mkdirSync(tempDir); + + await esort + .default({ + input: fs.createReadStream(infile), + output: fs.createWriteStream(outfile), + deserializer: JSON.parse, + serializer: JSON.stringify, + tempDir, + maxHeap: 100, + comparer: (a, b) => { + for (const item of sort) { + const { uniqueName, order } = item; + if (a[uniqueName] < b[uniqueName]) { + return order == 'ASC' ? -1 : 1; + } + if (a[uniqueName] > b[uniqueName]) { + return order == 'ASC' ? 1 : -1; + } + } + return 0; + }, + }) + .asc(); + + await rimraf(tempDir); } _closeReader() { @@ -43,6 +82,7 @@ class JsonLinesDatastore { this.readedSchemaRow = false; // this.firstRowToBeReturned = null; this.currentFilter = null; + this.currentSort = null; reader.close(() => {}); } @@ -56,9 +96,9 @@ class JsonLinesDatastore { if (call) call(); } - async _openReader() { + async _openReader(fileName) { return new Promise((resolve, reject) => - lineReader.open(this.file, (err, reader) => { + lineReader.open(fileName, (err, reader) => { if (err) reject(err); resolve(reader); }) @@ -140,14 +180,19 @@ class JsonLinesDatastore { // }); } - async _ensureReader(offset, filter) { - if (this.readedDataRowCount > offset || stableStringify(filter) != stableStringify(this.currentFilter)) { + async _ensureReader(offset, filter, sort) { + if ( + this.readedDataRowCount > offset || + stableStringify(filter) != stableStringify(this.currentFilter) || + stableStringify(sort) != stableStringify(this.currentSort) + ) { this._closeReader(); } if (!this.reader) { - const reader = await this._openReader(); + const reader = await this._openReader(sort ? this.sortedFiles[stableStringify(sort)] : this.file); this.reader = reader; this.currentFilter = filter; + this.currentSort = sort; } // if (!this.readedSchemaRow) { // const line = await this._readLine(true); // skip structure @@ -179,10 +224,16 @@ class JsonLinesDatastore { }); } - async getRows(offset, limit, filter) { + async getRows(offset, limit, filter, sort) { const res = []; + if (sort && !this.sortedFiles[stableStringify(sort)]) { + const jslid = uuidv1(); + const sortedFile = path.join(jsldir(), `${jslid}.jsonl`); + await JsonLinesDatastore.sortFile(this.file, sortedFile, sort); + this.sortedFiles[stableStringify(sort)] = sortedFile; + } await lock.acquire('reader', async () => { - await this._ensureReader(offset, filter); + await this._ensureReader(offset, filter, sort); // console.log(JSON.stringify(this.currentFilter, undefined, 2)); for (let i = 0; i < limit; i += 1) { const line = await this._readLine(true); diff --git a/packages/datalib/src/JslGridDisplay.ts b/packages/datalib/src/JslGridDisplay.ts index 34b77f37d..bc9696316 100644 --- a/packages/datalib/src/JslGridDisplay.ts +++ b/packages/datalib/src/JslGridDisplay.ts @@ -18,6 +18,7 @@ export class JslGridDisplay extends GridDisplay { super(config, setConfig, cache, setCache, null); this.filterable = true; + this.sortable = true; this.supportsReload = supportsReload; this.isDynamicStructure = isDynamicStructure; this.filterTypeOverride = 'eval'; diff --git a/packages/web/src/datagrid/JslDataGridCore.svelte b/packages/web/src/datagrid/JslDataGridCore.svelte index f3e95722b..b367e4408 100644 --- a/packages/web/src/datagrid/JslDataGridCore.svelte +++ b/packages/web/src/datagrid/JslDataGridCore.svelte @@ -20,6 +20,7 @@ limit, formatterFunction, filters: display ? display.compileFilters() : null, + sort: display.config.sort, }); return response; diff --git a/yarn.lock b/yarn.lock index 0f9628b97..e55c99552 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4235,6 +4235,13 @@ external-editor@^3.0.3: iconv-lite "^0.4.24" tmp "^0.0.33" +external-sorting@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/external-sorting/-/external-sorting-1.3.1.tgz#caec567906bd8d936cc94165c7daf657cd4e3163" + integrity sha512-eqI/TxUu4U5RW90ml7bRyvk/0Qh/Lf3JecZQKeLmC0eVRzPQ2UG3ZR7k66EDO1UnTJat0D8bn129K+gnZYMJuw== + dependencies: + fast-sort "^2.0.1" + extglob@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/extglob/-/extglob-2.0.4.tgz#ad00fe4dc612a9232e8718711dc5cb5ab0285543" @@ -4307,6 +4314,11 @@ fast-safe-stringify@^2.1.1: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884" integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== +fast-sort@^2.0.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/fast-sort/-/fast-sort-2.2.0.tgz#20903763531fbcbb41c9df5ab1bf5f2cefc8476a" + integrity sha512-W7zqnn2zsYoQA87FKmYtgOsbJohOrh7XrtZrCVHN5XZKqTBTv5UG+rSS3+iWbg/nepRQUOu+wnas8BwtK8kiCg== + fastq@^1.6.0: version "1.13.0" resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c" @@ -9471,6 +9483,11 @@ rimraf@^3.0.0, rimraf@^3.0.2: dependencies: glob "^7.1.3" +rimraf@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-4.1.2.tgz#20dfbc98083bdfaa28b01183162885ef213dbf7c" + integrity sha512-BlIbgFryTbw3Dz6hyoWFhKk+unCcHMSkZGrTFVAx2WmttdBSonsdtRlwiuTbDqTKr+UlXIUqJVS4QT5tUzGENQ== + ripemd160@^2.0.0, ripemd160@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c"