sort JSONL data & query results

packages/api/package.json

@@ -36,6 +36,7 @@
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
     "express-fileupload": "^1.2.0",
+    "external-sorting": "^1.3.1",
     "fs-extra": "^9.1.0",
     "fs-reverse": "^0.0.3",
     "get-port": "^5.1.1",
@@ -52,6 +53,7 @@
     "on-finished": "^2.4.1",
     "pinomin": "^1.0.1",
     "portfinder": "^1.0.28",
+    "rimraf": "^4.1.2",
     "simple-encryptor": "^4.0.0",
     "ssh2": "^1.11.0",
     "tar": "^6.0.5",

packages/api/src/controllers/jsldata.js

@@ -135,9 +135,9 @@ module.exports = {
   },
 
   getRows_meta: true,
-  async getRows({ jslid, offset, limit, filters, formatterFunction }) {
+  async getRows({ jslid, offset, limit, filters, sort, formatterFunction }) {
     const datastore = await this.ensureDatastore(jslid, formatterFunction);
-    return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters);
+    return datastore.getRows(offset, limit, _.isEmpty(filters) ? null : filters, _.isEmpty(sort) ? null : sort);
   },
 
   getStats_meta: true,

packages/api/src/utility/JsonLinesDatastore.js

@@ -1,9 +1,16 @@
+const fs = require('fs');
+const os = require('os');
+const rimraf = require('rimraf');
+const path = require('path');
 const lineReader = require('line-reader');
 const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
 const requirePluginFunction = require('./requirePluginFunction');
+const esort = require('external-sorting');
+const uuidv1 = require('uuid/v1');
+const { jsldir } = require('./directories');
 
 function fetchNextLineFromReader(reader) {
   return new Promise((resolve, reject) => {
@@ -32,7 +39,39 @@ class JsonLinesDatastore {
     // this.firstRowToBeReturned = null;
     this.notifyChangedCallback = null;
     this.currentFilter = null;
+    this.currentSort = null;
     this.rowFormatter = requirePluginFunction(formatterFunction);
+    this.sortedFiles = {};
+  }
+
+  static async sortFile(infile, outfile, sort) {
+    const tempDir = path.join(os.tmpdir(), uuidv1());
+    fs.mkdirSync(tempDir);
+
+    await esort
+      .default({
+        input: fs.createReadStream(infile),
+        output: fs.createWriteStream(outfile),
+        deserializer: JSON.parse,
+        serializer: JSON.stringify,
+        tempDir,
+        maxHeap: 100,
+        comparer: (a, b) => {
+          for (const item of sort) {
+            const { uniqueName, order } = item;
+            if (a[uniqueName] < b[uniqueName]) {
+              return order == 'ASC' ? -1 : 1;
+            }
+            if (a[uniqueName] > b[uniqueName]) {
+              return order == 'ASC' ? 1 : -1;
+            }
+          }
+          return 0;
+        },
+      })
+      .asc();
+
+    await rimraf(tempDir);
   }
 
   _closeReader() {
@@ -43,6 +82,7 @@ class JsonLinesDatastore {
     this.readedSchemaRow = false;
     // this.firstRowToBeReturned = null;
     this.currentFilter = null;
+    this.currentSort = null;
     reader.close(() => {});
   }
 
@@ -56,9 +96,9 @@ class JsonLinesDatastore {
     if (call) call();
   }
 
-  async _openReader() {
+  async _openReader(fileName) {
     return new Promise((resolve, reject) =>
-      lineReader.open(this.file, (err, reader) => {
+      lineReader.open(fileName, (err, reader) => {
         if (err) reject(err);
         resolve(reader);
       })
@@ -140,14 +180,19 @@ class JsonLinesDatastore {
     // });
   }
 
-  async _ensureReader(offset, filter) {
-    if (this.readedDataRowCount > offset || stableStringify(filter) != stableStringify(this.currentFilter)) {
+  async _ensureReader(offset, filter, sort) {
+    if (
+      this.readedDataRowCount > offset ||
+      stableStringify(filter) != stableStringify(this.currentFilter) ||
+      stableStringify(sort) != stableStringify(this.currentSort)
+    ) {
       this._closeReader();
     }
     if (!this.reader) {
-      const reader = await this._openReader();
+      const reader = await this._openReader(sort ? this.sortedFiles[stableStringify(sort)] : this.file);
       this.reader = reader;
       this.currentFilter = filter;
+      this.currentSort = sort;
     }
     // if (!this.readedSchemaRow) {
     //   const line = await this._readLine(true); // skip structure
@@ -179,10 +224,16 @@ class JsonLinesDatastore {
     });
   }
 
-  async getRows(offset, limit, filter) {
+  async getRows(offset, limit, filter, sort) {
     const res = [];
+    if (sort && !this.sortedFiles[stableStringify(sort)]) {
+      const jslid = uuidv1();
+      const sortedFile = path.join(jsldir(), `${jslid}.jsonl`);
+      await JsonLinesDatastore.sortFile(this.file, sortedFile, sort);
+      this.sortedFiles[stableStringify(sort)] = sortedFile;
+    }
     await lock.acquire('reader', async () => {
-      await this._ensureReader(offset, filter);
+      await this._ensureReader(offset, filter, sort);
       // console.log(JSON.stringify(this.currentFilter, undefined, 2));
       for (let i = 0; i < limit; i += 1) {
         const line = await this._readLine(true);

packages/datalib/src/JslGridDisplay.ts

@@ -18,6 +18,7 @@ export class JslGridDisplay extends GridDisplay {
     super(config, setConfig, cache, setCache, null);
 
     this.filterable = true;
+    this.sortable = true;
     this.supportsReload = supportsReload;
     this.isDynamicStructure = isDynamicStructure;
     this.filterTypeOverride = 'eval';

packages/web/src/datagrid/JslDataGridCore.svelte

@@ -20,6 +20,7 @@
       limit,
       formatterFunction,
       filters: display ? display.compileFilters() : null,
+      sort: display.config.sort,
     });
 
     return response;

yarn.lock

@@ -4235,6 +4235,13 @@ external-editor@^3.0.3:
     iconv-lite "^0.4.24"
     tmp "^0.0.33"
 
+external-sorting@^1.3.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/external-sorting/-/external-sorting-1.3.1.tgz#caec567906bd8d936cc94165c7daf657cd4e3163"
+  integrity sha512-eqI/TxUu4U5RW90ml7bRyvk/0Qh/Lf3JecZQKeLmC0eVRzPQ2UG3ZR7k66EDO1UnTJat0D8bn129K+gnZYMJuw==
+  dependencies:
+    fast-sort "^2.0.1"
+
 extglob@^2.0.4:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/extglob/-/extglob-2.0.4.tgz#ad00fe4dc612a9232e8718711dc5cb5ab0285543"
@@ -4307,6 +4314,11 @@ fast-safe-stringify@^2.1.1:
   resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884"
   integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==
 
+fast-sort@^2.0.1:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/fast-sort/-/fast-sort-2.2.0.tgz#20903763531fbcbb41c9df5ab1bf5f2cefc8476a"
+  integrity sha512-W7zqnn2zsYoQA87FKmYtgOsbJohOrh7XrtZrCVHN5XZKqTBTv5UG+rSS3+iWbg/nepRQUOu+wnas8BwtK8kiCg==
+
 fastq@^1.6.0:
   version "1.13.0"
   resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c"
@@ -9471,6 +9483,11 @@ rimraf@^3.0.0, rimraf@^3.0.2:
   dependencies:
     glob "^7.1.3"
 
+rimraf@^4.1.2:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-4.1.2.tgz#20dfbc98083bdfaa28b01183162885ef213dbf7c"
+  integrity sha512-BlIbgFryTbw3Dz6hyoWFhKk+unCcHMSkZGrTFVAx2WmttdBSonsdtRlwiuTbDqTKr+UlXIUqJVS4QT5tUzGENQ==
+
 ripemd160@^2.0.0, ripemd160@^2.0.1:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c"