diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index 2ba1eca4b..280e57f55 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -1,6 +1,7 @@ const fs = require('fs'); const lineReader = require('line-reader'); const _ = require('lodash'); +const { __ } = require('lodash/fp'); const DatastoreProxy = require('../utility/DatastoreProxy'); const { saveFreeTableData } = require('../utility/freeTableStorage'); const getJslFileName = require('../utility/getJslFileName'); @@ -111,7 +112,16 @@ module.exports = { async getInfo({ jslid }) { const file = getJslFileName(jslid); const firstLine = await readFirstLine(file); - if (firstLine) return JSON.parse(firstLine); + if (firstLine) { + const parsed = JSON.parse(firstLine); + if (parsed.__isStreamHeader) { + return parsed; + } + return { + __isStreamHeader: true, + __isDynamicStructure: true, + }; + } return null; }, diff --git a/packages/api/src/utility/JsonLinesDatastore.js b/packages/api/src/utility/JsonLinesDatastore.js index 37a7220f3..904ff9edd 100644 --- a/packages/api/src/utility/JsonLinesDatastore.js +++ b/packages/api/src/utility/JsonLinesDatastore.js @@ -27,6 +27,7 @@ class JsonLinesDatastore { this.reader = null; this.readedDataRowCount = 0; this.readedSchemaRow = false; + // this.firstRowToBeReturned = null; this.notifyChangedCallback = null; this.currentFilter = null; } @@ -37,6 +38,7 @@ class JsonLinesDatastore { this.reader = null; this.readedDataRowCount = 0; this.readedSchemaRow = false; + // this.firstRowToBeReturned = null; this.currentFilter = null; reader.close(() => {}); } @@ -61,6 +63,11 @@ class JsonLinesDatastore { } async _readLine(parse) { + // if (this.firstRowToBeReturned) { + // const res = this.firstRowToBeReturned; + // this.firstRowToBeReturned = null; + // return res; + // } for (;;) { const line = await fetchNextLineFromReader(this.reader); if (!line) { @@ -70,7 +77,11 @@ class JsonLinesDatastore { if (!this.readedSchemaRow) { this.readedSchemaRow = true; - return true; + const parsedLine = JSON.parse(line); + if (parsedLine.__isStreamHeader) { + // skip to next line + continue; + } } if (this.currentFilter) { const parsedLine = JSON.parse(line); @@ -79,6 +90,7 @@ class JsonLinesDatastore { return parse ? parsedLine : true; } } else { + console.log('NO !!!'); this.readedDataRowCount += 1; return parse ? JSON.parse(line) : true; } @@ -130,11 +142,21 @@ class JsonLinesDatastore { this.reader = reader; this.currentFilter = filter; } - if (!this.readedSchemaRow) { - await this._readLine(false); // skip structure - } + // if (!this.readedSchemaRow) { + // const line = await this._readLine(true); // skip structure + // if (!line.__isStreamHeader) { + // // line contains data + // this.firstRowToBeReturned = line; + // } + // } while (this.readedDataRowCount < offset) { - await this._readLine(false); + const line = await this._readLine(false); + if (line == null) break; + // if (this.firstRowToBeReturned) { + // this.firstRowToBeReturned = null; + // } else { + // await this._readLine(false); + // } } } @@ -148,7 +170,6 @@ class JsonLinesDatastore { res.push(line); } }); - // console.log('RETURN', res.length); return res; } } diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 296041435..7e58fad1b 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -90,6 +90,7 @@ export abstract class GridDisplay { isLoadedCorrectly = true; supportsReload = false; isDynamicStructure = false; + filterTypeOverride = null; setColumnVisibility(uniquePath: string[], isVisible: boolean) { const uniqueName = uniquePath.join('.'); @@ -622,16 +623,16 @@ export abstract class GridDisplay { if (!filters) return null; const conditions = []; for (const name in filters) { - const column = this.columns.find(x => (x.columnName = name)); - if (!column) continue; - const filterType = getFilterType(column.dataType); + const column = this.isDynamicStructure ? null : this.columns.find(x => x.columnName == name); + if (!this.isDynamicStructure && !column) continue; + const filterType = this.isDynamicStructure ? this.filterTypeOverride ?? 'mongo' : getFilterType(column.dataType); try { const condition = parseFilter(filters[name], filterType); const replaced = _.cloneDeepWith(condition, (expr: Expression) => { if (expr.exprType == 'placeholder') return { exprType: 'column', - columnName: column.columnName, + columnName: this.isDynamicStructure ? name : column.columnName, }; }); conditions.push(replaced); diff --git a/packages/datalib/src/JslGridDisplay.ts b/packages/datalib/src/JslGridDisplay.ts index b371277f2..4b3bbef13 100644 --- a/packages/datalib/src/JslGridDisplay.ts +++ b/packages/datalib/src/JslGridDisplay.ts @@ -12,11 +12,14 @@ export class JslGridDisplay extends GridDisplay { setConfig: ChangeConfigFunc, cache: GridCache, setCache: ChangeCacheFunc, - rows: any + rows: any, + isDynamicStructure: boolean ) { super(config, setConfig, cache, setCache, null); this.filterable = true; + this.isDynamicStructure = isDynamicStructure; + if (isDynamicStructure) this.filterTypeOverride = 'string'; if (structure?.columns) { this.columns = _.uniqBy( diff --git a/packages/web/src/datagrid/JslDataGrid.svelte b/packages/web/src/datagrid/JslDataGrid.svelte index a6134f746..73571011c 100644 --- a/packages/web/src/datagrid/JslDataGrid.svelte +++ b/packages/web/src/datagrid/JslDataGrid.svelte @@ -16,7 +16,7 @@ const config = writable(createGridConfig()); const cache = writable(createGridCache()); - $: display = new JslGridDisplay(jslid, $info, $config, config.update, $cache, cache.update, loadedRows); + $: display = new JslGridDisplay(jslid, $info, $config, config.update, $cache, cache.update, loadedRows, $info?.__isDynamicStructure); {#key jslid} diff --git a/packages/web/src/tabs/JsonLinesEditorTab.svelte b/packages/web/src/tabs/JsonLinesEditorTab.svelte index 81512bd17..3bf4b3bf7 100644 --- a/packages/web/src/tabs/JsonLinesEditorTab.svelte +++ b/packages/web/src/tabs/JsonLinesEditorTab.svelte @@ -30,6 +30,7 @@ category: 'JSON Lines editor', name: 'Preview', icon: 'icon preview', + keyText: 'F5', testEnabled: () => getCurrentEditor() != null, onClick: () => getCurrentEditor().preview(), }); @@ -99,6 +100,7 @@ title: 'Preview #', icon: 'img archive', tabComponent: 'ArchiveFileTab', + forceNewTab: true, props: { jslid, },