preview in import dialog

packages/api/src/shell/archiveReader.js

@@ -2,9 +2,9 @@ const path = require('path');
 const { archivedir } = require('../utility/directories');
 const jsonLinesReader = require('./jsonLinesReader');
 
-function archiveReader({ folderName, fileName }) {
+function archiveReader({ folderName, fileName, ...other }) {
   const jsonlFile = path.join(archivedir(), folderName, `${fileName}.jsonl`);
-  const res = jsonLinesReader({ fileName: jsonlFile });
+  const res = jsonLinesReader({ fileName: jsonlFile, ...other });
   return res;
 }
 

packages/api/src/shell/csvReader.js

@@ -37,12 +37,13 @@ class CsvPrepareStream extends stream.Transform {
   }
 }
 
-async function csvReader({ fileName, encoding = 'utf-8', header = true, delimiter, quoted }) {
+async function csvReader({ fileName, encoding = 'utf-8', header = true, delimiter, quoted, limitRows = undefined }) {
   console.log(`Reading file ${fileName}`);
   const csvStream = csv.parse({
     // @ts-ignore
     delimiter,
     quoted,
+    to_line: limitRows ? limitRows + 1 : -1,
   });
   const fileStream = fs.createReadStream(fileName, encoding);
   const csvPrepare = new CsvPrepareStream({ header });

packages/api/src/shell/excelSheetReader.js

@@ -14,7 +14,7 @@ async function loadWorkbook(fileName) {
   return workbook;
 }
 
-async function excelSheetReader({ fileName, sheetName }) {
+async function excelSheetReader({ fileName, sheetName, limitRows = undefined }) {
   const workbook = await loadWorkbook(fileName);
   const sheet = workbook.getWorksheet(sheetName);
 
@@ -27,6 +27,7 @@ async function excelSheetReader({ fileName, sheetName }) {
   };
   pass.write(structure);
   for (let rowIndex = 2; rowIndex <= sheet.rowCount; rowIndex++) {
+    if (limitRows && rowIndex > limitRows + 1) break;
     const row = sheet.getRow(rowIndex);
     const rowData = _.fromPairs(structure.columns.map((col, index) => [col.columnName, row.getCell(index + 1).value]));
     if (_.isEmpty(_.omitBy(rowData, (v) => v == null || v.toString().trim().length == 0))) continue;

packages/api/src/shell/jslDataReader.js

@@ -1,9 +1,9 @@
 const getJslFileName = require('../utility/getJslFileName');
 const jsonLinesReader = require('./jsonLinesReader');
 
-function jslDataReader({ jslid }) {
+function jslDataReader({ jslid, ...other }) {
   const fileName = getJslFileName(jslid);
-  return jsonLinesReader({ fileName });
+  return jsonLinesReader({ fileName, ...other });
 }
 
 module.exports = jslDataReader;

packages/api/src/shell/jsonLinesReader.js

@@ -3,10 +3,12 @@ const stream = require('stream');
 const byline = require('byline');
 
 class ParseStream extends stream.Transform {
-  constructor({ header }) {
+  constructor({ header, limitRows }) {
     super({ objectMode: true });
     this.header = header;
     this.wasHeader = false;
+    this.limitRows = limitRows;
+    this.rowsWritten = 0;
   }
   _transform(chunk, encoding, done) {
     const obj = JSON.parse(chunk);
@@ -14,17 +16,19 @@ class ParseStream extends stream.Transform {
       if (!this.header) this.push({ columns: Object.keys(obj).map((columnName) => ({ columnName })) });
       this.wasHeader = true;
     }
-    this.push(obj);
+    if (!this.limitRows || this.rowsWritten < this.limitRows) {
+      this.push(obj);
+    }
     done();
   }
 }
 
-async function jsonLinesReader({ fileName, encoding = 'utf-8', header = true }) {
+async function jsonLinesReader({ fileName, encoding = 'utf-8', header = true, limitRows = undefined }) {
   console.log(`Reading file ${fileName}`);
 
   const fileStream = fs.createReadStream(fileName, encoding);
   const liner = byline(fileStream);
-  const parser = new ParseStream({ header });
+  const parser = new ParseStream({ header, limitRows });
   liner.pipe(parser);
   return parser;
 }

packages/api/src/shell/tableReader.js

@@ -3,7 +3,7 @@ const driverConnect = require('../utility/driverConnect');
 
 const engines = require('@dbgate/engines');
 
-async function queryReader({ connection, pureName, schemaName }) {
+async function tableReader({ connection, pureName, schemaName }) {
   const driver = engines(connection);
   const pool = await driverConnect(driver, connection);
   console.log(`Connected.`);
@@ -25,4 +25,4 @@ async function queryReader({ connection, pureName, schemaName }) {
   return await driver.readQuery(pool, query);
 }
 
-module.exports = queryReader;
+module.exports = tableReader;