CSV - completely remove, logic moved to plugin

2026-04-17 23:45:59 +00:00 · 2020-11-22 09:17:51 +01:00
parent 3cdba4339f
commit e23e749cc5
6 changed files with 1 additions and 159 deletions
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -15,7 +15,6 @@
    "mssql",
    "mssql",
    "postgresql",
-    "csv",
    "excel",
    "json",
    "import",
@@ -30,7 +29,6 @@
    "byline": "^5.0.0",
    "cors": "^2.8.5",
    "cross-env": "^6.0.3",
-    "csv": "^5.3.2",
    "dbgate-engines": "^1.0.0",
    "dbgate-sqltree": "^1.0.0",
    "dbgate-tools": "^1.0.0",
--- a/packages/api/src/controllers/uploads.js
+++ b/packages/api/src/controllers/uploads.js
@@ -2,21 +2,6 @@ const path = require('path');
 const { uploadsdir } = require('../utility/directories');
 const uuidv1 = require('uuid/v1');

-// const extensions = [
-//   {
-//     ext: '.xlsx',
-//     type: 'excel',
-//   },
-//   {
-//     ext: '.jsonl',
-//     type: 'jsonl',
-//   },
-//   {
-//     ext: '.csv',
-//     type: 'csv',
-//   },
-// ];
-
 module.exports = {
  upload_meta: {
    method: 'post',
@@ -31,19 +16,10 @@ module.exports = {
    const uploadName = uuidv1();
    const filePath = path.join(uploadsdir(), uploadName);
    console.log(`Uploading file ${data.name}, size=${data.size}`);
-    // let storageType = null;
-    // let shortName = data.name;
-    // for (const { ext, type } of extensions) {
-    //   if (data.name.endsWith(ext)) {
-    //     storageType = type;
-    //     shortName = data.name.slice(0, -ext.length);
-    //   }
-    // }
+
    data.mv(filePath, () => {
      res.json({
        originalName: data.name,
-        // shortName,
-        // storageType,
        uploadName,
        filePath,
      });
--- a/packages/api/src/shell/csvReader.js
+++ b/packages/api/src/shell/csvReader.js
@@ -1,55 +0,0 @@
-const _ = require('lodash');
-const csv = require('csv');
-const fs = require('fs');
-const stream = require('stream');
-
-class CsvPrepareStream extends stream.Transform {
-  constructor({ header }) {
-    super({ objectMode: true });
-    this.structure = null;
-    this.header = header;
-  }
-  _transform(chunk, encoding, done) {
-    if (this.structure) {
-      this.push(
-        _.zipObject(
-          this.structure.columns.map((x) => x.columnName),
-          chunk
-        )
-      );
-      done();
-    } else {
-      if (this.header) {
-        this.structure = { columns: chunk.map((columnName) => ({ columnName })) };
-        this.push(this.structure);
-      } else {
-        this.structure = { columns: chunk.map((value, index) => ({ columnName: `col${index + 1}` })) };
-        this.push(this.structure);
-        this.push(
-          _.zipObject(
-            this.structure.columns.map((x) => x.columnName),
-            chunk
-          )
-        );
-      }
-      done();
-    }
-  }
-}
-
-async function csvReader({ fileName, encoding = 'utf-8', header = true, delimiter, limitRows = undefined }) {
-  console.log(`Reading file ${fileName}`);
-  const csvStream = csv.parse({
-    // @ts-ignore
-    delimiter,
-    skip_lines_with_error: true,
-    to_line: limitRows ? limitRows + 1 : -1,
-  });
-  const fileStream = fs.createReadStream(fileName, encoding);
-  const csvPrepare = new CsvPrepareStream({ header });
-  fileStream.pipe(csvStream);
-  csvStream.pipe(csvPrepare);
-  return csvPrepare;
-}
-
-module.exports = csvReader;
--- a/packages/api/src/shell/csvWriter.js
+++ b/packages/api/src/shell/csvWriter.js
@@ -1,36 +0,0 @@
-const csv = require('csv');
-const fs = require('fs');
-const stream = require('stream');
-
-class CsvPrepareStream extends stream.Transform {
-  constructor({ header }) {
-    super({ objectMode: true });
-    this.structure = null;
-    this.header = header;
-  }
-  _transform(chunk, encoding, done) {
-    if (this.structure) {
-      this.push(this.structure.columns.map((col) => chunk[col.columnName]));
-      done();
-    } else {
-      this.structure = chunk;
-      if (this.header) {
-        this.push(chunk.columns.map((x) => x.columnName));
-      }
-      done();
-    }
-  }
-}
-
-async function csvWriter({ fileName, encoding = 'utf-8', header = true, delimiter, quoted }) {
-  console.log(`Writing file ${fileName}`);
-  const csvPrepare = new CsvPrepareStream({ header });
-  const csvStream = csv.stringify({ delimiter, quoted });
-  const fileStream = fs.createWriteStream(fileName, encoding);
-  csvPrepare.pipe(csvStream);
-  csvStream.pipe(fileStream);
-  csvPrepare['finisher'] = fileStream;
-  return csvPrepare;
-}
-
-module.exports = csvWriter;
--- a/packages/api/src/shell/index.js
+++ b/packages/api/src/shell/index.js
@@ -1,6 +1,4 @@
 const queryReader = require('./queryReader');
-const csvWriter = require('./csvWriter');
-const csvReader = require('./csvReader');
 const runScript = require('./runScript');
 const tableWriter = require('./tableWriter');
 const tableReader = require('./tableReader');
@@ -19,8 +17,6 @@ const finalizer = require('./finalizer');

 module.exports = {
  queryReader,
-  csvWriter,
-  csvReader,
  runScript,
  tableWriter,
  tableReader,
--- a/yarn.lock
+++ b/yarn.lock
@@ -4069,31 +4069,6 @@ csstype@^2.5.7, csstype@^2.6.7:
  resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.10.tgz#e63af50e66d7c266edb6b32909cfd0aabe03928b"
  integrity sha512-D34BqZU4cIlMCY93rZHbrq9pjTAQJ3U8S8rfBqjwHxkGPThWFjzZDQpgMJY0QViLxth6ZKYiwFBo14RdN44U/w==

-csv-generate@^3.2.4:
-  version "3.2.4"
-  resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-3.2.4.tgz#440dab9177339ee0676c9e5c16f50e2b3463c019"
-  integrity sha512-qNM9eqlxd53TWJeGtY1IQPj90b563Zx49eZs8e0uMyEvPgvNVmX1uZDtdzAcflB3PniuH9creAzcFOdyJ9YGvA==
-
-csv-parse@^4.8.8:
-  version "4.10.1"
-  resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.10.1.tgz#1e26ba63d29c75e94d0eba6e9de9a8aaf89d72a6"
-  integrity sha512-gdDJVchi0oSLIcYXz1H/VSgLE6htHDqJyFsRU/vTkQgmVOZ3S0IR2LXnNbWUYG7VD76dYVwdfBLyx8AX9+An8A==
-
-csv-stringify@^5.3.6:
-  version "5.5.0"
-  resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-5.5.0.tgz#0bdeaaf60d6e15b89c752a0eceb4b4c2c8af5a8a"
-  integrity sha512-G05575DSO/9vFzQxZN+Srh30cNyHk0SM0ePyiTChMD5WVt7GMTVPBQf4rtgMF6mqhNCJUPw4pN8LDe8MF9EYOA==
-
-csv@^5.3.2:
-  version "5.3.2"
-  resolved "https://registry.yarnpkg.com/csv/-/csv-5.3.2.tgz#50b344e25dfbb8c62684a1bcec18c22468b2161e"
-  integrity sha512-odDyucr9OgJTdGM2wrMbJXbOkJx3nnUX3Pt8SFOwlAMOpsUQlz1dywvLMXJWX/4Ib0rjfOsaawuuwfI5ucqBGQ==
-  dependencies:
-    csv-generate "^3.2.4"
-    csv-parse "^4.8.8"
-    csv-stringify "^5.3.6"
-    stream-transform "^2.0.1"
-
 cyclist@^1.0.1:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"
@@ -8271,11 +8246,6 @@ mixin-object@^2.0.1:
    for-in "^0.1.3"
    is-extendable "^0.1.1"

-mixme@^0.3.1:
-  version "0.3.5"
-  resolved "https://registry.yarnpkg.com/mixme/-/mixme-0.3.5.tgz#304652cdaf24a3df0487205e61ac6162c6906ddd"
-  integrity sha512-SyV9uPETRig5ZmYev0ANfiGeB+g6N2EnqqEfBbCGmmJ6MgZ3E4qv5aPbnHVdZ60KAHHXV+T3sXopdrnIXQdmjQ==
-
 mkdirp@0.5.1, mkdirp@0.x, mkdirp@^0.5.1, mkdirp@~0.5.1:
  version "0.5.1"
  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
@@ -11926,13 +11896,6 @@ stream-shift@^1.0.0:
  resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d"
  integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==

-stream-transform@^2.0.1:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-2.0.2.tgz#3cb7a14c802eb39bc40caaab0535e584f3a65caf"
-  integrity sha512-J+D5jWPF/1oX+r9ZaZvEXFbu7znjxSkbNAHJ9L44bt/tCVuOEWZlDqU9qJk7N2xBU1S+K2DPpSKeR/MucmCA1Q==
-  dependencies:
-    mixme "^0.3.1"
-
 streamsearch@0.1.2:
  version "0.1.2"
  resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"