diff --git a/packages/api/package.json b/packages/api/package.json index 91a53b46a..e4482f94f 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -15,7 +15,6 @@ "mssql", "mssql", "postgresql", - "csv", "excel", "json", "import", @@ -30,7 +29,6 @@ "byline": "^5.0.0", "cors": "^2.8.5", "cross-env": "^6.0.3", - "csv": "^5.3.2", "dbgate-engines": "^1.0.0", "dbgate-sqltree": "^1.0.0", "dbgate-tools": "^1.0.0", diff --git a/packages/api/src/controllers/uploads.js b/packages/api/src/controllers/uploads.js index fc48f9b8d..342a7be8c 100644 --- a/packages/api/src/controllers/uploads.js +++ b/packages/api/src/controllers/uploads.js @@ -2,21 +2,6 @@ const path = require('path'); const { uploadsdir } = require('../utility/directories'); const uuidv1 = require('uuid/v1'); -// const extensions = [ -// { -// ext: '.xlsx', -// type: 'excel', -// }, -// { -// ext: '.jsonl', -// type: 'jsonl', -// }, -// { -// ext: '.csv', -// type: 'csv', -// }, -// ]; - module.exports = { upload_meta: { method: 'post', @@ -31,19 +16,10 @@ module.exports = { const uploadName = uuidv1(); const filePath = path.join(uploadsdir(), uploadName); console.log(`Uploading file ${data.name}, size=${data.size}`); - // let storageType = null; - // let shortName = data.name; - // for (const { ext, type } of extensions) { - // if (data.name.endsWith(ext)) { - // storageType = type; - // shortName = data.name.slice(0, -ext.length); - // } - // } + data.mv(filePath, () => { res.json({ originalName: data.name, - // shortName, - // storageType, uploadName, filePath, }); diff --git a/packages/api/src/shell/csvReader.js b/packages/api/src/shell/csvReader.js deleted file mode 100644 index 25261c8d0..000000000 --- a/packages/api/src/shell/csvReader.js +++ /dev/null @@ -1,55 +0,0 @@ -const _ = require('lodash'); -const csv = require('csv'); -const fs = require('fs'); -const stream = require('stream'); - -class CsvPrepareStream extends stream.Transform { - constructor({ header }) { - super({ objectMode: true }); - this.structure = null; - this.header = header; - } - _transform(chunk, encoding, done) { - if (this.structure) { - this.push( - _.zipObject( - this.structure.columns.map((x) => x.columnName), - chunk - ) - ); - done(); - } else { - if (this.header) { - this.structure = { columns: chunk.map((columnName) => ({ columnName })) }; - this.push(this.structure); - } else { - this.structure = { columns: chunk.map((value, index) => ({ columnName: `col${index + 1}` })) }; - this.push(this.structure); - this.push( - _.zipObject( - this.structure.columns.map((x) => x.columnName), - chunk - ) - ); - } - done(); - } - } -} - -async function csvReader({ fileName, encoding = 'utf-8', header = true, delimiter, limitRows = undefined }) { - console.log(`Reading file ${fileName}`); - const csvStream = csv.parse({ - // @ts-ignore - delimiter, - skip_lines_with_error: true, - to_line: limitRows ? limitRows + 1 : -1, - }); - const fileStream = fs.createReadStream(fileName, encoding); - const csvPrepare = new CsvPrepareStream({ header }); - fileStream.pipe(csvStream); - csvStream.pipe(csvPrepare); - return csvPrepare; -} - -module.exports = csvReader; diff --git a/packages/api/src/shell/csvWriter.js b/packages/api/src/shell/csvWriter.js deleted file mode 100644 index a5297c651..000000000 --- a/packages/api/src/shell/csvWriter.js +++ /dev/null @@ -1,36 +0,0 @@ -const csv = require('csv'); -const fs = require('fs'); -const stream = require('stream'); - -class CsvPrepareStream extends stream.Transform { - constructor({ header }) { - super({ objectMode: true }); - this.structure = null; - this.header = header; - } - _transform(chunk, encoding, done) { - if (this.structure) { - this.push(this.structure.columns.map((col) => chunk[col.columnName])); - done(); - } else { - this.structure = chunk; - if (this.header) { - this.push(chunk.columns.map((x) => x.columnName)); - } - done(); - } - } -} - -async function csvWriter({ fileName, encoding = 'utf-8', header = true, delimiter, quoted }) { - console.log(`Writing file ${fileName}`); - const csvPrepare = new CsvPrepareStream({ header }); - const csvStream = csv.stringify({ delimiter, quoted }); - const fileStream = fs.createWriteStream(fileName, encoding); - csvPrepare.pipe(csvStream); - csvStream.pipe(fileStream); - csvPrepare['finisher'] = fileStream; - return csvPrepare; -} - -module.exports = csvWriter; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index a7143d375..644608716 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -1,6 +1,4 @@ const queryReader = require('./queryReader'); -const csvWriter = require('./csvWriter'); -const csvReader = require('./csvReader'); const runScript = require('./runScript'); const tableWriter = require('./tableWriter'); const tableReader = require('./tableReader'); @@ -19,8 +17,6 @@ const finalizer = require('./finalizer'); module.exports = { queryReader, - csvWriter, - csvReader, runScript, tableWriter, tableReader, diff --git a/yarn.lock b/yarn.lock index 7879c854f..08c18f2d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4069,31 +4069,6 @@ csstype@^2.5.7, csstype@^2.6.7: resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.10.tgz#e63af50e66d7c266edb6b32909cfd0aabe03928b" integrity sha512-D34BqZU4cIlMCY93rZHbrq9pjTAQJ3U8S8rfBqjwHxkGPThWFjzZDQpgMJY0QViLxth6ZKYiwFBo14RdN44U/w== -csv-generate@^3.2.4: - version "3.2.4" - resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-3.2.4.tgz#440dab9177339ee0676c9e5c16f50e2b3463c019" - integrity sha512-qNM9eqlxd53TWJeGtY1IQPj90b563Zx49eZs8e0uMyEvPgvNVmX1uZDtdzAcflB3PniuH9creAzcFOdyJ9YGvA== - -csv-parse@^4.8.8: - version "4.10.1" - resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.10.1.tgz#1e26ba63d29c75e94d0eba6e9de9a8aaf89d72a6" - integrity sha512-gdDJVchi0oSLIcYXz1H/VSgLE6htHDqJyFsRU/vTkQgmVOZ3S0IR2LXnNbWUYG7VD76dYVwdfBLyx8AX9+An8A== - -csv-stringify@^5.3.6: - version "5.5.0" - resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-5.5.0.tgz#0bdeaaf60d6e15b89c752a0eceb4b4c2c8af5a8a" - integrity sha512-G05575DSO/9vFzQxZN+Srh30cNyHk0SM0ePyiTChMD5WVt7GMTVPBQf4rtgMF6mqhNCJUPw4pN8LDe8MF9EYOA== - -csv@^5.3.2: - version "5.3.2" - resolved "https://registry.yarnpkg.com/csv/-/csv-5.3.2.tgz#50b344e25dfbb8c62684a1bcec18c22468b2161e" - integrity sha512-odDyucr9OgJTdGM2wrMbJXbOkJx3nnUX3Pt8SFOwlAMOpsUQlz1dywvLMXJWX/4Ib0rjfOsaawuuwfI5ucqBGQ== - dependencies: - csv-generate "^3.2.4" - csv-parse "^4.8.8" - csv-stringify "^5.3.6" - stream-transform "^2.0.1" - cyclist@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" @@ -8271,11 +8246,6 @@ mixin-object@^2.0.1: for-in "^0.1.3" is-extendable "^0.1.1" -mixme@^0.3.1: - version "0.3.5" - resolved "https://registry.yarnpkg.com/mixme/-/mixme-0.3.5.tgz#304652cdaf24a3df0487205e61ac6162c6906ddd" - integrity sha512-SyV9uPETRig5ZmYev0ANfiGeB+g6N2EnqqEfBbCGmmJ6MgZ3E4qv5aPbnHVdZ60KAHHXV+T3sXopdrnIXQdmjQ== - mkdirp@0.5.1, mkdirp@0.x, mkdirp@^0.5.1, mkdirp@~0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" @@ -11926,13 +11896,6 @@ stream-shift@^1.0.0: resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== -stream-transform@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-2.0.2.tgz#3cb7a14c802eb39bc40caaab0535e584f3a65caf" - integrity sha512-J+D5jWPF/1oX+r9ZaZvEXFbu7znjxSkbNAHJ9L44bt/tCVuOEWZlDqU9qJk7N2xBU1S+K2DPpSKeR/MucmCA1Q== - dependencies: - mixme "^0.3.1" - streamsearch@0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"