From 4065e0501376131de53038010999b6723a59739c Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Tue, 17 Sep 2024 09:59:47 +0200 Subject: [PATCH] CSV import fixed --- .../__tests__/import-formats.spec.js | 23 ++++++++ plugins/dbgate-plugin-csv/package.json | 2 +- .../dbgate-plugin-csv/src/backend/reader.js | 1 + yarn.lock | 55 ++++++++----------- 4 files changed, 49 insertions(+), 32 deletions(-) diff --git a/integration-tests/__tests__/import-formats.spec.js b/integration-tests/__tests__/import-formats.spec.js index 4631531d6..9dfdbd541 100644 --- a/integration-tests/__tests__/import-formats.spec.js +++ b/integration-tests/__tests__/import-formats.spec.js @@ -27,6 +27,7 @@ test('csv import test', async () => { const reader = await dbgatePluginCsv.shellApi.reader({ fileName: csvFileName, + delimiter: ';', }); const writer = await dbgateApi.jsonLinesWriter({ @@ -39,5 +40,27 @@ test('csv import test', async () => { .split('\n') .filter(x => x.trim() !== '') .map(x => JSON.parse(x)); + expect(rows[0].columns).toEqual([ + { columnName: 'Issue Number' }, + { columnName: 'Title' }, + { columnName: 'Github URL' }, + { columnName: 'Labels' }, + { columnName: 'State' }, + { columnName: 'Created At' }, + { columnName: 'Updated At' }, + { columnName: 'Reporter' }, + { columnName: 'Assignee' }, + ]); expect(rows.length).toEqual(9); + expect(rows[1]).toEqual({ + 'Issue Number': '801', + Title: "Does it 'burst' the database on startup or first lUI load ? ", + 'Github URL': 'https://github.com/dbgate/dbgate/issues/801', + Labels: '', + State: 'open', + 'Created At': '05/23/2024', + 'Updated At': '05/23/2024', + Reporter: 'rgarrigue', + Assignee: '', + }); }); diff --git a/plugins/dbgate-plugin-csv/package.json b/plugins/dbgate-plugin-csv/package.json index 08fefed64..0519f5bec 100644 --- a/plugins/dbgate-plugin-csv/package.json +++ b/plugins/dbgate-plugin-csv/package.json @@ -32,7 +32,7 @@ "prepublishOnly": "yarn build" }, "devDependencies": { - "csv": "^5.3.2", + "csv": "^6.3.10", "dbgate-plugin-tools": "^1.0.7", "lodash": "^4.17.21", "webpack": "^5.91.0", diff --git a/plugins/dbgate-plugin-csv/src/backend/reader.js b/plugins/dbgate-plugin-csv/src/backend/reader.js index 6c07518f5..0429b8a68 100644 --- a/plugins/dbgate-plugin-csv/src/backend/reader.js +++ b/plugins/dbgate-plugin-csv/src/backend/reader.js @@ -51,6 +51,7 @@ async function reader({ fileName, encoding = 'utf-8', header = true, delimiter, delimiter, skip_lines_with_error: true, to_line: limitRows ? limitRows + 1 : undefined, + ltrim: true, }); const downloadedFile = await dbgateApi.download(fileName); const fileStream = fs.createReadStream(downloadedFile, encoding); diff --git a/yarn.lock b/yarn.lock index acc569e5f..824131276 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3116,30 +3116,30 @@ cssstyle@^2.3.0: dependencies: cssom "~0.3.6" -csv-generate@^3.4.3: - version "3.4.3" - resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-3.4.3.tgz#bc42d943b45aea52afa896874291da4b9108ffff" - integrity sha512-w/T+rqR0vwvHqWs/1ZyMDWtHHSJaN06klRqJXBEpDJaM/+dZkso0OKh1VcuuYvK3XM53KysVNq8Ko/epCK8wOw== +csv-generate@^4.4.1: + version "4.4.1" + resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-4.4.1.tgz#729781ace8d1b92f6bfb407d1ab9548728c55681" + integrity sha512-O/einO0v4zPmXaOV+sYqGa02VkST4GP5GLpWBNHEouIU7pF3kpGf3D0kCCvX82ydIY4EKkOK+R8b1BYsRXravg== -csv-parse@^4.16.3: - version "4.16.3" - resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.16.3.tgz#7ca624d517212ebc520a36873c3478fa66efbaf7" - integrity sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg== +csv-parse@^5.5.6: + version "5.5.6" + resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.5.6.tgz#0d726d58a60416361358eec291a9f93abe0b6b1a" + integrity sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A== -csv-stringify@^5.6.5: - version "5.6.5" - resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-5.6.5.tgz#c6d74badda4b49a79bf4e72f91cce1e33b94de00" - integrity sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A== +csv-stringify@^6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-6.5.1.tgz#a31837dd35e34787e3c248159c982a21af964d94" + integrity sha512-+9lpZfwpLntpTIEpFbwQyWuW/hmI/eHuJZD1XzeZpfZTqkf1fyvBbBLXTJJMsBuuS11uTShMqPwzx4A6ffXgRQ== -csv@^5.3.2: - version "5.5.3" - resolved "https://registry.yarnpkg.com/csv/-/csv-5.5.3.tgz#cd26c1e45eae00ce6a9b7b27dcb94955ec95207d" - integrity sha512-QTaY0XjjhTQOdguARF0lGKm5/mEq9PD9/VhZZegHDIBq2tQwgNpHc3dneD4mGo2iJs+fTKv5Bp0fZ+BRuY3Z0g== +csv@^6.3.10: + version "6.3.10" + resolved "https://registry.yarnpkg.com/csv/-/csv-6.3.10.tgz#960a3a9cef08573ecca2d80ddb71152aca383088" + integrity sha512-5NYZG4AN2ZUthmNxIudgBEdMPUnbQHu9V4QTzBPqQzUP3KQsFiJo+8HQ0+oVxj1PomIT1/f67VI1QH/hsrZLKA== dependencies: - csv-generate "^3.4.3" - csv-parse "^4.16.3" - csv-stringify "^5.6.5" - stream-transform "^2.1.3" + csv-generate "^4.4.1" + csv-parse "^5.5.6" + csv-stringify "^6.5.1" + stream-transform "^3.3.2" dashdash@^1.12.0: version "1.14.1" @@ -7456,11 +7456,6 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" -mixme@^0.5.1: - version "0.5.10" - resolved "https://registry.yarnpkg.com/mixme/-/mixme-0.5.10.tgz#d653b2984b75d9018828f1ea333e51717ead5f51" - integrity sha512-5H76ANWinB1H3twpJ6JY8uvAtpmFvHNArpilJAjXRKXSDDLPIMoZArw5SH0q9z+lLs8IrMw7Q2VWpWimFKFT1Q== - mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" @@ -9750,12 +9745,10 @@ stoppable@^1.1.0: resolved "https://registry.yarnpkg.com/stoppable/-/stoppable-1.1.0.tgz#32da568e83ea488b08e4d7ea2c3bcc9d75015d5b" integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw== -stream-transform@^2.1.3: - version "2.1.3" - resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-2.1.3.tgz#a1c3ecd72ddbf500aa8d342b0b9df38f5aa598e3" - integrity sha512-9GHUiM5hMiCi6Y03jD2ARC1ettBXkQBoQAe7nJsPknnI0ow10aXjTnew8QtYQmLjzn974BnmWEAJgCY6ZP1DeQ== - dependencies: - mixme "^0.5.1" +stream-transform@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-3.3.2.tgz#398c67b2f3b6ed5d04ceadde9e412bda8416c8ab" + integrity sha512-v64PUnPy9Qw94NGuaEMo+9RHQe4jTBYf+NkTtqkCgeuiNo8NlL0LtLR7fkKWNVFtp3RhIm5Dlxkgm5uz7TDimQ== streamsearch@^1.1.0: version "1.1.0"