add test for csv chunks #60

.github/workflows/test.yml

@@ -4,9 +4,6 @@ on:
   push:
     branches:
     - 'master'
-  pull_request:
-    branches:
-    - 'master'
 
 jobs:
   test:

package-lock.json

@@ -12,7 +12,7 @@
         "codemirror": "^5.57.0",
         "core-js": "^3.6.5",
         "nanoid": "^3.1.12",
-        "papaparse": "^5.3.1",
+        "papaparse": "^5.3.0",
         "plotly.js": "^1.58.4",
         "promise-worker": "^2.0.1",
         "react": "^16.13.1",
@@ -15448,9 +15448,9 @@
       "dev": true
     },
     "node_modules/papaparse": {
-      "version": "5.3.1",
-      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.1.tgz",
-      "integrity": "sha512-Dbt2yjLJrCwH2sRqKFFJaN5XgIASO9YOFeFP8rIBRG2Ain8mqk5r1M6DkfvqEVozVcz3r3HaUGw253hA1nLIcA=="
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.0.tgz",
+      "integrity": "sha512-Lb7jN/4bTpiuGPrYy4tkKoUS8sTki8zacB5ke1p5zolhcSE4TlWgrlsxjrDTbG/dFVh07ck7X36hUf/b5V68pg=="
     },
     "node_modules/parallel-transform": {
       "version": "1.2.0",
@@ -36573,9 +36573,9 @@
       "dev": true
     },
     "papaparse": {
-      "version": "5.3.1",
-      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.1.tgz",
-      "integrity": "sha512-Dbt2yjLJrCwH2sRqKFFJaN5XgIASO9YOFeFP8rIBRG2Ain8mqk5r1M6DkfvqEVozVcz3r3HaUGw253hA1nLIcA=="
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.0.tgz",
+      "integrity": "sha512-Lb7jN/4bTpiuGPrYy4tkKoUS8sTki8zacB5ke1p5zolhcSE4TlWgrlsxjrDTbG/dFVh07ck7X36hUf/b5V68pg=="
     },
     "parallel-transform": {
       "version": "1.2.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sqliteviz",
-  "version": "0.13.2",
+  "version": "0.13.0",
   "license": "Apache-2.0",
   "private": true,
   "scripts": {
@@ -13,7 +13,7 @@
     "codemirror": "^5.57.0",
     "core-js": "^3.6.5",
     "nanoid": "^3.1.12",
-    "papaparse": "^5.3.1",
+    "papaparse": "^5.3.0",
     "plotly.js": "^1.58.4",
     "promise-worker": "^2.0.1",
     "react": "^16.13.1",

src/components/CsvImport/csv.js

@@ -13,14 +13,7 @@ export default {
       result.columns = source.meta.fields.map(col => col.trim())
       result.values = source.data.map(row => {
         const resultRow = []
-        source.meta.fields.forEach(col => {
-          let value = row[col]
-          if (value instanceof Date) {
-            value = value.toISOString()
-          }
-          resultRow.push(value)
-        })
-
+        result.columns.forEach(col => { resultRow.push(row[col]) })
         return resultRow
       })
     } else {
@@ -35,6 +28,7 @@ export default {
   },
 
   parse (file, config = {}) {
+    let parsedData = null
     return new Promise((resolve, reject) => {
       const defaultConfig = {
         delimiter: '', // auto-detect
@@ -49,13 +43,22 @@ export default {
         worker: true,
         comments: false,
         step: undefined,
-        complete: results => {
+        chunk: results => {
+          if (parsedData === null) {
+            parsedData = results
+          } else {
+            parsedData.data = parsedData.data.concat(results.data)
+            parsedData.errors = parsedData.errors.concat(results.errors)
+          }
+        },
+        chunkSize: 1024 * 716,
+        complete: () => {
           const res = {
-            data: this.getResult(results),
-            delimiter: results.meta.delimiter,
+            data: this.getResult(parsedData),
+            delimiter: parsedData.meta.delimiter,
             hasErrors: false
           }
-          res.messages = results.errors.map(msg => {
+          res.messages = parsedData.errors.map(msg => {
             msg.type = msg.code === 'UndetectableDelimiter' ? 'info' : 'error'
             if (msg.type === 'error') res.hasErrors = true
             msg.hint = hintsByCode[msg.code]
@@ -70,8 +73,6 @@ export default {
         downloadRequestHeaders: undefined,
         downloadRequestBody: undefined,
         skipEmptyLines: 'greedy',
-        chunk: undefined,
-        chunkSize: undefined,
         fastMode: undefined,
         beforeFirstChunk: undefined,
         withCredentials: undefined,

tests/components/CsvImport/csv.spec.js

@@ -11,18 +11,18 @@ describe('csv.js', () => {
   it('getResult with fields', () => {
     const source = {
       data: [
-        { id: 1, 'name ': 'foo', date: new Date('2021-06-30T14:10:24.717Z') },
-        { id: 2, 'name ': 'bar', date: new Date('2021-07-30T14:10:15.717Z') }
+        { id: 1, name: 'foo' },
+        { id: 2, name: 'bar' }
       ],
       meta: {
-        fields: ['id', 'name ', 'date']
+        fields: ['id', 'name ']
       }
     }
     expect(csv.getResult(source)).to.eql({
-      columns: ['id', 'name', 'date'],
+      columns: ['id', 'name'],
       values: [
-        [1, 'foo', '2021-06-30T14:10:24.717Z'],
-        [2, 'bar', '2021-07-30T14:10:15.717Z']
+        [1, 'foo'],
+        [2, 'bar']
       ]
     })
   })
@@ -46,7 +46,7 @@ describe('csv.js', () => {
 
   it('parse resolves', async () => {
     sinon.stub(Papa, 'parse').callsFake((file, config) => {
-      config.complete({
+      config.chunk({
         data: [
           [1, 'foo'],
           [2, 'bar']
@@ -72,6 +72,7 @@ describe('csv.js', () => {
           truncated: true
         }
       })
+      config.complete()
     })
     const file = {}
     const result = await csv.parse(file)
@@ -112,4 +113,96 @@ describe('csv.js', () => {
     const file = {}
     await expect(csv.parse(file)).to.be.rejectedWith(err)
   })
+
+  it('concat chunks', async () => {
+    sinon.stub(Papa, 'parse').callsFake((file, config) => {
+      config.chunk({
+        data: [
+          [1, 'foo'],
+          [2, 'bar']
+        ],
+        errors: [
+          {
+            type: 'Quotes',
+            code: 'MissingQuotes',
+            message: 'Quote is missed',
+            row: 0
+          },
+          {
+            type: 'Delimiter',
+            code: 'UndetectableDelimiter',
+            message: 'Comma was used as a standart delimiter',
+            row: 0
+          }
+        ],
+        meta: {
+          delimiter: ',',
+          linebreak: '\n',
+          aborted: false,
+          truncated: true
+        }
+      })
+
+      config.chunk({
+        data: [
+          [3, 'baz'],
+          [4, 'boo']
+        ],
+        errors: [
+          {
+            type: 'Delimiter',
+            code: 'UndetectableDelimiter',
+            message: 'Comma was used as a standart delimiter',
+            row: 2
+          }
+        ],
+        meta: {
+          delimiter: ',',
+          linebreak: '\n',
+          aborted: false,
+          truncated: true
+        }
+      })
+      config.complete()
+    })
+    const file = {}
+    const result = await csv.parse(file)
+
+    expect(result).to.eql({
+      data: {
+        columns: ['col1', 'col2'],
+        values: [
+          [1, 'foo'],
+          [2, 'bar'],
+          [3, 'baz'],
+          [4, 'boo']
+        ]
+      },
+      delimiter: ',',
+      hasErrors: true,
+      messages: [
+        {
+          code: 'MissingQuotes',
+          message: 'Quote is missed',
+          row: 0,
+          type: 'error',
+          hint: 'Edit your CSV so that the field has a closing quote char.'
+        },
+        {
+          code: 'UndetectableDelimiter',
+          message: 'Comma was used as a standart delimiter',
+          row: 0,
+          type: 'info',
+          hint: undefined
+        },
+        {
+          code: 'UndetectableDelimiter',
+          message: 'Comma was used as a standart delimiter',
+          row: 2,
+          type: 'info',
+          hint: undefined
+        }
+      ]
+    })
+  })
 })