fix csv import with ISO dates #64

.github/workflows/test.yml

@@ -4,6 +4,9 @@ on:
   push:
     branches:
     - 'master'
+  pull_request:
+    branches:
+    - 'master'
 
 jobs:
   test:

package-lock.json

@@ -12,7 +12,7 @@
         "codemirror": "^5.57.0",
         "core-js": "^3.6.5",
         "nanoid": "^3.1.12",
-        "papaparse": "^5.3.0",
+        "papaparse": "^5.3.1",
         "plotly.js": "^1.58.4",
         "promise-worker": "^2.0.1",
         "react": "^16.13.1",
@@ -15448,9 +15448,9 @@
       "dev": true
     },
     "node_modules/papaparse": {
-      "version": "5.3.0",
+      "version": "5.3.1",
-      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.1.tgz",
-      "integrity": "sha512-Lb7jN/4bTpiuGPrYy4tkKoUS8sTki8zacB5ke1p5zolhcSE4TlWgrlsxjrDTbG/dFVh07ck7X36hUf/b5V68pg=="
+      "integrity": "sha512-Dbt2yjLJrCwH2sRqKFFJaN5XgIASO9YOFeFP8rIBRG2Ain8mqk5r1M6DkfvqEVozVcz3r3HaUGw253hA1nLIcA=="
     },
     "node_modules/parallel-transform": {
       "version": "1.2.0",
@@ -36573,9 +36573,9 @@
       "dev": true
     },
     "papaparse": {
-      "version": "5.3.0",
+      "version": "5.3.1",
-      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.1.tgz",
-      "integrity": "sha512-Lb7jN/4bTpiuGPrYy4tkKoUS8sTki8zacB5ke1p5zolhcSE4TlWgrlsxjrDTbG/dFVh07ck7X36hUf/b5V68pg=="
+      "integrity": "sha512-Dbt2yjLJrCwH2sRqKFFJaN5XgIASO9YOFeFP8rIBRG2Ain8mqk5r1M6DkfvqEVozVcz3r3HaUGw253hA1nLIcA=="
     },
     "parallel-transform": {
       "version": "1.2.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sqliteviz",
-  "version": "0.13.0",
+  "version": "0.13.2",
   "license": "Apache-2.0",
   "private": true,
   "scripts": {
@@ -13,7 +13,7 @@
     "codemirror": "^5.57.0",
     "core-js": "^3.6.5",
     "nanoid": "^3.1.12",
-    "papaparse": "^5.3.0",
+    "papaparse": "^5.3.1",
     "plotly.js": "^1.58.4",
     "promise-worker": "^2.0.1",
     "react": "^16.13.1",

src/components/CsvImport/csv.js

@@ -13,7 +13,14 @@ export default {
       result.columns = source.meta.fields.map(col => col.trim())
       result.values = source.data.map(row => {
         const resultRow = []
-        result.columns.forEach(col => { resultRow.push(row[col]) })
+        source.meta.fields.forEach(col => {
+          let value = row[col]
+          if (value instanceof Date) {
+            value = value.toISOString()
+          }
+          resultRow.push(value)
+        })
+
         return resultRow
       })
     } else {
@@ -28,7 +35,6 @@ export default {
   },
 
   parse (file, config = {}) {
-    let parsedData = null
     return new Promise((resolve, reject) => {
       const defaultConfig = {
         delimiter: '', // auto-detect
@@ -43,22 +49,13 @@ export default {
         worker: true,
         comments: false,
         step: undefined,
-        chunk: results => {
+        complete: results => {
-          if (parsedData === null) {
-            parsedData = results
-          } else {
-            parsedData.data = parsedData.data.concat(results.data)
-            parsedData.errors = parsedData.errors.concat(results.errors)
-          }
-        },
-        chunkSize: 1024 * 716,
-        complete: () => {
           const res = {
-            data: this.getResult(parsedData),
+            data: this.getResult(results),
-            delimiter: parsedData.meta.delimiter,
+            delimiter: results.meta.delimiter,
             hasErrors: false
           }
-          res.messages = parsedData.errors.map(msg => {
+          res.messages = results.errors.map(msg => {
             msg.type = msg.code === 'UndetectableDelimiter' ? 'info' : 'error'
             if (msg.type === 'error') res.hasErrors = true
             msg.hint = hintsByCode[msg.code]
@@ -73,6 +70,8 @@ export default {
         downloadRequestHeaders: undefined,
         downloadRequestBody: undefined,
         skipEmptyLines: 'greedy',
+        chunk: undefined,
+        chunkSize: undefined,
         fastMode: undefined,
         beforeFirstChunk: undefined,
         withCredentials: undefined,

tests/components/CsvImport/csv.spec.js

@@ -11,18 +11,18 @@ describe('csv.js', () => {
   it('getResult with fields', () => {
     const source = {
       data: [
-        { id: 1, name: 'foo' },
+        { id: 1, 'name ': 'foo', date: new Date('2021-06-30T14:10:24.717Z') },
-        { id: 2, name: 'bar' }
+        { id: 2, 'name ': 'bar', date: new Date('2021-07-30T14:10:15.717Z') }
       ],
       meta: {
-        fields: ['id', 'name ']
+        fields: ['id', 'name ', 'date']
       }
     }
     expect(csv.getResult(source)).to.eql({
-      columns: ['id', 'name'],
+      columns: ['id', 'name', 'date'],
       values: [
-        [1, 'foo'],
+        [1, 'foo', '2021-06-30T14:10:24.717Z'],
-        [2, 'bar']
+        [2, 'bar', '2021-07-30T14:10:15.717Z']
       ]
     })
   })
@@ -46,7 +46,7 @@ describe('csv.js', () => {
 
   it('parse resolves', async () => {
     sinon.stub(Papa, 'parse').callsFake((file, config) => {
-      config.chunk({
+      config.complete({
         data: [
           [1, 'foo'],
           [2, 'bar']
@@ -72,7 +72,6 @@ describe('csv.js', () => {
           truncated: true
         }
       })
-      config.complete()
     })
     const file = {}
     const result = await csv.parse(file)
@@ -113,96 +112,4 @@ describe('csv.js', () => {
     const file = {}
     await expect(csv.parse(file)).to.be.rejectedWith(err)
   })
-
-  it('concat chunks', async () => {
-    sinon.stub(Papa, 'parse').callsFake((file, config) => {
-      config.chunk({
-        data: [
-          [1, 'foo'],
-          [2, 'bar']
-        ],
-        errors: [
-          {
-            type: 'Quotes',
-            code: 'MissingQuotes',
-            message: 'Quote is missed',
-            row: 0
-          },
-          {
-            type: 'Delimiter',
-            code: 'UndetectableDelimiter',
-            message: 'Comma was used as a standart delimiter',
-            row: 0
-          }
-        ],
-        meta: {
-          delimiter: ',',
-          linebreak: '\n',
-          aborted: false,
-          truncated: true
-        }
-      })
-
-      config.chunk({
-        data: [
-          [3, 'baz'],
-          [4, 'boo']
-        ],
-        errors: [
-          {
-            type: 'Delimiter',
-            code: 'UndetectableDelimiter',
-            message: 'Comma was used as a standart delimiter',
-            row: 2
-          }
-        ],
-        meta: {
-          delimiter: ',',
-          linebreak: '\n',
-          aborted: false,
-          truncated: true
-        }
-      })
-      config.complete()
-    })
-    const file = {}
-    const result = await csv.parse(file)
-
-    expect(result).to.eql({
-      data: {
-        columns: ['col1', 'col2'],
-        values: [
-          [1, 'foo'],
-          [2, 'bar'],
-          [3, 'baz'],
-          [4, 'boo']
-        ]
-      },
-      delimiter: ',',
-      hasErrors: true,
-      messages: [
-        {
-          code: 'MissingQuotes',
-          message: 'Quote is missed',
-          row: 0,
-          type: 'error',
-          hint: 'Edit your CSV so that the field has a closing quote char.'
-        },
-        {
-          code: 'UndetectableDelimiter',
-          message: 'Comma was used as a standart delimiter',
-          row: 0,
-          type: 'info',
-          hint: undefined
-        },
-        {
-          code: 'UndetectableDelimiter',
-          message: 'Comma was used as a standart delimiter',
-          row: 2,
-          type: 'info',
-          hint: undefined
-        }
-      ]
-    })
-  })
 })