add test for csv chunks #60

src/components/CsvImport/csv.js

@@ -28,6 +28,7 @@ export default {
   },
 
   parse (file, config = {}) {
+    let parsedData = null
     return new Promise((resolve, reject) => {
       const defaultConfig = {
         delimiter: '', // auto-detect
@@ -42,13 +43,22 @@ export default {
         worker: true,
         comments: false,
         step: undefined,
-        complete: results => {
+        chunk: results => {
+          if (parsedData === null) {
+            parsedData = results
+          } else {
+            parsedData.data = parsedData.data.concat(results.data)
+            parsedData.errors = parsedData.errors.concat(results.errors)
+          }
+        },
+        chunkSize: 1024 * 716,
+        complete: () => {
           const res = {
-            data: this.getResult(results),
+            data: this.getResult(parsedData),
-            delimiter: results.meta.delimiter,
+            delimiter: parsedData.meta.delimiter,
             hasErrors: false
           }
-          res.messages = results.errors.map(msg => {
+          res.messages = parsedData.errors.map(msg => {
             msg.type = msg.code === 'UndetectableDelimiter' ? 'info' : 'error'
             if (msg.type === 'error') res.hasErrors = true
             msg.hint = hintsByCode[msg.code]
@@ -63,8 +73,6 @@ export default {
         downloadRequestHeaders: undefined,
         downloadRequestBody: undefined,
         skipEmptyLines: 'greedy',
-        chunk: undefined,
-        chunkSize: undefined,
         fastMode: undefined,
         beforeFirstChunk: undefined,
         withCredentials: undefined,

tests/components/CsvImport/csv.spec.js

@@ -46,7 +46,7 @@ describe('csv.js', () => {
 
   it('parse resolves', async () => {
     sinon.stub(Papa, 'parse').callsFake((file, config) => {
-      config.complete({
+      config.chunk({
         data: [
           [1, 'foo'],
           [2, 'bar']
@@ -72,6 +72,7 @@ describe('csv.js', () => {
           truncated: true
         }
       })
+      config.complete()
     })
     const file = {}
     const result = await csv.parse(file)
@@ -112,4 +113,96 @@ describe('csv.js', () => {
     const file = {}
     await expect(csv.parse(file)).to.be.rejectedWith(err)
   })
+
+  it('concat chunks', async () => {
+    sinon.stub(Papa, 'parse').callsFake((file, config) => {
+      config.chunk({
+        data: [
+          [1, 'foo'],
+          [2, 'bar']
+        ],
+        errors: [
+          {
+            type: 'Quotes',
+            code: 'MissingQuotes',
+            message: 'Quote is missed',
+            row: 0
+          },
+          {
+            type: 'Delimiter',
+            code: 'UndetectableDelimiter',
+            message: 'Comma was used as a standart delimiter',
+            row: 0
+          }
+        ],
+        meta: {
+          delimiter: ',',
+          linebreak: '\n',
+          aborted: false,
+          truncated: true
+        }
+      })
+
+      config.chunk({
+        data: [
+          [3, 'baz'],
+          [4, 'boo']
+        ],
+        errors: [
+          {
+            type: 'Delimiter',
+            code: 'UndetectableDelimiter',
+            message: 'Comma was used as a standart delimiter',
+            row: 2
+          }
+        ],
+        meta: {
+          delimiter: ',',
+          linebreak: '\n',
+          aborted: false,
+          truncated: true
+        }
+      })
+      config.complete()
+    })
+    const file = {}
+    const result = await csv.parse(file)
+
+    expect(result).to.eql({
+      data: {
+        columns: ['col1', 'col2'],
+        values: [
+          [1, 'foo'],
+          [2, 'bar'],
+          [3, 'baz'],
+          [4, 'boo']
+        ]
+      },
+      delimiter: ',',
+      hasErrors: true,
+      messages: [
+        {
+          code: 'MissingQuotes',
+          message: 'Quote is missed',
+          row: 0,
+          type: 'error',
+          hint: 'Edit your CSV so that the field has a closing quote char.'
+        },
+        {
+          code: 'UndetectableDelimiter',
+          message: 'Comma was used as a standart delimiter',
+          row: 0,
+          type: 'info',
+          hint: undefined
+        },
+        {
+          code: 'UndetectableDelimiter',
+          message: 'Comma was used as a standart delimiter',
+          row: 2,
+          type: 'info',
+          hint: undefined
+        }
+      ]
+    })
+  })
 })