CSV import as a table and db connection rework

- Add csv to existing db #32
- [RFE] Simplify working with temporary tables #53

tests/lib/database/_sql.spec.js

@@ -74,8 +74,8 @@ describe('_sql.js', () => {
     const progressCallback = sinon.stub()
     const progressCounterId = 1
     const sql = await Sql.build()
-    sql.import(data.columns, data.values, progressCounterId, progressCallback, 2)
-    const result = sql.exec('SELECT * from csv_import')
+    sql.import('foo', data.columns, data.values, progressCounterId, progressCallback, 2)
+    const result = sql.exec('SELECT * from foo')
     expect(result).to.have.lengthOf(1)
     expect(result[0].columns).to.eql(['id', 'name'])
     expect(result[0].values).to.have.lengthOf(4)
@@ -135,7 +135,7 @@ describe('_sql.js', () => {
     expect(sql.db.db).to.equal(null)
   })
 
-  it('overwrites', async () => {
+  it('adds', async () => {
     const sql = await Sql.build()
     sql.exec(`
       CREATE TABLE test (
@@ -160,12 +160,11 @@ describe('_sql.js', () => {
         [4, 'Ron Weasley']
       ]
     }
-    // rewrite the database by import
-    sql.import(data.columns, data.values, 1, sinon.stub(), 2)
-    result = sql.exec('SELECT * from csv_import')
+    // import adds table
+    sql.import('foo', data.columns, data.values, 1, sinon.stub(), 2)
+    result = sql.exec('SELECT * from foo')
     expect(result[0].values).to.have.lengthOf(4)
-
-    // test table oesn't exists anymore: the db was overwritten
-    expect(() => { sql.exec('SELECT * from test') }).to.throw('no such table: test')
+    result = sql.exec('SELECT * from test')
+    expect(result[0].values).to.have.lengthOf(2)
   })
 })

tests/lib/database/_statements.spec.js

@@ -1,11 +1,11 @@
 import { expect } from 'chai'
-import dbUtils from '@/lib/database/_statements'
+import stmts from '@/lib/database/_statements'
 
 describe('_statements.js', () => {
   it('generateChunks', () => {
     const arr = ['1', '2', '3', '4', '5']
     const size = 2
-    const chunks = dbUtils.generateChunks(arr, size)
+    const chunks = stmts.generateChunks(arr, size)
     const output = []
     for (const chunk of chunks) {
       output.push(chunk)
@@ -17,8 +17,8 @@ describe('_statements.js', () => {
 
   it('getInsertStmt', () => {
     const columns = ['id', 'name']
-    expect(dbUtils.getInsertStmt(columns))
-      .to.equal('INSERT INTO csv_import ("id", "name") VALUES (?, ?);')
+    expect(stmts.getInsertStmt('foo', columns))
+      .to.equal('INSERT INTO "foo" ("id", "name") VALUES (?, ?);')
   })
 
   it('getCreateStatement', () => {
@@ -27,8 +27,36 @@ describe('_statements.js', () => {
       [1, 'foo', true, new Date()],
       [2, 'bar', false, new Date()]
     ]
-    expect(dbUtils.getCreateStatement(columns, values)).to.equal(
-      'CREATE table csv_import("id" REAL, "name" TEXT, "isAdmin" INTEGER, "startDate" TEXT);'
+    expect(stmts.getCreateStatement('foo', columns, values)).to.equal(
+      'CREATE table "foo"("id" REAL, "name" TEXT, "isAdmin" INTEGER, "startDate" TEXT);'
     )
   })
+
+  it('getColumns', () => {
+    const sql = `CREATE TABLE test (
+      col1,
+      col2 integer,
+      col3 decimal(5,2),
+      col4 varchar(30)
+    )`
+    expect(stmts.getColumns(sql)).to.eql([
+      { name: 'col1', type: 'N/A' },
+      { name: 'col2', type: 'integer' },
+      { name: 'col3', type: 'decimal(5, 2)' },
+      { name: 'col4', type: 'varchar(30)' }
+    ])
+  })
+
+  it('getColumns with virtual table', async () => {
+    const sql = `
+      CREATE VIRTUAL TABLE test_virtual USING fts4(
+        col1, col2,
+        notindexed=col1, notindexed=col2,
+        tokenize=unicode61 "tokenchars=.+#")
+    `
+    expect(stmts.getColumns(sql)).to.eql([
+      { name: 'col1', type: 'N/A' },
+      { name: 'col2', type: 'N/A' }
+    ])
+  })
 })

tests/lib/database/database.spec.js

@@ -27,58 +27,39 @@ describe('database.js', () => {
     const tempDb = new SQL.Database()
     tempDb.run(`CREATE TABLE test (
       col1,
-      col2 integer,
-      col3 decimal(5,2),
-      col4 varchar(30)
+      col2 integer
     )`)
 
     const data = tempDb.export()
     const buffer = new Blob([data])
     buffer.name = 'foo.sqlite'
 
-    const { schema, dbName } = await db.loadDb(buffer)
-    expect(dbName).to.equal('foo')
+    sinon.spy(db, 'refreshSchema')
+
+    await db.loadDb(buffer)
+    await db.refreshSchema.returnValues[0]
+    const schema = db.schema
+    expect(db.dbName).to.equal('foo')
     expect(schema).to.have.lengthOf(1)
     expect(schema[0].name).to.equal('test')
+
     expect(schema[0].columns[0].name).to.equal('col1')
     expect(schema[0].columns[0].type).to.equal('N/A')
+
     expect(schema[0].columns[1].name).to.equal('col2')
     expect(schema[0].columns[1].type).to.equal('integer')
-    expect(schema[0].columns[2].name).to.equal('col3')
-    expect(schema[0].columns[2].type).to.equal('decimal(5, 2)')
-    expect(schema[0].columns[3].name).to.equal('col4')
-    expect(schema[0].columns[3].type).to.equal('varchar(30)')
   })
 
-  it('creates schema with virtual table', async () => {
-    const SQL = await getSQL
-    const tempDb = new SQL.Database()
-    tempDb.run(`
-      CREATE VIRTUAL TABLE test_virtual USING fts4(
-        col1, col2,
-        notindexed=col1, notindexed=col2,
-        tokenize=unicode61 "tokenchars=.+#")
-    `)
+  it('creates empty db with name database', async () => {
+    sinon.spy(db, 'refreshSchema')
 
-    const data = tempDb.export()
-    const buffer = new Blob([data])
-    buffer.name = 'foo.sqlite'
-
-    const { schema } = await db.loadDb(buffer)
-    expect(schema[0].name).to.equal('test_virtual')
-    expect(schema[0].columns[0].name).to.equal('col1')
-    expect(schema[0].columns[0].type).to.equal('N/A')
-    expect(schema[0].columns[1].name).to.equal('col2')
-    expect(schema[0].columns[1].type).to.equal('N/A')
+    await db.loadDb()
+    await db.refreshSchema.returnValues[0]
+    expect(db.dbName).to.equal('database')
   })
 
   it('loadDb throws errors', async () => {
-    const SQL = await getSQL
-    const tempDb = new SQL.Database()
-    tempDb.run('CREATE TABLE test (col1, col2)')
-
-    const data = tempDb.export()
-    const buffer = new Blob([data])
+    const buffer = new Blob([])
     buffer.name = 'foo.sqlite'
 
     sinon.stub(db.pw, 'postMessage').resolves({ error: new Error('foo') })
@@ -136,7 +117,7 @@ describe('database.js', () => {
     await expect(db.execute('SELECT * from foo')).to.be.rejectedWith(/^no such table: foo$/)
   })
 
-  it('creates db', async () => {
+  it('adds table from csv', async () => {
     const data = {
       columns: ['id', 'name', 'faculty'],
       values: [
@@ -146,16 +127,19 @@ describe('database.js', () => {
     }
     const progressHandler = sinon.spy()
     const progressCounterId = db.createProgressCounter(progressHandler)
-    const { dbName, schema } = await db.importDb('foo', data, progressCounterId)
-    expect(dbName).to.equal('foo')
-    expect(schema).to.have.lengthOf(1)
-    expect(schema[0].name).to.equal('csv_import')
-    expect(schema[0].columns).to.have.lengthOf(3)
-    expect(schema[0].columns[0]).to.eql({ name: 'id', type: 'real' })
-    expect(schema[0].columns[1]).to.eql({ name: 'name', type: 'text' })
-    expect(schema[0].columns[2]).to.eql({ name: 'faculty', type: 'text' })
+    sinon.spy(db, 'refreshSchema')
 
-    const result = await db.execute('SELECT * from csv_import')
+    await db.addTableFromCsv('foo', data, progressCounterId)
+    await db.refreshSchema.returnValues[0]
+    expect(db.dbName).to.equal('database')
+    expect(db.schema).to.have.lengthOf(1)
+    expect(db.schema[0].name).to.equal('foo')
+    expect(db.schema[0].columns).to.have.lengthOf(3)
+    expect(db.schema[0].columns[0]).to.eql({ name: 'id', type: 'real' })
+    expect(db.schema[0].columns[1]).to.eql({ name: 'name', type: 'text' })
+    expect(db.schema[0].columns[2]).to.eql({ name: 'faculty', type: 'text' })
+
+    const result = await db.execute('SELECT * from foo')
     expect(result.columns).to.eql(data.columns)
     expect(result.values).to.eql(data.values)
 
@@ -164,7 +148,7 @@ describe('database.js', () => {
     expect(progressHandler.secondCall.calledWith(100)).to.equal(true)
   })
 
-  it('importDb throws errors', async () => {
+  it('addTableFromCsv throws errors', async () => {
     const data = {
       columns: ['id', 'name'],
       values: [
@@ -174,7 +158,7 @@ describe('database.js', () => {
     }
     const progressHandler = sinon.stub()
     const progressCounterId = db.createProgressCounter(progressHandler)
-    await expect(db.importDb('foo', data, progressCounterId))
+    await expect(db.addTableFromCsv('foo', data, progressCounterId))
       .to.be.rejectedWith('column index out of range')
   })
 
@@ -242,4 +226,23 @@ describe('database.js', () => {
     expect(result.values).to.have.lengthOf(1)
     expect(result.values[0]).to.eql([1, 'Harry Potter'])
   })
+
+  it('sanitizeTableName', () => {
+    let name = 'foo[]bar'
+    expect(db.sanitizeTableName(name)).to.equal('foo_bar')
+
+    name = '1 foo(01.05.2020)'
+    expect(db.sanitizeTableName(name)).to.equal('_1_foo_01_05_2020_')
+  })
+
+  it('validateTableName', async () => {
+    await db.execute('CREATE TABLE foo(id)')
+    await expect(db.validateTableName('foo')).to.be.rejectedWith('table "foo" already exists')
+    await expect(db.validateTableName('1foo'))
+      .to.be.rejectedWith("Table name can't start with a digit")
+    await expect(db.validateTableName('foo(05.08.2020)'))
+      .to.be.rejectedWith('Table name can contain only letters, digits and underscores')
+    await expect(db.validateTableName('sqlite_foo'))
+      .to.be.rejectedWith("Table name can't start with sqlite_")
+  })
 })