CSV import as a table and db connection rework

- Add csv to existing db #32
- [RFE] Simplify working with temporary tables #53

src/lib/database/_sql.js

@@ -39,13 +39,15 @@ export default class Sql {
     return this.db.exec(sql, params)
   }
 
-  import (columns, values, progressCounterId, progressCallback, chunkSize = 1500) {
-    this.createDb()
-    this.db.exec(dbUtils.getCreateStatement(columns, values))
+  import (tabName, columns, values, progressCounterId, progressCallback, chunkSize = 1500) {
+    if (this.db === null) {
+      this.createDb()
+    }
+    this.db.exec(dbUtils.getCreateStatement(tabName, columns, values))
     const chunks = dbUtils.generateChunks(values, chunkSize)
     const chunksAmount = Math.ceil(values.length / chunkSize)
     let count = 0
-    const insertStr = dbUtils.getInsertStmt(columns)
+    const insertStr = dbUtils.getInsertStmt(tabName, columns)
     const insertStmt = this.db.prepare(insertStr)
 
     progressCallback({ progress: 0, id: progressCounterId })

src/lib/database/_statements.js

@@ -1,3 +1,5 @@
+import sqliteParser from 'sqlite-parser'
+
 export default {
   * generateChunks (arr, size) {
     const count = Math.ceil(arr.length / size)
@@ -9,14 +11,14 @@ export default {
     }
   },
 
-  getInsertStmt (columns) {
+  getInsertStmt (tabName, columns) {
     const colList = `"${columns.join('", "')}"`
     const params = columns.map(() => '?').join(', ')
-    return `INSERT INTO csv_import (${colList}) VALUES (${params});`
+    return `INSERT INTO "${tabName}" (${colList}) VALUES (${params});`
   },
 
-  getCreateStatement (columns, values) {
-    let result = 'CREATE table csv_import('
+  getCreateStatement (tabName, columns, values) {
+    let result = `CREATE table "${tabName}"(`
     columns.forEach((col, index) => {
     // Get the first row of values to determine types
       const value = values[0][index]
@@ -40,5 +42,49 @@ export default {
     })
     result = result.replace(/,\s$/, ');')
     return result
+  },
+
+  getAst (sql) {
+    // There is a bug is sqlite-parser
+    // It throws an error if tokenizer has an arguments:
+    // https://github.com/codeschool/sqlite-parser/issues/59
+    const fixedSql = sql
+      .replace(/(tokenize=[^,]+)"tokenchars=.+?"/, '$1')
+      .replace(/(tokenize=[^,]+)"remove_diacritics=.+?"/, '$1')
+      .replace(/(tokenize=[^,]+)"separators=.+?"/, '$1')
+      .replace(/tokenize=.+?(,|\))/, 'tokenize=unicode61$1')
+
+    return sqliteParser(fixedSql)
+  },
+
+  /*
+   * Return an array of columns with name and type. E.g.:
+   * [
+   *   { name: 'id',    type: 'INTEGER' },
+   *   { name: 'title', type: 'NVARCHAR(30)' },
+   * ]
+  */
+  getColumns (sql) {
+    const columns = []
+    const ast = this.getAst(sql)
+
+    const columnDefinition = ast.statement[0].format === 'table'
+      ? ast.statement[0].definition
+      : ast.statement[0].result.args.expression // virtual table
+
+    columnDefinition.forEach(item => {
+      if (item.variant === 'column' && ['identifier', 'definition'].includes(item.type)) {
+        let type = item.datatype ? item.datatype.variant : 'N/A'
+        if (item.datatype && item.datatype.args) {
+          type = type + '(' + item.datatype.args.expression[0].value
+          if (item.datatype.args.expression.length === 2) {
+            type = type + ', ' + item.datatype.args.expression[1].value
+          }
+          type = type + ')'
+        }
+        columns.push({ name: item.name, type: type })
+      }
+    })
+    return columns
   }
 }

src/lib/database/_worker.js

@@ -8,10 +8,18 @@ function processMsg (sql) {
   switch (data && data.action) {
     case 'open':
       return sql.open(data.buffer)
+    case 'reopen':
+      return sql.open(sql.export())
     case 'exec':
       return sql.exec(data.sql, data.params)
     case 'import':
-      return sql.import(data.columns, data.values, data.progressCounterId, postMessage)
+      return sql.import(
+        data.tabName,
+        data.columns,
+        data.values,
+        data.progressCounterId,
+        postMessage
+      )
     case 'export':
       return sql.export()
     case 'close':

src/lib/database/index.js

@@ -1,4 +1,4 @@
-import sqliteParser from 'sqlite-parser'
+import stms from './_statements'
 import fu from '@/lib/utils/fileIo'
 // We can import workers like so because of worker-loader:
 // https://webpack.js.org/loaders/worker-loader/
@@ -20,6 +20,8 @@ export default {
 let progressCounterIds = 0
 class Database {
   constructor (worker) {
+    this.dbName = null
+    this.schema = null
     this.worker = worker
     this.pw = new PromiseWorker(worker)
 
@@ -50,19 +52,20 @@ class Database {
     delete this.importProgresses[id]
   }
 
-  async importDb (name, data, progressCounterId) {
+  async addTableFromCsv (tabName, data, progressCounterId) {
     const result = await this.pw.postMessage({
       action: 'import',
       columns: data.columns,
       values: data.values,
-      progressCounterId
+      progressCounterId,
+      tabName
     })
 
     if (result.error) {
       throw new Error(result.error)
     }
-
-    return await this.getSchema(name)
+    this.dbName = this.dbName || 'database'
+    this.refreshSchema()
   }
 
   async loadDb (file) {
@@ -73,11 +76,11 @@ class Database {
       throw new Error(res.error)
     }
 
-    const dbName = file ? file.name.replace(/\.[^.]+$/, '') : 'database'
-    return this.getSchema(dbName)
+    this.dbName = file ? fu.getFileName(file) : 'database'
+    this.refreshSchema()
   }
 
-  async getSchema (name) {
+  async refreshSchema () {
     const getSchemaSql = `
       SELECT name, sql
       FROM sqlite_master
@@ -90,19 +93,17 @@ class Database {
       result.values.forEach(item => {
         parsedSchema.push({
           name: item[0],
-          columns: getColumns(item[1])
+          columns: stms.getColumns(item[1])
         })
       })
     }
 
-    // Return db name and schema
-    return {
-      dbName: name,
-      schema: parsedSchema
-    }
+    // Refresh schema
+    this.schema = parsedSchema
   }
 
   async execute (commands) {
+    await this.pw.postMessage({ action: 'reopen' })
     const results = await this.pw.postMessage({ action: 'exec', sql: commands })
 
     if (results.error) {
@@ -120,48 +121,27 @@ class Database {
     }
     fu.exportToFile(data, fileName)
   }
-}
 
-function getAst (sql) {
-  // There is a bug is sqlite-parser
-  // It throws an error if tokenizer has an arguments:
-  // https://github.com/codeschool/sqlite-parser/issues/59
-  const fixedSql = sql
-    .replace(/(tokenize=[^,]+)"tokenchars=.+?"/, '$1')
-    .replace(/(tokenize=[^,]+)"remove_diacritics=.+?"/, '$1')
-    .replace(/(tokenize=[^,]+)"separators=.+?"/, '$1')
-    .replace(/tokenize=.+?(,|\))/, 'tokenize=unicode61$1')
-
-  return sqliteParser(fixedSql)
-}
-
-/*
- * Return an array of columns with name and type. E.g.:
- * [
- *   { name: 'id',    type: 'INTEGER' },
- *   { name: 'title', type: 'NVARCHAR(30)' },
- * ]
-*/
-function getColumns (sql) {
-  const columns = []
-  const ast = getAst(sql)
-
-  const columnDefinition = ast.statement[0].format === 'table'
-    ? ast.statement[0].definition
-    : ast.statement[0].result.args.expression // virtual table
-
-  columnDefinition.forEach(item => {
-    if (item.variant === 'column' && ['identifier', 'definition'].includes(item.type)) {
-      let type = item.datatype ? item.datatype.variant : 'N/A'
-      if (item.datatype && item.datatype.args) {
-        type = type + '(' + item.datatype.args.expression[0].value
-        if (item.datatype.args.expression.length === 2) {
-          type = type + ', ' + item.datatype.args.expression[1].value
-        }
-        type = type + ')'
-      }
-      columns.push({ name: item.name, type: type })
+  async validateTableName (name) {
+    if (name.startsWith('sqlite_')) {
+      throw new Error("Table name can't start with sqlite_")
     }
-  })
-  return columns
+
+    if (/[^\w]/.test(name)) {
+      throw new Error('Table name can contain only letters, digits and underscores')
+    }
+
+    if (/^(\d)/.test(name)) {
+      throw new Error("Table name can't start with a digit")
+    }
+
+    await this.execute(`BEGIN; CREATE TABLE "${name}"(id); ROLLBACK;`)
+  }
+
+  sanitizeTableName (tabName) {
+    return tabName
+      .replace(/[^\w]/g, '_') // replace everything that is not letter, digit or _  with _
+      .replace(/^(\d)/, '_$1') // add _ at beginning if starts with digit
+      .replace(/_{2,}/g, '_') // replace multiple _ with one _
+  }
 }

src/lib/utils/fileIo.js

@@ -6,6 +6,10 @@ export default {
       : /\.(db|sqlite(3)?)+$/.test(file.name)
   },
 
+  getFileName (file) {
+    return file.name.replace(/\.[^.]+$/, '')
+  },
+
   exportToFile (str, fileName, type = 'octet/stream') {
     // Create downloader
     const downloader = document.createElement('a')

src/lib/utils/time.js

@@ -3,5 +3,13 @@ export default {
     const diff = end.getTime() - start.getTime()
     const seconds = diff / 1000
     return seconds.toFixed(3) + 's'
+  },
+
+  debounce (func, ms) {
+    let timeout
+    return function () {
+      clearTimeout(timeout)
+      timeout = setTimeout(() => func.apply(this, arguments), ms)
+    }
   }
 }