change code structure

src/lib/database/_sql.js

@@ -0,0 +1,79 @@
+import initSqlJs from 'sql.js/dist/sql-wasm.js'
+import dbUtils from './_statements'
+
+let SQL = null
+const sqlModuleReady = initSqlJs().then(sqlModule => { SQL = sqlModule })
+
+export default class Sql {
+  constructor () {
+    this.db = null
+  }
+
+  static build () {
+    return sqlModuleReady
+      .then(() => {
+        return new Sql()
+      })
+  }
+
+  createDb (buffer) {
+    if (this.db != null) this.db.close()
+    this.db = new SQL.Database(buffer)
+    return this.db
+  }
+
+  open (buffer) {
+    this.createDb(buffer && new Uint8Array(buffer))
+    return {
+      ready: true
+    }
+  }
+
+  exec (sql, params) {
+    if (this.db === null) {
+      this.createDb()
+    }
+    if (!sql) {
+      throw new Error('exec: Missing query string')
+    }
+    return this.db.exec(sql, params)
+  }
+
+  import (columns, values, progressCounterId, progressCallback, chunkSize = 1500) {
+    this.createDb()
+    this.db.exec(dbUtils.getCreateStatement(columns, values))
+    const chunks = dbUtils.generateChunks(values, chunkSize)
+    const chunksAmount = Math.ceil(values.length / chunkSize)
+    let count = 0
+    const insertStr = dbUtils.getInsertStmt(columns)
+    const insertStmt = this.db.prepare(insertStr)
+
+    progressCallback({ progress: 0, id: progressCounterId })
+    for (const chunk of chunks) {
+      this.db.exec('BEGIN')
+      for (const row of chunk) {
+        insertStmt.run(row)
+      }
+      this.db.exec('COMMIT')
+      count++
+      progressCallback({ progress: 100 * (count / chunksAmount), id: progressCounterId })
+    }
+
+    return {
+      finish: true
+    }
+  }
+
+  export () {
+    return this.db.export()
+  }
+
+  close () {
+    if (this.db) {
+      this.db.close()
+    }
+    return {
+      finished: true
+    }
+  }
+}

src/lib/database/_statements.js

@@ -0,0 +1,44 @@
+export default {
+  * generateChunks (arr, size) {
+    const count = Math.ceil(arr.length / size)
+
+    for (let i = 0; i <= count - 1; i++) {
+      const start = size * i
+      const end = start + size
+      yield arr.slice(start, end)
+    }
+  },
+
+  getInsertStmt (columns) {
+    const colList = `"${columns.join('", "')}"`
+    const params = columns.map(() => '?').join(', ')
+    return `INSERT INTO csv_import (${colList}) VALUES (${params});`
+  },
+
+  getCreateStatement (columns, values) {
+    let result = 'CREATE table csv_import('
+    columns.forEach((col, index) => {
+    // Get the first row of values to determine types
+      const value = values[0][index]
+      let type = ''
+      switch (typeof value) {
+        case 'number': {
+          type = 'REAL'
+          break
+        }
+        case 'boolean': {
+          type = 'INTEGER'
+          break
+        }
+        case 'string': {
+          type = 'TEXT'
+          break
+        }
+        default: type = 'TEXT'
+      }
+      result += `"${col}" ${type}, `
+    })
+    result = result.replace(/,\s$/, ');')
+    return result
+  }
+}

src/lib/database/_worker.js

@@ -0,0 +1,34 @@
+import registerPromiseWorker from 'promise-worker/register'
+import Sql from './_sql'
+
+const sqlReady = Sql.build()
+
+function processMsg (sql) {
+  const data = this
+  switch (data && data.action) {
+    case 'open':
+      return sql.open(data.buffer)
+    case 'exec':
+      return sql.exec(data.sql, data.params)
+    case 'import':
+      return sql.import(data.columns, data.values, data.progressCounterId, postMessage)
+    case 'export':
+      return sql.export()
+    case 'close':
+      return sql.close()
+    default:
+      throw new Error('Invalid action : ' + (data && data.action))
+  }
+}
+
+function onError (error) {
+  return {
+    error: error.message
+  }
+}
+
+registerPromiseWorker(data => {
+  return sqlReady
+    .then(processMsg.bind(data))
+    .catch(onError)
+})

src/lib/database/index.js

@@ -0,0 +1,164 @@
+import sqliteParser from 'sqlite-parser'
+import fu from '@/lib/utils/fileIo'
+// We can import workers like so because of worker-loader:
+// https://webpack.js.org/loaders/worker-loader/
+import Worker from './_worker.js'
+
+// Use promise-worker in order to turn worker into the promise based one:
+// https://github.com/nolanlawson/promise-worker
+import PromiseWorker from 'promise-worker'
+
+function getNewDatabase () {
+  const worker = new Worker()
+  return new Database(worker)
+}
+
+export default {
+  getNewDatabase
+}
+
+let progressCounterIds = 0
+class Database {
+  constructor (worker) {
+    this.worker = worker
+    this.pw = new PromiseWorker(worker)
+
+    this.importProgresses = {}
+    worker.addEventListener('message', e => {
+      const progress = e.data.progress
+      if (progress !== undefined) {
+        const id = e.data.id
+        this.importProgresses[id].dispatchEvent(new CustomEvent('progress', {
+          detail: progress
+        }))
+      }
+    })
+  }
+
+  shutDown () {
+    this.worker.terminate()
+  }
+
+  createProgressCounter (callback) {
+    const id = progressCounterIds++
+    this.importProgresses[id] = new EventTarget()
+    this.importProgresses[id].addEventListener('progress', e => { callback(e.detail) })
+    return id
+  }
+
+  deleteProgressCounter (id) {
+    delete this.importProgresses[id]
+  }
+
+  async createDb (name, data, progressCounterId) {
+    const result = await this.pw.postMessage({
+      action: 'import',
+      columns: data.columns,
+      values: data.values,
+      progressCounterId
+    })
+
+    if (result.error) {
+      throw new Error(result.error)
+    }
+
+    return await this.getSchema(name)
+  }
+
+  async loadDb (file) {
+    const fileContent = await fu.readAsArrayBuffer(file)
+    const res = await this.pw.postMessage({ action: 'open', buffer: fileContent })
+
+    if (res.error) {
+      throw new Error(res.error)
+    }
+
+    return this.getSchema(file.name.replace(/\.[^.]+$/, ''))
+  }
+
+  async getSchema (name) {
+    const getSchemaSql = `
+      SELECT name, sql
+      FROM sqlite_master
+      WHERE type='table' AND name NOT LIKE 'sqlite_%';
+    `
+    const result = await this.execute(getSchemaSql)
+    // Parse DDL statements to get column names and types
+    const parsedSchema = []
+    result.values.forEach(item => {
+      parsedSchema.push({
+        name: item[0],
+        columns: getColumns(item[1])
+      })
+    })
+
+    // Return db name and schema
+    return {
+      dbName: name,
+      schema: parsedSchema
+    }
+  }
+
+  async execute (commands) {
+    const results = await this.pw.postMessage({ action: 'exec', sql: commands })
+
+    if (results.error) {
+      throw new Error(results.error)
+    }
+    // if it was more than one select - take only the last one
+    return results[results.length - 1]
+  }
+
+  async export (fileName) {
+    const data = await this.pw.postMessage({ action: 'export' })
+
+    if (data.error) {
+      throw new Error(data.error)
+    }
+    fu.exportToFile(data, fileName)
+  }
+}
+
+function getAst (sql) {
+  // There is a bug is sqlite-parser
+  // It throws an error if tokenizer has an arguments:
+  // https://github.com/codeschool/sqlite-parser/issues/59
+  const fixedSql = sql
+    .replace(/(?<=tokenize=.+)"tokenchars=.+"/, '')
+    .replace(/(?<=tokenize=.+)"remove_diacritics=.+"/, '')
+    .replace(/(?<=tokenize=.+)"separators=.+"/, '')
+    .replace(/tokenize=.+(?=(,|\)))/, 'tokenize=unicode61')
+
+  return sqliteParser(fixedSql)
+}
+
+/*
+ * Return an array of columns with name and type. E.g.:
+ * [
+ *   { name: 'id',    type: 'INTEGER' },
+ *   { name: 'title', type: 'NVARCHAR(30)' },
+ * ]
+*/
+function getColumns (sql) {
+  const columns = []
+  const ast = getAst(sql)
+
+  const columnDefinition = ast.statement[0].format === 'table'
+    ? ast.statement[0].definition
+    : ast.statement[0].result.args.expression // virtual table
+
+  columnDefinition.forEach(item => {
+    if (item.variant === 'column' && ['identifier', 'definition'].includes(item.type)) {
+      let type = item.datatype ? item.datatype.variant : 'N/A'
+      if (item.datatype && item.datatype.args) {
+        type = type + '(' + item.datatype.args.expression[0].value
+        if (item.datatype.args.expression.length === 2) {
+          type = type + ', ' + item.datatype.args.expression[1].value
+        }
+        type = type + ')'
+      }
+      columns.push({ name: item.name, type: type })
+    }
+  })
+  return columns
+}