change code structure

src/lib/database/_sql.js

@@ -0,0 +1,79 @@
+import initSqlJs from 'sql.js/dist/sql-wasm.js'
+import dbUtils from './_statements'
+
+let SQL = null
+const sqlModuleReady = initSqlJs().then(sqlModule => { SQL = sqlModule })
+
+export default class Sql {
+  constructor () {
+    this.db = null
+  }
+
+  static build () {
+    return sqlModuleReady
+      .then(() => {
+        return new Sql()
+      })
+  }
+
+  createDb (buffer) {
+    if (this.db != null) this.db.close()
+    this.db = new SQL.Database(buffer)
+    return this.db
+  }
+
+  open (buffer) {
+    this.createDb(buffer && new Uint8Array(buffer))
+    return {
+      ready: true
+    }
+  }
+
+  exec (sql, params) {
+    if (this.db === null) {
+      this.createDb()
+    }
+    if (!sql) {
+      throw new Error('exec: Missing query string')
+    }
+    return this.db.exec(sql, params)
+  }
+
+  import (columns, values, progressCounterId, progressCallback, chunkSize = 1500) {
+    this.createDb()
+    this.db.exec(dbUtils.getCreateStatement(columns, values))
+    const chunks = dbUtils.generateChunks(values, chunkSize)
+    const chunksAmount = Math.ceil(values.length / chunkSize)
+    let count = 0
+    const insertStr = dbUtils.getInsertStmt(columns)
+    const insertStmt = this.db.prepare(insertStr)
+
+    progressCallback({ progress: 0, id: progressCounterId })
+    for (const chunk of chunks) {
+      this.db.exec('BEGIN')
+      for (const row of chunk) {
+        insertStmt.run(row)
+      }
+      this.db.exec('COMMIT')
+      count++
+      progressCallback({ progress: 100 * (count / chunksAmount), id: progressCounterId })
+    }
+
+    return {
+      finish: true
+    }
+  }
+
+  export () {
+    return this.db.export()
+  }
+
+  close () {
+    if (this.db) {
+      this.db.close()
+    }
+    return {
+      finished: true
+    }
+  }
+}

src/lib/database/_statements.js

@@ -0,0 +1,44 @@
+export default {
+  * generateChunks (arr, size) {
+    const count = Math.ceil(arr.length / size)
+
+    for (let i = 0; i <= count - 1; i++) {
+      const start = size * i
+      const end = start + size
+      yield arr.slice(start, end)
+    }
+  },
+
+  getInsertStmt (columns) {
+    const colList = `"${columns.join('", "')}"`
+    const params = columns.map(() => '?').join(', ')
+    return `INSERT INTO csv_import (${colList}) VALUES (${params});`
+  },
+
+  getCreateStatement (columns, values) {
+    let result = 'CREATE table csv_import('
+    columns.forEach((col, index) => {
+    // Get the first row of values to determine types
+      const value = values[0][index]
+      let type = ''
+      switch (typeof value) {
+        case 'number': {
+          type = 'REAL'
+          break
+        }
+        case 'boolean': {
+          type = 'INTEGER'
+          break
+        }
+        case 'string': {
+          type = 'TEXT'
+          break
+        }
+        default: type = 'TEXT'
+      }
+      result += `"${col}" ${type}, `
+    })
+    result = result.replace(/,\s$/, ');')
+    return result
+  }
+}

src/lib/database/_worker.js

@@ -0,0 +1,34 @@
+import registerPromiseWorker from 'promise-worker/register'
+import Sql from './_sql'
+
+const sqlReady = Sql.build()
+
+function processMsg (sql) {
+  const data = this
+  switch (data && data.action) {
+    case 'open':
+      return sql.open(data.buffer)
+    case 'exec':
+      return sql.exec(data.sql, data.params)
+    case 'import':
+      return sql.import(data.columns, data.values, data.progressCounterId, postMessage)
+    case 'export':
+      return sql.export()
+    case 'close':
+      return sql.close()
+    default:
+      throw new Error('Invalid action : ' + (data && data.action))
+  }
+}
+
+function onError (error) {
+  return {
+    error: error.message
+  }
+}
+
+registerPromiseWorker(data => {
+  return sqlReady
+    .then(processMsg.bind(data))
+    .catch(onError)
+})

src/lib/database/index.js

@@ -0,0 +1,164 @@
+import sqliteParser from 'sqlite-parser'
+import fu from '@/lib/utils/fileIo'
+// We can import workers like so because of worker-loader:
+// https://webpack.js.org/loaders/worker-loader/
+import Worker from './_worker.js'
+
+// Use promise-worker in order to turn worker into the promise based one:
+// https://github.com/nolanlawson/promise-worker
+import PromiseWorker from 'promise-worker'
+
+function getNewDatabase () {
+  const worker = new Worker()
+  return new Database(worker)
+}
+
+export default {
+  getNewDatabase
+}
+
+let progressCounterIds = 0
+class Database {
+  constructor (worker) {
+    this.worker = worker
+    this.pw = new PromiseWorker(worker)
+
+    this.importProgresses = {}
+    worker.addEventListener('message', e => {
+      const progress = e.data.progress
+      if (progress !== undefined) {
+        const id = e.data.id
+        this.importProgresses[id].dispatchEvent(new CustomEvent('progress', {
+          detail: progress
+        }))
+      }
+    })
+  }
+
+  shutDown () {
+    this.worker.terminate()
+  }
+
+  createProgressCounter (callback) {
+    const id = progressCounterIds++
+    this.importProgresses[id] = new EventTarget()
+    this.importProgresses[id].addEventListener('progress', e => { callback(e.detail) })
+    return id
+  }
+
+  deleteProgressCounter (id) {
+    delete this.importProgresses[id]
+  }
+
+  async createDb (name, data, progressCounterId) {
+    const result = await this.pw.postMessage({
+      action: 'import',
+      columns: data.columns,
+      values: data.values,
+      progressCounterId
+    })
+
+    if (result.error) {
+      throw new Error(result.error)
+    }
+
+    return await this.getSchema(name)
+  }
+
+  async loadDb (file) {
+    const fileContent = await fu.readAsArrayBuffer(file)
+    const res = await this.pw.postMessage({ action: 'open', buffer: fileContent })
+
+    if (res.error) {
+      throw new Error(res.error)
+    }
+
+    return this.getSchema(file.name.replace(/\.[^.]+$/, ''))
+  }
+
+  async getSchema (name) {
+    const getSchemaSql = `
+      SELECT name, sql
+      FROM sqlite_master
+      WHERE type='table' AND name NOT LIKE 'sqlite_%';
+    `
+    const result = await this.execute(getSchemaSql)
+    // Parse DDL statements to get column names and types
+    const parsedSchema = []
+    result.values.forEach(item => {
+      parsedSchema.push({
+        name: item[0],
+        columns: getColumns(item[1])
+      })
+    })
+
+    // Return db name and schema
+    return {
+      dbName: name,
+      schema: parsedSchema
+    }
+  }
+
+  async execute (commands) {
+    const results = await this.pw.postMessage({ action: 'exec', sql: commands })
+
+    if (results.error) {
+      throw new Error(results.error)
+    }
+    // if it was more than one select - take only the last one
+    return results[results.length - 1]
+  }
+
+  async export (fileName) {
+    const data = await this.pw.postMessage({ action: 'export' })
+
+    if (data.error) {
+      throw new Error(data.error)
+    }
+    fu.exportToFile(data, fileName)
+  }
+}
+
+function getAst (sql) {
+  // There is a bug is sqlite-parser
+  // It throws an error if tokenizer has an arguments:
+  // https://github.com/codeschool/sqlite-parser/issues/59
+  const fixedSql = sql
+    .replace(/(?<=tokenize=.+)"tokenchars=.+"/, '')
+    .replace(/(?<=tokenize=.+)"remove_diacritics=.+"/, '')
+    .replace(/(?<=tokenize=.+)"separators=.+"/, '')
+    .replace(/tokenize=.+(?=(,|\)))/, 'tokenize=unicode61')
+
+  return sqliteParser(fixedSql)
+}
+
+/*
+ * Return an array of columns with name and type. E.g.:
+ * [
+ *   { name: 'id',    type: 'INTEGER' },
+ *   { name: 'title', type: 'NVARCHAR(30)' },
+ * ]
+*/
+function getColumns (sql) {
+  const columns = []
+  const ast = getAst(sql)
+
+  const columnDefinition = ast.statement[0].format === 'table'
+    ? ast.statement[0].definition
+    : ast.statement[0].result.args.expression // virtual table
+
+  columnDefinition.forEach(item => {
+    if (item.variant === 'column' && ['identifier', 'definition'].includes(item.type)) {
+      let type = item.datatype ? item.datatype.variant : 'N/A'
+      if (item.datatype && item.datatype.args) {
+        type = type + '(' + item.datatype.args.expression[0].value
+        if (item.datatype.args.expression.length === 2) {
+          type = type + ', ' + item.datatype.args.expression[1].value
+        }
+        type = type + ')'
+      }
+      columns.push({ name: item.name, type: type })
+    }
+  })
+  return columns
+}

src/lib/storedQueries.js

@@ -0,0 +1,96 @@
+import { nanoid } from 'nanoid'
+import fu from '@/lib/utils/fileIo'
+
+export default {
+  getStoredQueries () {
+    return JSON.parse(localStorage.getItem('myQueries')) || []
+  },
+
+  duplicateQuery (baseQuery) {
+    const newQuery = JSON.parse(JSON.stringify(baseQuery))
+    newQuery.name = newQuery.name + ' Copy'
+    newQuery.id = nanoid()
+    newQuery.createdAt = new Date()
+    delete newQuery.isPredefined
+
+    return newQuery
+  },
+
+  isTabNeedName (queryTab) {
+    const isFromScratch = !queryTab.initName
+    return queryTab.isPredefined || isFromScratch
+  },
+
+  save (queryTab, newName) {
+    const value = {
+      id: queryTab.isPredefined ? nanoid() : queryTab.id,
+      query: queryTab.query,
+      chart: queryTab.$refs.chart.getChartStateForSave(),
+      name: newName || queryTab.initName
+    }
+
+    // Get queries from local storage
+    const myQueries = this.getStoredQueries()
+
+    // Set createdAt
+    if (newName) {
+      value.createdAt = new Date()
+    } else {
+      var queryIndex = myQueries.findIndex(oldQuery => oldQuery.id === queryTab.id)
+      value.createdAt = myQueries[queryIndex].createdAt
+    }
+
+    // Insert in queries list
+    if (newName) {
+      myQueries.push(value)
+    } else {
+      myQueries[queryIndex] = value
+    }
+
+    // Save to local storage
+    this.updateStorage(myQueries)
+    return value
+  },
+
+  updateStorage (value) {
+    localStorage.setItem('myQueries', JSON.stringify(value))
+  },
+
+  serialiseQueries (queryList) {
+    const preparedData = JSON.parse(JSON.stringify(queryList))
+    preparedData.forEach(query => delete query.isPredefined)
+    return JSON.stringify(preparedData, null, 4)
+  },
+
+  deserialiseQueries (str) {
+    let queryList = JSON.parse(str)
+    // Turn data into array if they are not
+    if (!Array.isArray(queryList)) {
+      queryList = [queryList]
+    }
+
+    // Generate new ids if they are the same as existing queries
+    queryList.forEach(query => {
+      const allQueriesIds = this.getStoredQueries().map(query => query.id)
+      if (allQueriesIds.includes(query.id)) {
+        query.id = nanoid()
+      }
+    })
+
+    return queryList
+  },
+
+  importQueries () {
+    return fu.importFile()
+      .then(data => {
+        return this.deserialiseQueries(data)
+      })
+  },
+
+  readPredefinedQueries () {
+    return fu.readFile('./queries.json')
+      .then(resp => {
+        return resp.json()
+      })
+  }
+}

src/lib/utils/fileIo.js

@@ -0,0 +1,71 @@
+export default {
+  exportToFile (str, fileName, type = 'octet/stream') {
+    // Create downloader
+    const downloader = document.createElement('a')
+    const blob = new Blob([str], { type })
+    const url = URL.createObjectURL(blob)
+    downloader.href = url
+    downloader.download = fileName
+
+    // Trigger click
+    downloader.click()
+
+    // Clean up
+    URL.revokeObjectURL(url)
+  },
+
+  /**
+   * Note: if user press Cancel in file choosing dialog
+   * it will be an unsettled promise. But it's grabbed by
+   * the garbage collector (tested with FinalizationRegistry).
+   */
+  getFileFromUser (type) {
+    return new Promise(resolve => {
+      const uploader = document.createElement('input')
+
+      uploader.type = 'file'
+      uploader.accept = type
+
+      uploader.addEventListener('change', () => {
+        const file = uploader.files[0]
+        resolve(file)
+      })
+
+      uploader.click()
+    })
+  },
+
+  importFile () {
+    const reader = new FileReader()
+
+    return this.getFileFromUser('.json')
+      .then(file => {
+        return new Promise((resolve, reject) => {
+          reader.onload = e => {
+            resolve(e.target.result)
+          }
+          reader.readAsText(file)
+        })
+      })
+  },
+
+  readFile (path) {
+    return fetch(path)
+  },
+
+  readAsArrayBuffer (file) {
+    const fileReader = new FileReader()
+
+    return new Promise((resolve, reject) => {
+      fileReader.onerror = () => {
+        fileReader.abort()
+        reject(new Error('Problem parsing input file.'))
+      }
+
+      fileReader.onload = () => {
+        resolve(fileReader.result)
+      }
+      fileReader.readAsArrayBuffer(file)
+    })
+  }
+}

src/lib/utils/time.js

@@ -0,0 +1,36 @@
+export default {
+  getPeriod (start, end) {
+    let diff = end.getTime() - start.getTime()
+    let result = ''
+
+    const days = Math.floor(diff / (1000 * 60 * 60 * 24))
+    diff -= days * (1000 * 60 * 60 * 24)
+    if (days) {
+      result += days + ' d '
+    }
+
+    const hours = Math.floor(diff / (1000 * 60 * 60))
+    diff -= hours * (1000 * 60 * 60)
+    if (hours) {
+      result += hours + ' h '
+    }
+
+    const mins = Math.floor(diff / (1000 * 60))
+    diff -= mins * (1000 * 60)
+    if (mins) {
+      result += mins + ' m '
+    }
+
+    const seconds = Math.floor(diff / (1000))
+    diff -= seconds * (1000)
+    if (seconds) {
+      result += seconds + ' s '
+    }
+
+    if (diff) {
+      result += diff + ' ms '
+    }
+
+    return result.replace(/\s$/, '')
+  }
+}