#116 JSON file import

src/components/CsvJsonImport/DelimiterSelector/ascii.js

@@ -0,0 +1,130 @@
+export default {
+  0: { name: 'null character' },
+  1: { name: 'start of header' },
+  2: { name: 'start of text' },
+  3: { name: 'end of text' },
+  4: { name: 'end of transmission' },
+  5: { name: 'enquiry' },
+  6: { name: 'acknowledge' },
+  7: { name: 'bell (ring)' },
+  8: { name: 'backspace' },
+  9: { name: 'horizontal tab' },
+  10: { name: 'line feed' },
+  11: { name: 'vertical tab' },
+  12: { name: 'form feed' },
+  13: { name: 'carriage return' },
+  14: { name: 'shift out' },
+  15: { name: 'shift in' },
+  16: { name: 'data link escape' },
+  17: { name: 'device control 1' },
+  18: { name: 'device control 2' },
+  19: { name: 'device control 3' },
+  20: { name: 'device control 4' },
+  21: { name: 'negative acknowledge' },
+  22: { name: 'synchronize' },
+  23: { name: 'end transmission block' },
+  24: { name: 'cancel' },
+  25: { name: 'end of medium' },
+  26: { name: 'substitute' },
+  27: { name: 'escape' },
+  28: { name: 'file separator' },
+  29: { name: 'group separator' },
+  30: { name: 'record separator' },
+  31: { name: 'unit separator' },
+  32: { name: 'space' },
+  33: { name: 'exclamation mark' },
+  34: { name: 'quotation mark' },
+  35: { name: 'number sign' },
+  36: { name: 'dollar sign' },
+  37: { name: 'percent sign' },
+  38: { name: 'ampersand' },
+  39: { name: 'apostrophe' },
+  40: { name: 'left parenthesis' },
+  41: { name: 'right parenthesis' },
+  42: { name: 'asterisk' },
+  43: { name: 'plus sign' },
+  44: { name: 'comma' },
+  45: { name: 'hyphen' },
+  46: { name: 'period' },
+  47: { name: 'slash' },
+  48: { name: 'digit 0' },
+  49: { name: 'digit 1' },
+  50: { name: 'digit 2' },
+  51: { name: 'digit 3' },
+  52: { name: 'digit 4' },
+  53: { name: 'digit 5' },
+  54: { name: 'digit 6' },
+  55: { name: 'digit 7' },
+  56: { name: 'digit 8' },
+  57: { name: 'digit 9' },
+  58: { name: 'colon' },
+  59: { name: 'semicolon' },
+  60: { name: 'less-than' },
+  61: { name: 'equals-to' },
+  62: { name: 'greater-than' },
+  63: { name: 'question mark' },
+  64: { name: 'at sign' },
+  65: { name: 'uppercase A' },
+  66: { name: 'uppercase B' },
+  67: { name: 'uppercase C' },
+  68: { name: 'uppercase D' },
+  69: { name: 'uppercase E' },
+  70: { name: 'uppercase F' },
+  71: { name: 'uppercase G' },
+  72: { name: 'uppercase H' },
+  73: { name: 'uppercase I' },
+  74: { name: 'uppercase J' },
+  75: { name: 'uppercase K' },
+  76: { name: 'uppercase L' },
+  77: { name: 'uppercase M' },
+  78: { name: 'uppercase N' },
+  79: { name: 'uppercase O' },
+  80: { name: 'uppercase P' },
+  81: { name: 'uppercase Q' },
+  82: { name: 'uppercase R' },
+  83: { name: 'uppercase S' },
+  84: { name: 'uppercase T' },
+  85: { name: 'uppercase U' },
+  86: { name: 'uppercase V' },
+  87: { name: 'uppercase W' },
+  88: { name: 'uppercase X' },
+  89: { name: 'uppercase Y' },
+  90: { name: 'uppercase Z' },
+  91: { name: 'left square bracket' },
+  92: { name: 'backslash' },
+  93: { name: 'right square bracket' },
+  94: { name: 'caret' },
+  95: { name: 'underscore' },
+  96: { name: 'grave accent' },
+  97: { name: 'lowercase a' },
+  98: { name: 'lowercase b' },
+  99: { name: 'lowercase c' },
+  100: { name: 'lowercase d' },
+  101: { name: 'lowercase e' },
+  102: { name: 'lowercase f' },
+  103: { name: 'lowercase g' },
+  104: { name: 'lowercase h' },
+  105: { name: 'lowercase i' },
+  106: { name: 'lowercase j' },
+  107: { name: 'lowercase k' },
+  108: { name: 'lowercase l' },
+  109: { name: 'lowercase m' },
+  110: { name: 'lowercase n' },
+  111: { name: 'lowercase o' },
+  112: { name: 'lowercase p' },
+  113: { name: 'lowercase q' },
+  114: { name: 'lowercase r' },
+  115: { name: 'lowercase s' },
+  116: { name: 'lowercase t' },
+  117: { name: 'lowercase u' },
+  118: { name: 'lowercase v' },
+  119: { name: 'lowercase w' },
+  120: { name: 'lowercase x' },
+  121: { name: 'lowercase y' },
+  122: { name: 'lowercase z' },
+  123: { name: 'left curly brace' },
+  124: { name: 'vertical bar' },
+  125: { name: 'right curly brace' },
+  126: { name: 'tilde' },
+  127: { name: 'delete (rubout)' }
+}

src/components/CsvJsonImport/DelimiterSelector/index.vue

@@ -0,0 +1,204 @@
+<template>
+  <div :class="{ 'disabled': disabled }">
+    <div class="text-field-label">Delimiter</div>
+    <div
+      class="delimiter-selector-container"
+      :style="{ width: width }"
+      @click="onContainerClick"
+    >
+      <div class="value">
+        <input
+          :class="{ 'filled': filled }"
+          ref="delimiterInput"
+          type="text"
+          maxlength="1"
+          v-model="inputValue"
+          @click.stop
+          :disabled="disabled"
+        />
+        <div class="name">{{ getSymbolName(value) }}</div>
+      </div>
+      <div class="controls" @click.stop>
+        <clear-icon @click.native="clear" :disabled="disabled"/>
+        <drop-down-chevron
+          :disabled="disabled"
+          @click.native="!disabled && (showOptions = !showOptions)"
+        />
+      </div>
+    </div>
+    <div v-show="showOptions" class="options" :style="{ width: width }">
+      <div
+        v-for="(option, index) in options"
+        :key="index"
+        @click="chooseOption(option)"
+        class="option"
+      >
+        <pre>{{option}}</pre><div>{{ getSymbolName(option) }}</div>
+      </div>
+    </div>
+  </div>
+</template>
+
+<script>
+import ascii from './ascii'
+import DropDownChevron from '@/components/svg/dropDownChevron'
+import ClearIcon from '@/components/svg/clear'
+
+export default {
+  name: 'DelimiterSelector',
+  props: ['value', 'width', 'disabled'],
+  components: { DropDownChevron, ClearIcon },
+  data () {
+    return {
+      showOptions: false,
+      options: [',', '\t', ' ', '|', ';', '\u001F', '\u001E'],
+      filled: false,
+      inputValue: ''
+    }
+  },
+  watch: {
+    inputValue () {
+      if (this.inputValue) {
+        this.filled = true
+        if (this.inputValue !== this.value) {
+          this.$emit('input', this.inputValue)
+        }
+      } else {
+        this.filled = false
+      }
+    }
+  },
+  created () {
+    this.inputValue = this.value
+  },
+  methods: {
+    getSymbolName (str) {
+      if (!str) {
+        return ''
+      }
+      return ascii[str.charCodeAt(0).toString()].name
+    },
+    chooseOption (option) {
+      this.inputValue = option
+      this.showOptions = false
+    },
+    onContainerClick (event) {
+      this.$refs.delimiterInput.focus()
+    },
+
+    clear () {
+      if (!this.disabled) {
+        this.inputValue = ''
+        this.$refs.delimiterInput.focus()
+      }
+    }
+  }
+}
+</script>
+
+<style scoped>
+.delimiter-selector-container {
+  background: var(--color-white);
+  border: 1px solid var(--color-border);
+  border-radius: var(--border-radius-medium-2);
+  height: 36px;
+  padding: 0 8px;
+  font-size: 12px;
+  box-sizing: border-box;
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+}
+
+.value {
+  display: flex;
+  align-items: center;
+}
+
+.value .name {
+  color: var(--color-text-light-2);
+  cursor: default;
+  margin-left: 4px;
+}
+
+.controls {
+  display: flex;
+  align-items: center;
+}
+
+.options {
+  background: var(--color-white);
+  border: 1px solid var(--color-border);
+  border-width: 0 1px 1px 1px;
+  color: var(--color-text-base);
+  border-radius: var(--border-radius-medium-2);
+  font-size: 12px;
+  box-sizing: border-box;
+  position: absolute;
+  z-index: 2;
+}
+
+.option {
+  display: flex;
+  align-items: center;
+  height: 24px;
+  padding: 0 6px;
+}
+
+.option:hover {
+  background-color: var(--color-bg-light);
+  color: var(--color-text-active);
+  cursor: pointer;
+}
+
+.option pre {
+  background-color: var(--color-bg-warning);
+  line-height: 20px;
+  margin-right: 6px;
+  tab-size: 1;
+  font-family: monospace;
+  width: 16px;
+  text-align: center;
+}
+
+input {
+  background: var(--color-white);
+  border: none;
+  color: var(--color-text-base);
+  height: 20px;
+  font-family: monospace;
+  font-size: 12px;
+  box-sizing: border-box;
+  width: 16px;
+  text-align: center;
+}
+
+input.filled {
+  background: var(--color-bg-warning);
+}
+
+input:focus {
+  outline: none;
+}
+
+input:disabled {
+  background: var(--color-bg-light);
+  color: var(--color-text-light-2);
+  cursor: default;
+}
+
+.text-field-label {
+  font-size: 12px;
+  color: var(--color-text-base);
+  padding-left: 8px;
+  margin-bottom: 2px;
+}
+
+.disabled .text-field-label {
+  color: var(--color-text-light-2);
+}
+
+.disabled .delimiter-selector-container {
+  background: var(--color-bg-light);
+}
+</style>

src/components/CsvJsonImport/index.vue

@@ -0,0 +1,501 @@
+<template>
+  <modal
+    :name="dialogName"
+    classes="dialog"
+    height="auto"
+    width="80%"
+    scrollable
+    :clickToClose="false"
+  >
+    <div class="dialog-header">
+      {{ typeName }} import
+      <close-icon @click="cancelImport" :disabled="disableDialog"/>
+    </div>
+    <div class="dialog-body">
+      <text-field
+        label="Table name"
+        v-model="tableName"
+        width="484px"
+        :disabled="disableDialog"
+        :error-msg="tableNameError"
+        id="csv-json-table-name"
+      />
+      <div v-if="!isJson && !isNdJson" class="chars">
+        <delimiter-selector
+          v-model="delimiter"
+          width="210px"
+          class="char-input"
+          :disabled="disableDialog"
+          @input="preview"
+        />
+        <text-field
+          label="Quote char"
+          hint="The character used to quote fields."
+          v-model="quoteChar"
+          width="93px"
+          :disabled="disableDialog"
+          class="char-input"
+          id="quote-char"
+          @input="preview"
+        />
+        <text-field
+          label="Escape char"
+          hint='
+            The character used to escape the quote character within a field
+            (e.g. "column with ""quotes"" in text").
+          '
+          max-hint-width="242px"
+          v-model="escapeChar"
+          width="93px"
+          :disabled="disableDialog"
+          class="char-input"
+          id="escape-char"
+          @input="preview"
+        />
+      </div>
+      <check-box
+        v-if="!isJson && !isNdJson"
+        :init="header"
+        label="Use first row as column headers"
+        :disabled="disableDialog"
+        @click="changeHeaderDisplaying"
+      />
+      <sql-table
+        v-if="previewData && previewData.rowCount > 0"
+        :data-set="previewData"
+        :preview="true"
+        class="preview-table"
+      />
+      <div v-else class="no-data">No data</div>
+      <logs
+        class="import-errors"
+        :messages="importMessages"
+      />
+    </div>
+    <div class="dialog-buttons-container">
+      <button
+        class="secondary"
+        :disabled="disableDialog"
+        @click="cancelImport"
+        id="import-cancel"
+      >
+        Cancel
+      </button>
+      <button
+        v-show="!importCompleted"
+        class="primary"
+        :disabled="disableDialog || disableImport"
+        @click="loadToDb(file)"
+        id="import-start"
+      >
+        Import
+      </button>
+      <button
+        v-show="importCompleted"
+        class="primary"
+        :disabled="disableDialog"
+        @click="finish"
+        id="import-finish"
+      >
+        Finish
+      </button>
+    </div>
+  </modal>
+</template>
+
+<script>
+import csv from '@/lib/csv'
+import CloseIcon from '@/components/svg/close'
+import TextField from '@/components/TextField'
+import DelimiterSelector from './DelimiterSelector'
+import CheckBox from '@/components/CheckBox'
+import SqlTable from '@/components/SqlTable'
+import Logs from '@/components/Logs'
+import time from '@/lib/utils/time'
+import fIo from '@/lib/utils/fileIo'
+import events from '@/lib/utils/events'
+
+export default {
+  name: 'CsvJsonImport',
+  components: {
+    CloseIcon,
+    TextField,
+    DelimiterSelector,
+    CheckBox,
+    SqlTable,
+    Logs
+  },
+  props: {
+    file: File,
+    db: Object,
+    dialogName: String
+  },
+  data () {
+    return {
+      disableDialog: false,
+      disableImport: false,
+      tableName: '',
+      delimiter: '',
+      quoteChar: '"',
+      escapeChar: '"',
+      header: true,
+      importCompleted: false,
+      importMessages: [],
+      previewData: null,
+      addedTable: null,
+      tableNameError: ''
+    }
+  },
+  computed: {
+    isJson () {
+      return fIo.isJSON(this.file)
+    },
+    isNdJson () {
+      return fIo.isNDJSON(this.file)
+    },
+    typeName () {
+      return this.isJson || this.isNdJson ? 'JSON' : 'CSV'
+    }
+  },
+  watch: {
+    isJson () {
+      if (this.isJson) {
+        this.delimiter = '\u001E'
+        this.header = false
+      }
+    },
+    isNdJson () {
+      if (this.isNdJson) {
+        this.delimiter = '\u001E'
+        this.header = false
+      }
+    },
+    tableName: time.debounce(function () {
+      this.tableNameError = ''
+      if (!this.tableName) {
+        return
+      }
+      this.db.validateTableName(this.tableName)
+        .catch(err => {
+          this.tableNameError = err.message + '. Try another table name.'
+        })
+    }, 400)
+  },
+  methods: {
+    changeHeaderDisplaying (e) {
+      this.header = e
+      this.preview()
+    },
+    cancelImport () {
+      if (!this.disableDialog) {
+        if (this.addedTable) {
+          this.db.execute(`DROP TABLE "${this.addedTable}"`)
+          this.db.refreshSchema()
+        }
+        this.$modal.hide(this.dialogName)
+        this.$emit('cancel')
+      }
+    },
+    reset () {
+      this.header = !this.isJson && !this.isNdJson
+      this.quoteChar = '"'
+      this.escapeChar = '"'
+      this.delimiter = !this.isJson && !this.isNdJson ? '' : '\u001E'
+      this.tableName = ''
+      this.disableDialog = false
+      this.disableImport = false
+      this.importCompleted = false
+      this.importMessages = []
+      this.previewData = null
+      this.addedTable = null
+      this.tableNameError = ''
+    },
+    open () {
+      this.tableName = this.db.sanitizeTableName(fIo.getFileName(this.file))
+      this.$modal.show(this.dialogName)
+    },
+    async preview () {
+      this.disableImport = false
+      if (!this.file) {
+        return
+      }
+      this.importCompleted = false
+      const config = {
+        preview: 3,
+        quoteChar: this.quoteChar || '"',
+        escapeChar: this.escapeChar,
+        header: this.header,
+        delimiter: this.delimiter,
+        columns: !this.isJson && !this.isNdJson ? null : ['doc']
+      }
+      try {
+        const start = new Date()
+        const parseResult = this.isJson
+          ? await this.getJsonParseResult(this.file)
+          : await csv.parse(this.file, config)
+        const end = new Date()
+        this.previewData = parseResult.data
+        this.previewData.rowCount = parseResult.rowCount
+        this.delimiter = parseResult.delimiter
+
+        // In parseResult.messages we can get parse errors
+        this.importMessages = parseResult.messages || []
+
+        if (this.previewData.rowCount === 0) {
+          this.disableImport = true
+          this.importMessages.push({
+            type: 'info',
+            message: 'No rows to import.'
+          })
+        }
+
+        if (!parseResult.hasErrors) {
+          this.importMessages.push({
+            message: `Preview parsing is completed in ${time.getPeriod(start, end)}.`,
+            type: 'success'
+          })
+        }
+      } catch (err) {
+        console.error(err)
+        this.importMessages = [{
+          message: err,
+          type: 'error'
+        }]
+      }
+    },
+    async getJsonParseResult (file) {
+      const jsonContent = await fIo.getFileContent(file)
+      const isEmpty = !jsonContent.trim()
+      return {
+        data: {
+          columns: ['doc'],
+          values: { doc: !isEmpty ? [jsonContent] : [] }
+        },
+        hasErrors: false,
+        messages: [],
+        rowCount: +(!isEmpty)
+      }
+    },
+    async loadToDb (file) {
+      if (!this.tableName) {
+        this.tableNameError = "Table name can't be empty"
+        return
+      }
+
+      this.disableDialog = true
+      const config = {
+        quoteChar: this.quoteChar || '"',
+        escapeChar: this.escapeChar,
+        header: this.header,
+        delimiter: this.delimiter,
+        columns: !this.isJson && !this.isNdJson ? null : ['doc']
+      }
+      const parsingMsg = {
+        message: `Parsing ${this.typeName}...`,
+        type: 'info'
+      }
+      this.importMessages.push(parsingMsg)
+      const parsingLoadingIndicator = setTimeout(() => { parsingMsg.type = 'loading' }, 1000)
+
+      const importMsg = {
+        message: `Importing ${this.typeName} into a SQLite database...`,
+        type: 'info'
+      }
+      let importLoadingIndicator = null
+
+      const updateProgress = progress => {
+        this.$set(importMsg, 'progress', progress)
+      }
+      const progressCounterId = this.db.createProgressCounter(updateProgress)
+
+      try {
+        let start = new Date()
+        const parseResult = this.isJson
+          ? await this.getJsonParseResult(file)
+          : await csv.parse(this.file, config)
+
+        let end = new Date()
+
+        if (!parseResult.hasErrors) {
+          const rowCount = parseResult.rowCount
+          let period = time.getPeriod(start, end)
+          parsingMsg.type = 'success'
+
+          if (parseResult.messages.length > 0) {
+            this.importMessages = this.importMessages.concat(parseResult.messages)
+            parsingMsg.message = `${rowCount} rows are parsed in ${period}.`
+          } else {
+            // Inform about parsing success
+            parsingMsg.message = `${rowCount} rows are parsed successfully in ${period}.`
+          }
+
+          // Loading indicator for parsing is not needed anymore
+          clearTimeout(parsingLoadingIndicator)
+
+          // Add info about import start
+          this.importMessages.push(importMsg)
+
+          // Show import progress after 1 second
+          importLoadingIndicator = setTimeout(() => {
+            importMsg.type = 'loading'
+          }, 1000)
+
+          // Add table
+          start = new Date()
+          await this.db.addTableFromCsv(this.tableName, parseResult.data, progressCounterId)
+          end = new Date()
+
+          this.addedTable = this.tableName
+          // Inform about import success
+          period = time.getPeriod(start, end)
+          importMsg.message = `Importing ${this.typeName} into a SQLite database is completed in ${period}.`
+          importMsg.type = 'success'
+
+          // Loading indicator for import is not needed anymore
+          clearTimeout(importLoadingIndicator)
+
+          this.importCompleted = true
+        } else {
+          parsingMsg.message = 'Parsing ended with errors.'
+          parsingMsg.type = 'info'
+          this.importMessages = this.importMessages.concat(parseResult.messages)
+        }
+      } catch (err) {
+        console.error(err)
+        if (parsingMsg.type === 'loading') {
+          parsingMsg.type = 'info'
+        }
+
+        if (importMsg.type === 'loading') {
+          importMsg.type = 'info'
+        }
+
+        this.importMessages.push({
+          message: err,
+          type: 'error'
+        })
+      }
+
+      clearTimeout(parsingLoadingIndicator)
+      clearTimeout(importLoadingIndicator)
+      this.db.deleteProgressCounter(progressCounterId)
+      this.disableDialog = false
+    },
+    async finish () {
+      this.$modal.hide(this.dialogName)
+      const stmt = this.getQueryExample()
+      const tabId = await this.$store.dispatch('addTab', { query: stmt })
+      this.$store.commit('setCurrentTabId', tabId)
+      this.importCompleted = false
+      this.$emit('finish')
+      events.send('inquiry.create', null, { auto: true })
+    },
+    getQueryExample () {
+      return this.isNdJson ? this.getNdJsonQueryExample()
+        : this.isJson ? this.getJsonQueryExample()
+          : [
+            '/*',
+        ` * Your CSV file has been imported into ${this.addedTable} table.`,
+        ' * You can run this SQL query to make all CSV records available for charting.',
+        ' */',
+        `SELECT * FROM "${this.addedTable}"`
+          ].join('\n')
+    },
+    getNdJsonQueryExample () {
+      try {
+        const firstRowJson = JSON.parse(this.previewData.values.doc[0])
+        const firstKey = Object.keys(firstRowJson)[0]
+        return [
+          '/*',
+          ` * Your NDJSON file has been imported into ${this.addedTable} table.`,
+          ` * Run this SQL query to get values of property ${firstKey} and make them available for charting.`,
+          ' */',
+          `SELECT doc->>'${firstKey}'`,
+          `FROM "${this.addedTable}"`
+        ].join('\n')
+      } catch (err) {
+        console.error(err)
+        return [
+          '/*',
+          ` * Your NDJSON file has been imported into ${this.addedTable} table.`,
+          ' */',
+          'SELECT *',
+          `FROM "${this.addedTable}"`
+        ].join('\n')
+      }
+    },
+    getJsonQueryExample () {
+      try {
+        const firstRowJson = JSON.parse(this.previewData.values.doc[0])
+        const firstKey = Object.keys(firstRowJson)[0]
+        return [
+          '/*',
+          ` * Your JSON file has been imported into ${this.addedTable} table.`,
+          ` * Run this SQL query to get values of property ${firstKey} and make them available for charting.`,
+          ' */',
+          'SELECT *',
+          `FROM "${this.addedTable}"`,
+          `JOIN json_each(doc, '$.${firstKey}')`
+        ].join('\n')
+      } catch (err) {
+        console.error(err)
+        return [
+          '/*',
+          ` * Your NDJSON file has been imported into ${this.addedTable} table.`,
+          ' */',
+          'SELECT *',
+          `FROM "${this.addedTable}"`
+        ].join('\n')
+      }
+    }
+  }
+}
+</script>
+
+<style scoped>
+.dialog-body {
+  padding-bottom: 0;
+}
+
+#csv-json-table-name {
+margin-bottom: 24px;
+}
+
+.chars {
+  display: flex;
+  align-items: flex-end;
+  margin: 0 0 20px;
+}
+.char-input {
+  margin-right: 44px;
+}
+.preview-table {
+  margin-top: 18px;
+}
+
+.import-errors {
+  height: 136px;
+  margin-top: 8px;
+}
+.no-data {
+  margin-top: 32px;
+  background-color: white;
+  border-radius: 5px;
+  position: relative;
+  border: 1px solid var(--color-border-light);
+  box-sizing: border-box;
+  height: 147px;
+  font-size: 13px;
+  color: var(--color-text-base);
+  display: flex;
+  justify-content: center;
+  align-items: center;
+}
+
+/* https://github.com/euvl/vue-js-modal/issues/623 */
+>>> .vm--modal {
+  max-width: 1152px;
+  margin: auto;
+  left: 0 !important;
+}
+</style>