-
@@ -115,7 +116,7 @@ import fIo from '@/lib/utils/fileIo'
import events from '@/lib/utils/events'
export default {
- name: 'CsvImport',
+ name: 'CsvJsonImport',
components: {
CloseIcon,
TextField,
@@ -124,33 +125,50 @@ export default {
SqlTable,
Logs
},
- props: ['file', 'db', 'dialogName'],
+ props: {
+ file: File,
+ db: Object,
+ dialogName: String
+ },
data () {
return {
disableDialog: false,
+ disableImport: false,
tableName: '',
delimiter: '',
quoteChar: '"',
escapeChar: '"',
header: true,
- importCsvCompleted: false,
- importCsvMessages: [],
+ importCompleted: false,
+ importMessages: [],
previewData: null,
addedTable: null,
tableNameError: ''
}
},
+ computed: {
+ isJson () {
+ return fIo.isJSON(this.file)
+ },
+ isNdJson () {
+ return fIo.isNDJSON(this.file)
+ },
+ typeName () {
+ return this.isJson || this.isNdJson ? 'JSON' : 'CSV'
+ }
+ },
watch: {
- quoteChar () {
- this.previewCsv()
+ isJson () {
+ if (this.isJson) {
+ this.delimiter = '\u001E'
+ this.header = false
+ }
},
-
- escapeChar () {
- this.previewCsv()
- },
-
- header () {
- this.previewCsv()
+ isNdJson () {
+ if (this.isNdJson) {
+ this.delimiter = '\u001E'
+ this.header = false
+ }
},
tableName: time.debounce(function () {
this.tableNameError = ''
@@ -164,7 +182,11 @@ export default {
}, 400)
},
methods: {
- cancelCsvImport () {
+ changeHeaderDisplaying (e) {
+ this.header = e
+ this.preview()
+ },
+ cancelImport () {
if (!this.disableDialog) {
if (this.addedTable) {
this.db.execute(`DROP TABLE "${this.addedTable}"`)
@@ -175,14 +197,15 @@ export default {
}
},
reset () {
- this.header = true
+ this.header = !this.isJson && !this.isNdJson
this.quoteChar = '"'
this.escapeChar = '"'
- this.delimiter = ''
+ this.delimiter = !this.isJson && !this.isNdJson ? '' : '\u001E'
this.tableName = ''
this.disableDialog = false
- this.importCsvCompleted = false
- this.importCsvMessages = []
+ this.disableImport = false
+ this.importCompleted = false
+ this.importMessages = []
this.previewData = null
this.addedTable = null
this.tableNameError = ''
@@ -191,39 +214,69 @@ export default {
this.tableName = this.db.sanitizeTableName(fIo.getFileName(this.file))
this.$modal.show(this.dialogName)
},
- async previewCsv () {
- this.importCsvCompleted = false
+ async preview () {
+ this.disableImport = false
+ if (!this.file) {
+ return
+ }
+ this.importCompleted = false
const config = {
preview: 3,
quoteChar: this.quoteChar || '"',
escapeChar: this.escapeChar,
header: this.header,
- delimiter: this.delimiter
+ delimiter: this.delimiter,
+ columns: !this.isJson && !this.isNdJson ? null : ['doc']
}
try {
const start = new Date()
- const parseResult = await csv.parse(this.file, config)
+ const parseResult = this.isJson
+ ? await this.getJsonParseResult(this.file)
+ : await csv.parse(this.file, config)
const end = new Date()
this.previewData = parseResult.data
+ this.previewData.rowCount = parseResult.rowCount
this.delimiter = parseResult.delimiter
// In parseResult.messages we can get parse errors
- this.importCsvMessages = parseResult.messages || []
+ this.importMessages = parseResult.messages || []
+
+ if (this.previewData.rowCount === 0) {
+ this.disableImport = true
+ this.importMessages.push({
+ type: 'info',
+ message: 'No rows to import.'
+ })
+ }
if (!parseResult.hasErrors) {
- this.importCsvMessages.push({
+ this.importMessages.push({
message: `Preview parsing is completed in ${time.getPeriod(start, end)}.`,
type: 'success'
})
}
} catch (err) {
- this.importCsvMessages = [{
+ console.error(err)
+ this.importMessages = [{
message: err,
type: 'error'
}]
}
},
- async loadFromCsv (file) {
+ async getJsonParseResult (file) {
+ const jsonContent = await fIo.getFileContent(file)
+ const isEmpty = !jsonContent.trim()
+ return {
+ data: {
+ columns: ['doc'],
+ values: { doc: !isEmpty ? [jsonContent] : [] }
+ },
+ hasErrors: false,
+ messages: [],
+ rowCount: +(!isEmpty)
+ }
+ },
+ async loadToDb (file) {
if (!this.tableName) {
this.tableNameError = "Table name can't be empty"
return
@@ -234,17 +287,18 @@ export default {
quoteChar: this.quoteChar || '"',
escapeChar: this.escapeChar,
header: this.header,
- delimiter: this.delimiter
+ delimiter: this.delimiter,
+ columns: !this.isJson && !this.isNdJson ? null : ['doc']
}
- const parseCsvMsg = {
- message: 'Parsing CSV...',
+ const parsingMsg = {
+ message: `Parsing ${this.typeName}...`,
type: 'info'
}
- this.importCsvMessages.push(parseCsvMsg)
- const parseCsvLoadingIndicator = setTimeout(() => { parseCsvMsg.type = 'loading' }, 1000)
+ this.importMessages.push(parsingMsg)
+ const parsingLoadingIndicator = setTimeout(() => { parsingMsg.type = 'loading' }, 1000)
const importMsg = {
- message: 'Importing CSV into a SQLite database...',
+ message: `Importing ${this.typeName} into a SQLite database...`,
type: 'info'
}
let importLoadingIndicator = null
@@ -256,27 +310,30 @@ export default {
try {
let start = new Date()
- const parseResult = await csv.parse(this.file, config)
+ const parseResult = this.isJson
+ ? await this.getJsonParseResult(file)
+ : await csv.parse(this.file, config)
+
let end = new Date()
if (!parseResult.hasErrors) {
const rowCount = parseResult.rowCount
let period = time.getPeriod(start, end)
- parseCsvMsg.type = 'success'
+ parsingMsg.type = 'success'
if (parseResult.messages.length > 0) {
- this.importCsvMessages = this.importCsvMessages.concat(parseResult.messages)
- parseCsvMsg.message = `${rowCount} rows are parsed in ${period}.`
+ this.importMessages = this.importMessages.concat(parseResult.messages)
+ parsingMsg.message = `${rowCount} rows are parsed in ${period}.`
} else {
- // Inform about csv parsing success
- parseCsvMsg.message = `${rowCount} rows are parsed successfully in ${period}.`
+ // Inform about parsing success
+ parsingMsg.message = `${rowCount} rows are parsed successfully in ${period}.`
}
- // Loading indicator for csv parsing is not needed anymore
- clearTimeout(parseCsvLoadingIndicator)
+ // Loading indicator for parsing is not needed anymore
+ clearTimeout(parsingLoadingIndicator)
// Add info about import start
- this.importCsvMessages.push(importMsg)
+ this.importMessages.push(importMsg)
// Show import progress after 1 second
importLoadingIndicator = setTimeout(() => {
@@ -291,52 +348,105 @@ export default {
this.addedTable = this.tableName
// Inform about import success
period = time.getPeriod(start, end)
- importMsg.message = `Importing CSV into a SQLite database is completed in ${period}.`
+ importMsg.message = `Importing ${this.typeName} into a SQLite database is completed in ${period}.`
importMsg.type = 'success'
// Loading indicator for import is not needed anymore
clearTimeout(importLoadingIndicator)
- this.importCsvCompleted = true
+ this.importCompleted = true
} else {
- parseCsvMsg.message = 'Parsing ended with errors.'
- parseCsvMsg.type = 'info'
- this.importCsvMessages = this.importCsvMessages.concat(parseResult.messages)
+ parsingMsg.message = 'Parsing ended with errors.'
+ parsingMsg.type = 'info'
+ this.importMessages = this.importMessages.concat(parseResult.messages)
}
} catch (err) {
- if (parseCsvMsg.type === 'loading') {
- parseCsvMsg.type = 'info'
+ console.error(err)
+ if (parsingMsg.type === 'loading') {
+ parsingMsg.type = 'info'
}
if (importMsg.type === 'loading') {
importMsg.type = 'info'
}
- this.importCsvMessages.push({
+ this.importMessages.push({
message: err,
type: 'error'
})
}
- clearTimeout(parseCsvLoadingIndicator)
+ clearTimeout(parsingLoadingIndicator)
clearTimeout(importLoadingIndicator)
this.db.deleteProgressCounter(progressCounterId)
this.disableDialog = false
},
async finish () {
this.$modal.hide(this.dialogName)
- const stmt = [
- '/*',
+ const stmt = this.getQueryExample()
+ const tabId = await this.$store.dispatch('addTab', { query: stmt })
+ this.$store.commit('setCurrentTabId', tabId)
+ this.importCompleted = false
+ this.$emit('finish')
+ events.send('inquiry.create', null, { auto: true })
+ },
+ getQueryExample () {
+ return this.isNdJson ? this.getNdJsonQueryExample()
+ : this.isJson ? this.getJsonQueryExample()
+ : [
+ '/*',
` * Your CSV file has been imported into ${this.addedTable} table.`,
' * You can run this SQL query to make all CSV records available for charting.',
' */',
`SELECT * FROM "${this.addedTable}"`
- ].join('\n')
- const tabId = await this.$store.dispatch('addTab', { query: stmt })
- this.$store.commit('setCurrentTabId', tabId)
- this.importCsvCompleted = false
- this.$emit('finish')
- events.send('inquiry.create', null, { auto: true })
+ ].join('\n')
+ },
+ getNdJsonQueryExample () {
+ try {
+ const firstRowJson = JSON.parse(this.previewData.values.doc[0])
+ const firstKey = Object.keys(firstRowJson)[0]
+ return [
+ '/*',
+ ` * Your NDJSON file has been imported into ${this.addedTable} table.`,
+ ` * Run this SQL query to get values of property ${firstKey} and make them available for charting.`,
+ ' */',
+ `SELECT doc->>'${firstKey}'`,
+ `FROM "${this.addedTable}"`
+ ].join('\n')
+ } catch (err) {
+ console.error(err)
+ return [
+ '/*',
+ ` * Your NDJSON file has been imported into ${this.addedTable} table.`,
+ ' */',
+ 'SELECT *',
+ `FROM "${this.addedTable}"`
+ ].join('\n')
+ }
+ },
+ getJsonQueryExample () {
+ try {
+ const firstRowJson = JSON.parse(this.previewData.values.doc[0])
+ const firstKey = Object.keys(firstRowJson)[0]
+ return [
+ '/*',
+ ` * Your JSON file has been imported into ${this.addedTable} table.`,
+ ` * Run this SQL query to get values of property ${firstKey} and make them available for charting.`,
+ ' */',
+ 'SELECT *',
+ `FROM "${this.addedTable}"`,
+ `JOIN json_each(doc, '$.${firstKey}')`
+ ].join('\n')
+ } catch (err) {
+ console.error(err)
+ return [
+ '/*',
+ ` * Your NDJSON file has been imported into ${this.addedTable} table.`,
+ ' */',
+ 'SELECT *',
+ `FROM "${this.addedTable}"`
+ ].join('\n')
+ }
}
}
}
@@ -347,10 +457,14 @@ export default {
padding-bottom: 0;
}
+#csv-json-table-name {
+margin-bottom: 24px;
+}
+
.chars {
display: flex;
align-items: flex-end;
- margin: 24px 0 20px;
+ margin: 0 0 20px;
}
.char-input {
margin-right: 44px;
@@ -359,7 +473,7 @@ export default {
margin-top: 18px;
}
-.import-csv-errors {
+.import-errors {
height: 136px;
margin-top: 8px;
}
diff --git a/src/components/DbUploader.vue b/src/components/DbUploader.vue
index ade029d..7694f11 100644
--- a/src/components/DbUploader.vue
+++ b/src/components/DbUploader.vue
@@ -41,13 +41,13 @@
-
-
+
@@ -57,7 +57,7 @@
import fIo from '@/lib/utils/fileIo'
import ChangeDbIcon from '@/components/svg/changeDb'
import database from '@/lib/database'
-import CsvImport from '@/components/CsvImport'
+import CsvJsonImport from '@/components/CsvJsonImport'
import events from '@/lib/utils/events'
export default {
@@ -79,7 +79,7 @@ export default {
},
components: {
ChangeDbIcon,
- CsvImport
+ CsvJsonImport
},
data () {
return {
@@ -102,7 +102,7 @@ export default {
}
},
methods: {
- cancelCsvImport () {
+ cancelImport () {
if (this.newDb) {
this.newDb.shutDown()
this.newDb = null
@@ -128,21 +128,22 @@ export default {
if (fIo.isDatabase(file)) {
this.loadDb(file)
} else {
+ const isJson = fIo.isJSON(file) || fIo.isNDJSON(file)
events.send('database.import', file.size, {
- from: 'csv',
+ from: isJson ? 'json' : 'csv',
new_db: true
})
this.file = file
await this.$nextTick()
- const csvImport = this.$refs.addCsv
- csvImport.reset()
- return Promise.all([csvImport.previewCsv(), this.animationPromise])
- .then(csvImport.open)
+ const csvJsonImportModal = this.$refs.addCsvJson
+ csvJsonImportModal.reset()
+ return Promise.all([csvJsonImportModal.preview(), this.animationPromise])
+ .then(csvJsonImportModal.open)
}
},
browse () {
- fIo.getFileFromUser('.db,.sqlite,.sqlite3,.csv')
+ fIo.getFileFromUser('.db,.sqlite,.sqlite3,.csv,.json,.ndjson')
.then(this.checkFile)
},
diff --git a/src/lib/csv.js b/src/lib/csv.js
index e2dc4b4..1ab0871 100644
--- a/src/lib/csv.js
+++ b/src/lib/csv.js
@@ -7,9 +7,9 @@ const hintsByCode = {
}
export default {
- getResult (source) {
+ getResult (source, columns) {
const result = {
- columns: []
+ columns: columns || []
}
const values = {}
if (source.meta.fields) {
@@ -24,8 +24,18 @@ export default {
return value
})
})
+ } else if (columns) {
+ columns.forEach((col, i) => {
+ values[col] = source.data.map(row => {
+ let value = row[i]
+ if (value instanceof Date) {
+ value = value.toISOString()
+ }
+ return value
+ })
+ })
} else {
- for (let i = 0; i <= source.data[0].length - 1; i++) {
+ for (let i = 0; source.data[0] && i <= source.data[0].length - 1; i++) {
const colName = `col${i + 1}`
result.columns.push(colName)
values[colName] = source.data.map(row => {
@@ -76,7 +86,7 @@ export default {
let res
try {
res = {
- data: this.getResult(results),
+ data: this.getResult(results, config.columns),
delimiter: results.meta.delimiter,
hasErrors: false,
rowCount: results.data.length
diff --git a/src/lib/utils/fileIo.js b/src/lib/utils/fileIo.js
index d4ba38d..d537124 100644
--- a/src/lib/utils/fileIo.js
+++ b/src/lib/utils/fileIo.js
@@ -1,4 +1,10 @@
export default {
+ isJSON (file) {
+ return file && file.type === 'application/json'
+ },
+ isNDJSON (file) {
+ return file && file.name.endsWith('.ndjson')
+ },
isDatabase (file) {
const dbTypes = ['application/vnd.sqlite3', 'application/x-sqlite3']
return file.type
@@ -51,19 +57,20 @@ export default {
},
importFile () {
- const reader = new FileReader()
-
return this.getFileFromUser('.json')
.then(file => {
- return new Promise((resolve, reject) => {
- reader.onload = e => {
- resolve(e.target.result)
- }
- reader.readAsText(file)
- })
+ return this.getFileContent(file)
})
},
+ getFileContent (file) {
+ const reader = new FileReader()
+ return new Promise(resolve => {
+ reader.onload = e => resolve(e.target.result)
+ reader.readAsText(file)
+ })
+ },
+
readFile (path) {
return fetch(path)
},
diff --git a/src/views/Main/Workspace/Schema/index.vue b/src/views/Main/Workspace/Schema/index.vue
index 4133ef9..c41ffb0 100644
--- a/src/views/Main/Workspace/Schema/index.vue
+++ b/src/views/Main/Workspace/Schema/index.vue
@@ -10,7 +10,7 @@