From 1b6b7c71e97e73f728ebfbf5c66fd393c2a67eff Mon Sep 17 00:00:00 2001 From: lana-k Date: Thu, 5 Sep 2024 22:15:12 +0200 Subject: [PATCH] #116 JSON file import --- package.json | 2 +- .../DelimiterSelector/ascii.js | 0 .../DelimiterSelector/index.vue | 0 .../{CsvImport => CsvJsonImport}/index.vue | 276 +++++++++++++----- src/components/DbUploader.vue | 29 +- src/lib/csv.js | 18 +- src/lib/utils/fileIo.js | 23 +- src/views/Main/Workspace/Schema/index.vue | 29 +- src/views/Main/index.vue | 2 +- tests/components/CsvImport/CsvImport.spec.js | 148 +++++----- .../CsvImport/DelimiterSelector.spec.js | 2 +- tests/components/DbUploader.spec.js | 10 +- .../Main/Workspace/Schema/Schema.spec.js | 20 +- 13 files changed, 346 insertions(+), 213 deletions(-) rename src/components/{CsvImport => CsvJsonImport}/DelimiterSelector/ascii.js (100%) rename src/components/{CsvImport => CsvJsonImport}/DelimiterSelector/index.vue (100%) rename src/components/{CsvImport => CsvJsonImport}/index.vue (53%) diff --git a/package.json b/package.json index 1541f16..5068a09 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sqliteviz", - "version": "0.24.1", + "version": "0.25.0", "license": "Apache-2.0", "private": true, "scripts": { diff --git a/src/components/CsvImport/DelimiterSelector/ascii.js b/src/components/CsvJsonImport/DelimiterSelector/ascii.js similarity index 100% rename from src/components/CsvImport/DelimiterSelector/ascii.js rename to src/components/CsvJsonImport/DelimiterSelector/ascii.js diff --git a/src/components/CsvImport/DelimiterSelector/index.vue b/src/components/CsvJsonImport/DelimiterSelector/index.vue similarity index 100% rename from src/components/CsvImport/DelimiterSelector/index.vue rename to src/components/CsvJsonImport/DelimiterSelector/index.vue diff --git a/src/components/CsvImport/index.vue b/src/components/CsvJsonImport/index.vue similarity index 53% rename from src/components/CsvImport/index.vue rename to src/components/CsvJsonImport/index.vue index a984629..b542d45 100644 --- a/src/components/CsvImport/index.vue +++ b/src/components/CsvJsonImport/index.vue @@ -8,8 +8,8 @@ :clickToClose="false" >
- CSV import - + {{ typeName }} import +
-
+
No data
@@ -115,7 +116,7 @@ import fIo from '@/lib/utils/fileIo' import events from '@/lib/utils/events' export default { - name: 'CsvImport', + name: 'CsvJsonImport', components: { CloseIcon, TextField, @@ -124,33 +125,50 @@ export default { SqlTable, Logs }, - props: ['file', 'db', 'dialogName'], + props: { + file: File, + db: Object, + dialogName: String + }, data () { return { disableDialog: false, + disableImport: false, tableName: '', delimiter: '', quoteChar: '"', escapeChar: '"', header: true, - importCsvCompleted: false, - importCsvMessages: [], + importCompleted: false, + importMessages: [], previewData: null, addedTable: null, tableNameError: '' } }, + computed: { + isJson () { + return fIo.isJSON(this.file) + }, + isNdJson () { + return fIo.isNDJSON(this.file) + }, + typeName () { + return this.isJson || this.isNdJson ? 'JSON' : 'CSV' + } + }, watch: { - quoteChar () { - this.previewCsv() + isJson () { + if (this.isJson) { + this.delimiter = '\u001E' + this.header = false + } }, - - escapeChar () { - this.previewCsv() - }, - - header () { - this.previewCsv() + isNdJson () { + if (this.isNdJson) { + this.delimiter = '\u001E' + this.header = false + } }, tableName: time.debounce(function () { this.tableNameError = '' @@ -164,7 +182,11 @@ export default { }, 400) }, methods: { - cancelCsvImport () { + changeHeaderDisplaying (e) { + this.header = e + this.preview() + }, + cancelImport () { if (!this.disableDialog) { if (this.addedTable) { this.db.execute(`DROP TABLE "${this.addedTable}"`) @@ -175,14 +197,15 @@ export default { } }, reset () { - this.header = true + this.header = !this.isJson && !this.isNdJson this.quoteChar = '"' this.escapeChar = '"' - this.delimiter = '' + this.delimiter = !this.isJson && !this.isNdJson ? '' : '\u001E' this.tableName = '' this.disableDialog = false - this.importCsvCompleted = false - this.importCsvMessages = [] + this.disableImport = false + this.importCompleted = false + this.importMessages = [] this.previewData = null this.addedTable = null this.tableNameError = '' @@ -191,39 +214,69 @@ export default { this.tableName = this.db.sanitizeTableName(fIo.getFileName(this.file)) this.$modal.show(this.dialogName) }, - async previewCsv () { - this.importCsvCompleted = false + async preview () { + this.disableImport = false + if (!this.file) { + return + } + this.importCompleted = false const config = { preview: 3, quoteChar: this.quoteChar || '"', escapeChar: this.escapeChar, header: this.header, - delimiter: this.delimiter + delimiter: this.delimiter, + columns: !this.isJson && !this.isNdJson ? null : ['doc'] } try { const start = new Date() - const parseResult = await csv.parse(this.file, config) + const parseResult = this.isJson + ? await this.getJsonParseResult(this.file) + : await csv.parse(this.file, config) const end = new Date() this.previewData = parseResult.data + this.previewData.rowCount = parseResult.rowCount this.delimiter = parseResult.delimiter // In parseResult.messages we can get parse errors - this.importCsvMessages = parseResult.messages || [] + this.importMessages = parseResult.messages || [] + + if (this.previewData.rowCount === 0) { + this.disableImport = true + this.importMessages.push({ + type: 'info', + message: 'No rows to import.' + }) + } if (!parseResult.hasErrors) { - this.importCsvMessages.push({ + this.importMessages.push({ message: `Preview parsing is completed in ${time.getPeriod(start, end)}.`, type: 'success' }) } } catch (err) { - this.importCsvMessages = [{ + console.error(err) + this.importMessages = [{ message: err, type: 'error' }] } }, - async loadFromCsv (file) { + async getJsonParseResult (file) { + const jsonContent = await fIo.getFileContent(file) + const isEmpty = !jsonContent.trim() + return { + data: { + columns: ['doc'], + values: { doc: !isEmpty ? [jsonContent] : [] } + }, + hasErrors: false, + messages: [], + rowCount: +(!isEmpty) + } + }, + async loadToDb (file) { if (!this.tableName) { this.tableNameError = "Table name can't be empty" return @@ -234,17 +287,18 @@ export default { quoteChar: this.quoteChar || '"', escapeChar: this.escapeChar, header: this.header, - delimiter: this.delimiter + delimiter: this.delimiter, + columns: !this.isJson && !this.isNdJson ? null : ['doc'] } - const parseCsvMsg = { - message: 'Parsing CSV...', + const parsingMsg = { + message: `Parsing ${this.typeName}...`, type: 'info' } - this.importCsvMessages.push(parseCsvMsg) - const parseCsvLoadingIndicator = setTimeout(() => { parseCsvMsg.type = 'loading' }, 1000) + this.importMessages.push(parsingMsg) + const parsingLoadingIndicator = setTimeout(() => { parsingMsg.type = 'loading' }, 1000) const importMsg = { - message: 'Importing CSV into a SQLite database...', + message: `Importing ${this.typeName} into a SQLite database...`, type: 'info' } let importLoadingIndicator = null @@ -256,27 +310,30 @@ export default { try { let start = new Date() - const parseResult = await csv.parse(this.file, config) + const parseResult = this.isJson + ? await this.getJsonParseResult(file) + : await csv.parse(this.file, config) + let end = new Date() if (!parseResult.hasErrors) { const rowCount = parseResult.rowCount let period = time.getPeriod(start, end) - parseCsvMsg.type = 'success' + parsingMsg.type = 'success' if (parseResult.messages.length > 0) { - this.importCsvMessages = this.importCsvMessages.concat(parseResult.messages) - parseCsvMsg.message = `${rowCount} rows are parsed in ${period}.` + this.importMessages = this.importMessages.concat(parseResult.messages) + parsingMsg.message = `${rowCount} rows are parsed in ${period}.` } else { - // Inform about csv parsing success - parseCsvMsg.message = `${rowCount} rows are parsed successfully in ${period}.` + // Inform about parsing success + parsingMsg.message = `${rowCount} rows are parsed successfully in ${period}.` } - // Loading indicator for csv parsing is not needed anymore - clearTimeout(parseCsvLoadingIndicator) + // Loading indicator for parsing is not needed anymore + clearTimeout(parsingLoadingIndicator) // Add info about import start - this.importCsvMessages.push(importMsg) + this.importMessages.push(importMsg) // Show import progress after 1 second importLoadingIndicator = setTimeout(() => { @@ -291,52 +348,105 @@ export default { this.addedTable = this.tableName // Inform about import success period = time.getPeriod(start, end) - importMsg.message = `Importing CSV into a SQLite database is completed in ${period}.` + importMsg.message = `Importing ${this.typeName} into a SQLite database is completed in ${period}.` importMsg.type = 'success' // Loading indicator for import is not needed anymore clearTimeout(importLoadingIndicator) - this.importCsvCompleted = true + this.importCompleted = true } else { - parseCsvMsg.message = 'Parsing ended with errors.' - parseCsvMsg.type = 'info' - this.importCsvMessages = this.importCsvMessages.concat(parseResult.messages) + parsingMsg.message = 'Parsing ended with errors.' + parsingMsg.type = 'info' + this.importMessages = this.importMessages.concat(parseResult.messages) } } catch (err) { - if (parseCsvMsg.type === 'loading') { - parseCsvMsg.type = 'info' + console.error(err) + if (parsingMsg.type === 'loading') { + parsingMsg.type = 'info' } if (importMsg.type === 'loading') { importMsg.type = 'info' } - this.importCsvMessages.push({ + this.importMessages.push({ message: err, type: 'error' }) } - clearTimeout(parseCsvLoadingIndicator) + clearTimeout(parsingLoadingIndicator) clearTimeout(importLoadingIndicator) this.db.deleteProgressCounter(progressCounterId) this.disableDialog = false }, async finish () { this.$modal.hide(this.dialogName) - const stmt = [ - '/*', + const stmt = this.getQueryExample() + const tabId = await this.$store.dispatch('addTab', { query: stmt }) + this.$store.commit('setCurrentTabId', tabId) + this.importCompleted = false + this.$emit('finish') + events.send('inquiry.create', null, { auto: true }) + }, + getQueryExample () { + return this.isNdJson ? this.getNdJsonQueryExample() + : this.isJson ? this.getJsonQueryExample() + : [ + '/*', ` * Your CSV file has been imported into ${this.addedTable} table.`, ' * You can run this SQL query to make all CSV records available for charting.', ' */', `SELECT * FROM "${this.addedTable}"` - ].join('\n') - const tabId = await this.$store.dispatch('addTab', { query: stmt }) - this.$store.commit('setCurrentTabId', tabId) - this.importCsvCompleted = false - this.$emit('finish') - events.send('inquiry.create', null, { auto: true }) + ].join('\n') + }, + getNdJsonQueryExample () { + try { + const firstRowJson = JSON.parse(this.previewData.values.doc[0]) + const firstKey = Object.keys(firstRowJson)[0] + return [ + '/*', + ` * Your NDJSON file has been imported into ${this.addedTable} table.`, + ` * Run this SQL query to get values of property ${firstKey} and make them available for charting.`, + ' */', + `SELECT doc->>'${firstKey}'`, + `FROM "${this.addedTable}"` + ].join('\n') + } catch (err) { + console.error(err) + return [ + '/*', + ` * Your NDJSON file has been imported into ${this.addedTable} table.`, + ' */', + 'SELECT *', + `FROM "${this.addedTable}"` + ].join('\n') + } + }, + getJsonQueryExample () { + try { + const firstRowJson = JSON.parse(this.previewData.values.doc[0]) + const firstKey = Object.keys(firstRowJson)[0] + return [ + '/*', + ` * Your JSON file has been imported into ${this.addedTable} table.`, + ` * Run this SQL query to get values of property ${firstKey} and make them available for charting.`, + ' */', + 'SELECT *', + `FROM "${this.addedTable}"`, + `JOIN json_each(doc, '$.${firstKey}')` + ].join('\n') + } catch (err) { + console.error(err) + return [ + '/*', + ` * Your NDJSON file has been imported into ${this.addedTable} table.`, + ' */', + 'SELECT *', + `FROM "${this.addedTable}"` + ].join('\n') + } } } } @@ -347,10 +457,14 @@ export default { padding-bottom: 0; } +#csv-json-table-name { +margin-bottom: 24px; +} + .chars { display: flex; align-items: flex-end; - margin: 24px 0 20px; + margin: 0 0 20px; } .char-input { margin-right: 44px; @@ -359,7 +473,7 @@ export default { margin-top: 18px; } -.import-csv-errors { +.import-errors { height: 136px; margin-top: 8px; } diff --git a/src/components/DbUploader.vue b/src/components/DbUploader.vue index ade029d..7694f11 100644 --- a/src/components/DbUploader.vue +++ b/src/components/DbUploader.vue @@ -41,13 +41,13 @@
- - +
@@ -57,7 +57,7 @@ import fIo from '@/lib/utils/fileIo' import ChangeDbIcon from '@/components/svg/changeDb' import database from '@/lib/database' -import CsvImport from '@/components/CsvImport' +import CsvJsonImport from '@/components/CsvJsonImport' import events from '@/lib/utils/events' export default { @@ -79,7 +79,7 @@ export default { }, components: { ChangeDbIcon, - CsvImport + CsvJsonImport }, data () { return { @@ -102,7 +102,7 @@ export default { } }, methods: { - cancelCsvImport () { + cancelImport () { if (this.newDb) { this.newDb.shutDown() this.newDb = null @@ -128,21 +128,22 @@ export default { if (fIo.isDatabase(file)) { this.loadDb(file) } else { + const isJson = fIo.isJSON(file) || fIo.isNDJSON(file) events.send('database.import', file.size, { - from: 'csv', + from: isJson ? 'json' : 'csv', new_db: true }) this.file = file await this.$nextTick() - const csvImport = this.$refs.addCsv - csvImport.reset() - return Promise.all([csvImport.previewCsv(), this.animationPromise]) - .then(csvImport.open) + const csvJsonImportModal = this.$refs.addCsvJson + csvJsonImportModal.reset() + return Promise.all([csvJsonImportModal.preview(), this.animationPromise]) + .then(csvJsonImportModal.open) } }, browse () { - fIo.getFileFromUser('.db,.sqlite,.sqlite3,.csv') + fIo.getFileFromUser('.db,.sqlite,.sqlite3,.csv,.json,.ndjson') .then(this.checkFile) }, diff --git a/src/lib/csv.js b/src/lib/csv.js index e2dc4b4..1ab0871 100644 --- a/src/lib/csv.js +++ b/src/lib/csv.js @@ -7,9 +7,9 @@ const hintsByCode = { } export default { - getResult (source) { + getResult (source, columns) { const result = { - columns: [] + columns: columns || [] } const values = {} if (source.meta.fields) { @@ -24,8 +24,18 @@ export default { return value }) }) + } else if (columns) { + columns.forEach((col, i) => { + values[col] = source.data.map(row => { + let value = row[i] + if (value instanceof Date) { + value = value.toISOString() + } + return value + }) + }) } else { - for (let i = 0; i <= source.data[0].length - 1; i++) { + for (let i = 0; source.data[0] && i <= source.data[0].length - 1; i++) { const colName = `col${i + 1}` result.columns.push(colName) values[colName] = source.data.map(row => { @@ -76,7 +86,7 @@ export default { let res try { res = { - data: this.getResult(results), + data: this.getResult(results, config.columns), delimiter: results.meta.delimiter, hasErrors: false, rowCount: results.data.length diff --git a/src/lib/utils/fileIo.js b/src/lib/utils/fileIo.js index d4ba38d..d537124 100644 --- a/src/lib/utils/fileIo.js +++ b/src/lib/utils/fileIo.js @@ -1,4 +1,10 @@ export default { + isJSON (file) { + return file && file.type === 'application/json' + }, + isNDJSON (file) { + return file && file.name.endsWith('.ndjson') + }, isDatabase (file) { const dbTypes = ['application/vnd.sqlite3', 'application/x-sqlite3'] return file.type @@ -51,19 +57,20 @@ export default { }, importFile () { - const reader = new FileReader() - return this.getFileFromUser('.json') .then(file => { - return new Promise((resolve, reject) => { - reader.onload = e => { - resolve(e.target.result) - } - reader.readAsText(file) - }) + return this.getFileContent(file) }) }, + getFileContent (file) { + const reader = new FileReader() + return new Promise(resolve => { + reader.onload = e => resolve(e.target.result) + reader.readAsText(file) + }) + }, + readFile (path) { return fetch(path) }, diff --git a/src/views/Main/Workspace/Schema/index.vue b/src/views/Main/Workspace/Schema/index.vue index 4133ef9..c41ffb0 100644 --- a/src/views/Main/Workspace/Schema/index.vue +++ b/src/views/Main/Workspace/Schema/index.vue @@ -10,7 +10,7 @@ - +
- - + @@ -40,7 +40,7 @@ import TreeChevron from '@/components/svg/treeChevron' import DbUploader from '@/components/DbUploader' import ExportIcon from '@/components/svg/export' import AddTableIcon from '@/components/svg/addTable' -import CsvImport from '@/components/CsvImport' +import CsvJsonImport from '@/components/CsvJsonImport' export default { name: 'Schema', @@ -51,7 +51,7 @@ export default { DbUploader, ExportIcon, AddTableIcon, - CsvImport + CsvJsonImport }, data () { return { @@ -80,16 +80,17 @@ export default { exportToFile () { this.$store.state.db.export(`${this.dbName}.sqlite`) }, - async addCsv () { - this.file = await fIo.getFileFromUser('.csv') + async addCsvJson () { + this.file = await fIo.getFileFromUser('.csv,.json,.ndjson') await this.$nextTick() - const csvImport = this.$refs.addCsv - csvImport.reset() - await csvImport.previewCsv() - csvImport.open() + const csvJsonImportModal = this.$refs.addCsvJson + csvJsonImportModal.reset() + await csvJsonImportModal.preview() + csvJsonImportModal.open() + const isJson = fIo.isJSON(this.file) || fIo.isNDJSON(this.file) events.send('database.import', this.file.size, { - from: 'csv', + from: isJson ? 'json' : 'csv', new_db: false }) } diff --git a/src/views/Main/index.vue b/src/views/Main/index.vue index fdcd5e8..ed02390 100644 --- a/src/views/Main/index.vue +++ b/src/views/Main/index.vue @@ -1,7 +1,7 @@