diff --git a/.eslintrc.js b/.eslintrc.js index 654134e..cf516ef 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -12,7 +12,8 @@ module.exports = { }, rules: { 'no-console': process.env.NODE_ENV === 'production' ? 'warn' : 'off', - 'no-debugger': process.env.NODE_ENV === 'production' ? 'warn' : 'off' + 'no-debugger': process.env.NODE_ENV === 'production' ? 'warn' : 'off', + 'no-case-declarations': 'off' }, overrides: [ { diff --git a/karma.conf.js b/karma.conf.js index c21d11f..ceb8523 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -25,13 +25,6 @@ module.exports = function (config) { included: false, served: true, nocache: false - }, - { - pattern: 'node_modules/sql.js/dist/worker.sql-wasm.js', - watched: false, - included: false, - served: true, - nocache: false } ], @@ -136,6 +129,10 @@ module.exports = function (config) { } ] }, + { + test: /\.worker\.js$/, + loader: 'worker-loader' + }, { test: /\.(png|jpe?g|gif|svg)(\?.*)?$/, loader: 'url-loader' @@ -181,7 +178,8 @@ module.exports = function (config) { } }, proxies: { - '/js/': '/base/node_modules/sql.js/dist/' + '/_karma_webpack_/sql-wasm.wasm': '/base/node_modules/sql.js/dist/sql-wasm.wasm', + '/base/sql-wasm.wasm': '/base/node_modules/sql.js/dist/sql-wasm.wasm' } }) // Fix the timezone diff --git a/package-lock.json b/package-lock.json index e6a3a93..910c47c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,9 @@ "core-js": "^3.6.5", "debounce": "^1.2.0", "nanoid": "^3.1.12", + "papaparse": "^5.3.0", "plotly.js": "^1.57.1", + "promise-worker": "^2.0.1", "react": "^16.13.1", "react-chart-editor": "^0.42.0", "react-dom": "^16.13.1", @@ -37,6 +39,7 @@ "@vue/test-utils": "^1.1.2", "babel-eslint": "^10.1.0", "chai": "^4.1.2", + "chai-as-promised": "^7.1.1", "eslint": "^6.7.2", "eslint-plugin-import": "^2.20.2", "eslint-plugin-node": "^11.1.0", @@ -46,7 +49,8 @@ "karma": "^3.1.4", "karma-webpack": "^4.0.2", "vue-cli-plugin-ui-karma": "^0.2.5", - "vue-template-compiler": "^2.6.11" + "vue-template-compiler": "^2.6.11", + "worker-loader": "^3.0.8" } }, "node_modules/@babel/code-frame": { @@ -1559,9 +1563,9 @@ } }, "node_modules/@types/json-schema": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.5.tgz", - "integrity": "sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ==", + "version": "7.0.7", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz", + "integrity": "sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==", "dev": true }, "node_modules/@types/json5": { @@ -2510,15 +2514,19 @@ } }, "node_modules/ajv": { - "version": "6.12.3", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.3.tgz", - "integrity": "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==", + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" } }, "node_modules/ajv-errors": { @@ -2528,10 +2536,13 @@ "dev": true }, "node_modules/ajv-keywords": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.1.tgz", - "integrity": "sha512-KWcq3xN8fDjSB+IMoh2VaXVhRI0BBGxoYp3rx7Pkb6z0cFjYR9Q9l4yZqqals0/zsioCmocC5H6UvsGD4MoIBA==", - "dev": true + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "dev": true, + "peerDependencies": { + "ajv": "^6.9.1" + } }, "node_modules/align-text": { "version": "0.1.4", @@ -4296,6 +4307,18 @@ "node": ">=4" } }, + "node_modules/chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "dependencies": { + "check-error": "^1.0.2" + }, + "peerDependencies": { + "chai": ">= 2.1.2 < 5" + } + }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -14826,6 +14849,11 @@ "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", "dev": true }, + "node_modules/papaparse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.0.tgz", + "integrity": "sha512-Lb7jN/4bTpiuGPrYy4tkKoUS8sTki8zacB5ke1p5zolhcSE4TlWgrlsxjrDTbG/dFVh07ck7X36hUf/b5V68pg==" + }, "node_modules/parallel-transform": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz", @@ -16287,6 +16315,11 @@ "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=", "dev": true }, + "node_modules/promise-worker": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promise-worker/-/promise-worker-2.0.1.tgz", + "integrity": "sha512-jR7vHqMEwWJ15i9vA3qyCKwRHihyLJp1sAa3RyY5F35m3u5s2lQUfq0nzVjbA8Xc7+3mL3Y9+9MHBO9UFRpFxA==" + }, "node_modules/prop-types": { "version": "15.7.2", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", @@ -21999,6 +22032,58 @@ "errno": "~0.1.7" } }, + "node_modules/worker-loader": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-3.0.8.tgz", + "integrity": "sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g==", + "dev": true, + "dependencies": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/worker-loader/node_modules/loader-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.0.tgz", + "integrity": "sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ==", + "dev": true, + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + }, + "engines": { + "node": ">=8.9.0" + } + }, + "node_modules/worker-loader/node_modules/schema-utils": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.0.0.tgz", + "integrity": "sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.6", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, "node_modules/world-calendars": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/world-calendars/-/world-calendars-1.0.3.tgz", @@ -23752,9 +23837,9 @@ } }, "@types/json-schema": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.5.tgz", - "integrity": "sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ==", + "version": "7.0.7", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz", + "integrity": "sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==", "dev": true }, "@types/json5": { @@ -24613,9 +24698,9 @@ } }, "ajv": { - "version": "6.12.3", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.3.tgz", - "integrity": "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==", + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, "requires": { "fast-deep-equal": "^3.1.1", @@ -24631,10 +24716,11 @@ "dev": true }, "ajv-keywords": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.1.tgz", - "integrity": "sha512-KWcq3xN8fDjSB+IMoh2VaXVhRI0BBGxoYp3rx7Pkb6z0cFjYR9Q9l4yZqqals0/zsioCmocC5H6UvsGD4MoIBA==", - "dev": true + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "dev": true, + "requires": {} }, "align-text": { "version": "0.1.4", @@ -26184,6 +26270,15 @@ "type-detect": "^4.0.5" } }, + "chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "requires": { + "check-error": "^1.0.2" + } + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -35229,6 +35324,11 @@ "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", "dev": true }, + "papaparse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.0.tgz", + "integrity": "sha512-Lb7jN/4bTpiuGPrYy4tkKoUS8sTki8zacB5ke1p5zolhcSE4TlWgrlsxjrDTbG/dFVh07ck7X36hUf/b5V68pg==" + }, "parallel-transform": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz", @@ -36516,6 +36616,11 @@ "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=", "dev": true }, + "promise-worker": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promise-worker/-/promise-worker-2.0.1.tgz", + "integrity": "sha512-jR7vHqMEwWJ15i9vA3qyCKwRHihyLJp1sAa3RyY5F35m3u5s2lQUfq0nzVjbA8Xc7+3mL3Y9+9MHBO9UFRpFxA==" + }, "prop-types": { "version": "15.7.2", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", @@ -41542,6 +41647,40 @@ "errno": "~0.1.7" } }, + "worker-loader": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-3.0.8.tgz", + "integrity": "sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g==", + "dev": true, + "requires": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "dependencies": { + "loader-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.0.tgz", + "integrity": "sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ==", + "dev": true, + "requires": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + } + }, + "schema-utils": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.0.0.tgz", + "integrity": "sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.6", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + } + } + } + }, "world-calendars": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/world-calendars/-/world-calendars-1.0.3.tgz", diff --git a/package.json b/package.json index 77dc061..4497e6d 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,9 @@ "core-js": "^3.6.5", "debounce": "^1.2.0", "nanoid": "^3.1.12", + "papaparse": "^5.3.0", "plotly.js": "^1.57.1", + "promise-worker": "^2.0.1", "react": "^16.13.1", "react-chart-editor": "^0.42.0", "react-dom": "^16.13.1", @@ -38,6 +40,7 @@ "@vue/test-utils": "^1.1.2", "babel-eslint": "^10.1.0", "chai": "^4.1.2", + "chai-as-promised": "^7.1.1", "eslint": "^6.7.2", "eslint-plugin-import": "^2.20.2", "eslint-plugin-node": "^11.1.0", @@ -47,6 +50,7 @@ "karma": "^3.1.4", "karma-webpack": "^4.0.2", "vue-cli-plugin-ui-karma": "^0.2.5", - "vue-template-compiler": "^2.6.11" + "vue-template-compiler": "^2.6.11", + "worker-loader": "^3.0.8" } } diff --git a/src/components/DbUpload.vue b/src/components/DbUpload.vue index 852d357..bd96032 100644 --- a/src/components/DbUpload.vue +++ b/src/components/DbUpload.vue @@ -1,6 +1,7 @@ diff --git a/src/components/Schema.vue b/src/components/Schema.vue index bdefa9f..b9d0b92 100644 --- a/src/components/Schema.vue +++ b/src/components/Schema.vue @@ -8,9 +8,7 @@ {{ dbName }} -
- -
+
import TableDescription from '@/components/TableDescription' import TextField from '@/components/TextField' -import ChangeDbIcon from '@/components/svg/changeDb' import TreeChevron from '@/components/svg/treeChevron' -import fu from '@/fileUtils' +import dbUpload from '@/components/DbUpload' export default { name: 'Schema', components: { TableDescription, TextField, - ChangeDbIcon, - TreeChevron + TreeChevron, + dbUpload }, data () { return { @@ -59,17 +56,6 @@ export default { dbName () { return this.$store.state.dbName } - }, - methods: { - changeDb () { - fu.getFileFromUser('.db,.sqlite,.sqlite3') - .then(file => { - return this.$db.loadDb(file) - }) - .then((schema) => { - this.$store.commit('saveSchema', schema) - }) - } } } diff --git a/src/components/Tab.vue b/src/components/Tab.vue index 85c83dc..bb08140 100644 --- a/src/components/Tab.vue +++ b/src/components/Tab.vue @@ -104,21 +104,16 @@ export default { }, methods: { // Run a command in the database - execute () { - // this.$refs.output.textContent = 'Fetching results...' */ + async execute () { this.isGettingResults = true this.result = null this.error = null - return this.$db.execute(this.query + ';') - .then(result => { - this.result = result - }) - .catch(err => { - this.error = err - }) - .finally(() => { - this.isGettingResults = false - }) + try { + this.result = await this.$store.state.db.execute(this.query + ';') + } catch (err) { + this.error = err + } + this.isGettingResults = false }, handleResize () { if (this.view === 'chart') { diff --git a/src/csv.js b/src/csv.js new file mode 100644 index 0000000..d1b1ab9 --- /dev/null +++ b/src/csv.js @@ -0,0 +1,78 @@ +import Papa from 'papaparse' + +const hintsByCode = { + MissingQuotes: 'Edit your CSV so that the field has a closing quote char.', + TooFewFields: 'Add fields or try another delimiter.', + TooManyFields: 'Edit your CSV or try another delimiter.' +} + +export default { + getResut (source) { + const result = {} + if (source.meta.fields) { + result.columns = source.meta.fields + result.values = source.data.map(row => { + const resultRow = [] + result.columns.forEach(col => { resultRow.push(row[col]) }) + return resultRow + }) + } else { + result.values = source.data + result.columns = [] + for (let i = 1; i <= source.data[0].length; i++) { + result.columns.push(`col ${i}`) + } + } + + return result + }, + + parse (file, config = {}) { + return new Promise((resolve, reject) => { + const defaultConfig = { + delimiter: '', // auto-detect + newline: '', // auto-detect + quoteChar: '"', + escapeChar: '"', + header: false, + transformHeader: undefined, + dynamicTyping: true, + preview: 0, + encoding: 'UTF-8', + worker: true, + comments: false, + step: undefined, + complete: results => { + const res = { + data: this.getResut(results), + delimiter: results.meta.delimiter, + hasErrors: false + } + res.messages = results.errors.map(msg => { + msg.type = msg.code === 'UndetectableDelimiter' ? 'info' : 'error' + if (msg.type === 'error') res.hasErrors = true + msg.hint = hintsByCode[msg.code] + return msg + }) + resolve(res) + }, + error: (error, file) => { + reject(error) + }, + download: false, + downloadRequestHeaders: undefined, + downloadRequestBody: undefined, + skipEmptyLines: 'greedy', + chunk: undefined, + chunkSize: undefined, + fastMode: undefined, + beforeFirstChunk: undefined, + withCredentials: undefined, + transform: undefined, + delimitersToGuess: [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP] + } + + Papa.parse(file, { ...defaultConfig, ...config }) + }) + } +} diff --git a/src/database.js b/src/database.js index 6601444..446eff4 100644 --- a/src/database.js +++ b/src/database.js @@ -1,59 +1,112 @@ import sqliteParser from 'sqlite-parser' -const worker = new Worker('js/worker.sql-wasm.js') +import fu from '@/fileUtils' +// We can import workers like so because of worker-loader: +// https://webpack.js.org/loaders/worker-loader/ +import Worker from '@/db.worker.js' + +// Use promise-worker in order to turn worker into the promise based one: +// https://github.com/nolanlawson/promise-worker +import PromiseWorker from 'promise-worker' + +function getNewDatabase () { + const worker = new Worker() + return new Database(worker) +} export default { - loadDb (file) { - return new Promise((resolve, reject) => { - const f = file - const r = new FileReader() - r.onload = () => { - // on 'action: open' completed - worker.onmessage = () => { - const getSchemaSql = ` - SELECT name, sql - FROM sqlite_master - WHERE type='table' AND name NOT LIKE 'sqlite_%';` + getNewDatabase +} - this.execute(getSchemaSql) - .then(result => { - // Parse DDL statements to get column names and types - const parsedSchema = [] - result.values.forEach(item => { - parsedSchema.push({ - name: item[0], - columns: getColumns(item[1]) - }) - }) +let progressCounterIds = 0 +class Database { + constructor (worker) { + this.worker = worker + this.pw = new PromiseWorker(worker) - // Return db name and schema - resolve({ - dbName: file.name, - schema: parsedSchema - }) - }) - } - - try { - worker.postMessage({ action: 'open', buffer: r.result }, [r.result]) - } catch (exception) { - worker.postMessage({ action: 'open', buffer: r.result }) - } + this.importProgresses = {} + worker.addEventListener('message', e => { + const progress = e.data.progress + if (progress !== undefined) { + const id = e.data.id + this.importProgresses[id].dispatchEvent(new CustomEvent('progress', { + detail: progress + })) } - r.readAsArrayBuffer(f) }) - }, - execute (commands) { - return new Promise((resolve, reject) => { - worker.onmessage = (event) => { - if (event.data.error) { - reject(event.data.error) - } else { - // if it was more than one select - take only the first one - resolve(event.data.results[0]) - } - } - worker.postMessage({ action: 'exec', sql: commands }) + } + + shutDown () { + this.worker.terminate() + } + + createProgressCounter (callback) { + const id = progressCounterIds++ + this.importProgresses[id] = new EventTarget() + this.importProgresses[id].addEventListener('progress', callback) + return id + } + + deleteProgressCounter (id) { + delete this.importProgresses[id] + } + + async createDb (name, data, progressCounterId) { + const result = await this.pw.postMessage({ + action: 'import', + columns: data.columns, + values: data.values, + progressCounterId }) + + if (result.error) { + throw result.error + } + + return await this.getSchema(name) + } + + async loadDb (file) { + const fileContent = await fu.readAsArrayBuffer(file) + const res = await this.pw.postMessage({ action: 'open', buffer: fileContent }) + + if (res.error) { + throw res.error + } + + return this.getSchema(file.name) + } + + async getSchema (name) { + const getSchemaSql = ` + SELECT name, sql + FROM sqlite_master + WHERE type='table' AND name NOT LIKE 'sqlite_%'; + ` + const result = await this.execute(getSchemaSql) + // Parse DDL statements to get column names and types + const parsedSchema = [] + result.values.forEach(item => { + parsedSchema.push({ + name: item[0], + columns: getColumns(item[1]) + }) + }) + + // Return db name and schema + return { + dbName: name, + schema: parsedSchema + } + } + + async execute (commands) { + const results = await this.pw.postMessage({ action: 'exec', sql: commands }) + + if (results.error) { + throw results.error + } + // if it was more than one select - take only the first one + return results[0] } } diff --git a/src/db.worker.js b/src/db.worker.js new file mode 100644 index 0000000..2be2009 --- /dev/null +++ b/src/db.worker.js @@ -0,0 +1,98 @@ +import registerPromiseWorker from 'promise-worker/register' +import initSqlJs from 'sql.js/dist/sql-wasm.js' +import dbUtils from '@/dbUtils' + +const sqlModuleReady = initSqlJs() +let db = null + +function onModuleReady (SQL) { + function createDb (data) { + if (db != null) db.close() + db = new SQL.Database(data) + return db + } + + const data = this + + switch (data && data.action) { + case 'open': + const buff = data.buffer + createDb(buff && new Uint8Array(buff)) + return { + ready: true + } + case 'exec': + if (db === null) { + createDb() + } + if (!data.sql) { + throw new Error('exec: Missing query string') + } + return db.exec(data.sql, data.params) + case 'each': + if (db === null) { + createDb() + } + const callback = function callback (row) { + return { + row: row, + finished: false + } + } + const done = function done () { + return { + finished: true + } + } + return db.each(data.sql, data.params, callback, done) + case 'import': + createDb() + const values = data.values + const columns = data.columns + const chunkSize = 1500 + db.exec(dbUtils.getCreateStatement(columns, values)) + const chunks = dbUtils.generateChunks(values, chunkSize) + const chunksAmount = Math.ceil(values.length / chunkSize) + let count = 0 + const insertStr = dbUtils.getInsertStmt(columns) + const insertStmt = db.prepare(insertStr) + + postMessage({ progress: 0, id: data.progressCounterId }) + for (const chunk of chunks) { + db.exec('BEGIN') + for (const row of chunk) { + insertStmt.run(row) + } + db.exec('COMMIT') + count++ + postMessage({ progress: 100 * (count / chunksAmount), id: data.progressCounterId }) + } + + return { + finish: true + } + case 'export': + return db.export() + case 'close': + if (db) { + db.close() + } + return { + finished: true + } + default: + throw new Error('Invalid action : ' + (data && data.action)) + } +} + +function onError (err) { + return { + error: new Error(err.message) + } +} + +registerPromiseWorker(data => { + return sqlModuleReady + .then(onModuleReady.bind(data)) + .catch(onError) +}) diff --git a/src/dbUtils.js b/src/dbUtils.js new file mode 100644 index 0000000..024c5a2 --- /dev/null +++ b/src/dbUtils.js @@ -0,0 +1,44 @@ +export default { + * generateChunks (arr, size) { + const count = Math.ceil(arr.length / size) + + for (let i = 0; i <= count - 1; i++) { + const start = size * i + const end = start + size + yield arr.slice(start, end) + } + }, + + getInsertStmt (columns) { + const colList = `"${columns.join('", "')}"` + const params = columns.map(() => '?').join(' ,') + return `INSERT INTO csv_import (${colList}) VALUES (${params});` + }, + + getCreateStatement (columns, values) { + let result = 'CREATE table csv_import(' + columns.forEach((col, index) => { + // Get the first row of values to determine types + const value = values[0][index] + let type = '' + switch (typeof value) { + case 'number': { + type = 'REAL' + break + } + case 'boolean': { + type = 'INTEGER' + break + } + case 'string': { + type = 'TEXT' + break + } + default: type = 'TEXT' + } + result += `"${col}" ${type},` + }) + result = result.replace(/.$/, ');') + return result + } +} diff --git a/src/fileUtils.js b/src/fileUtils.js index 179bd51..4c23253 100644 --- a/src/fileUtils.js +++ b/src/fileUtils.js @@ -50,5 +50,21 @@ export default { readFile (path) { return fetch(path) + }, + + readAsArrayBuffer (file) { + const fileReader = new FileReader() + + return new Promise((resolve, reject) => { + fileReader.onerror = () => { + fileReader.abort() + reject(new DOMException('Problem parsing input file.')) + } + + fileReader.onload = () => { + resolve(fileReader.result) + } + fileReader.readAsArrayBuffer(file) + }) } } diff --git a/src/main.js b/src/main.js index 312b8de..8cc0f11 100644 --- a/src/main.js +++ b/src/main.js @@ -4,7 +4,6 @@ import router from './router' import store from './store' import { VuePlugin } from 'vuera' import VModal from 'vue-js-modal' -import db from '@/database' import '@/assets/styles/variables.css' import '@/assets/styles/buttons.css' @@ -17,7 +16,6 @@ Vue.use(VuePlugin) Vue.use(VModal) Vue.config.productionTip = false -Vue.prototype.$db = db new Vue({ router, diff --git a/src/store/index.js b/src/store/index.js index de6b708..c82331c 100644 --- a/src/store/index.js +++ b/src/store/index.js @@ -12,10 +12,17 @@ export const state = { currentTab: null, currentTabId: null, untitledLastIndex: 0, - predefinedQueries: [] + predefinedQueries: [], + db: null } export const mutations = { + setDb (state, db) { + if (state.db) { + state.db.shutDown() + } + state.db = db + }, saveSchema (state, { dbName, schema }) { state.dbName = dbName state.schema = schema @@ -73,19 +80,18 @@ export const mutations = { export const actions = { async addTab ({ state }, data) { - let tab + const tab = data ? JSON.parse(JSON.stringify(data)) : {} // If no data then create a new blank one... - if (!data) { - tab = { - id: nanoid(), - name: null, - tempName: state.untitledLastIndex - ? `Untitled ${state.untitledLastIndex}` - : 'Untitled', - isUnsaved: true - } + // No data.id means to create new tab, but not blank, + // e.g. with 'select * from csv_import' query after csv import + if (!data || !data.id) { + tab.id = nanoid() + tab.name = null + tab.tempName = state.untitledLastIndex + ? `Untitled ${state.untitledLastIndex}` + : 'Untitled' + tab.isUnsaved = true } else { - tab = JSON.parse(JSON.stringify(data)) tab.isUnsaved = false } diff --git a/src/time.js b/src/time.js new file mode 100644 index 0000000..8295734 --- /dev/null +++ b/src/time.js @@ -0,0 +1,36 @@ +export default { + getPeriod (start, end) { + let diff = end.getTime() - start.getTime() + let result = '' + + const days = Math.floor(diff / (1000 * 60 * 60 * 24)) + diff -= days * (1000 * 60 * 60 * 24) + if (days) { + result += days + ' d ' + } + + const hours = Math.floor(diff / (1000 * 60 * 60)) + diff -= hours * (1000 * 60 * 60) + if (hours) { + result += hours + ' h ' + } + + const mins = Math.floor(diff / (1000 * 60)) + diff -= mins * (1000 * 60) + if (mins) { + result += mins + ' m ' + } + + const seconds = Math.floor(diff / (1000)) + diff -= seconds * (1000) + if (seconds) { + result += seconds + ' s ' + } + + if (diff) { + result += diff + ' ms ' + } + + return result.replace(/\s$/, '') + } +} diff --git a/src/views/Home.vue b/src/views/Home.vue index c3cf840..851ef38 100644 --- a/src/views/Home.vue +++ b/src/views/Home.vue @@ -1,6 +1,6 @@