1
0
mirror of https://github.com/lana-k/sqliteviz.git synced 2025-12-07 02:28:54 +08:00

11 Commits

Author SHA1 Message Date
lana-k
310a939109 #89 add tests 2021-12-24 16:13:42 +01:00
lana-k
bb9ba08902 #89 remove head and body 2021-12-22 20:42:53 +01:00
lana-k
c7c727ff78 fix lint 2021-12-21 22:15:21 +01:00
lana-k
8669a6a9e5 #89 export to html plolty charts and pivots 2021-12-21 22:13:02 +01:00
lana-k
c1cc5bb95e getHtml for chart #89 2021-12-20 22:31:08 +01:00
lana-k
9c55e76a41 update version 2021-12-19 16:00:11 +01:00
lana-k
70a9edf57e fix lint 2021-12-19 15:57:30 +01:00
lana-k
b2c2344951 update version 2021-12-19 15:38:17 +01:00
lana-k
cbec91e78a Merge branch 'master' of github.com:lana-k/sqliteviz 2021-12-19 15:37:01 +01:00
lana-k
816b0e6218 show plotly warnings and errors #55 2021-12-19 15:36:46 +01:00
saaj
4ed93bbea7 Two more extensions and improved extension documentation (#86) 2021-09-10 20:11:37 +02:00
16 changed files with 956 additions and 241 deletions

View File

@@ -13,6 +13,21 @@ SQLite [amalgamation][2] extensions included:
1. [FTS5][4] -- virtual table module that provides full-text search
functionality
2. [FTS3/FTS4][15] -- older virtual table modules for full-text search
3. [JSON1][16] -- scalar, aggregate and table-valued functions for managing JSON data
SQLite [contribution extensions][17]:
1. [extension-functions][18] -- mathematical and string extension functions for SQL queries.
Math: `acos`, `asin`, `atan`, `atn2`, `atan2`, `acosh`, `asinh`, `atanh`, `difference`,
`degrees`, `radians`, `cos`, `sin`, `tan`, `cot`, `cosh`, `sinh`, `tanh`, `coth`,
`exp`, `log`, `log10`, `power`, `sign`, `sqrt`, `square`, `ceil`, `floor`, `pi`.
String: `replicate`, `charindex`, `leftstr`, `rightstr`, `ltrim`, `rtrim`, `trim`,
`replace`, `reverse`, `proper`, `padl`, `padr`, `padc`, `strfilter`.
Aggregate: `stdev`, `variance`, `mode`, `median`, `lower_quartile`, `upper_quartile`.
SQLite [miscellaneous extensions][3] included:
@@ -21,6 +36,9 @@ SQLite [miscellaneous extensions][3] included:
[Querying Tree Structures in SQLite][11] ([closure.c][8])
3. `uuid`, `uuid_str` and `uuid_blob` RFC-4122 UUID functions ([uuid.c][9])
4. `regexp` (hence `REGEXP` operator) and `regexpi` functions ([regexp.c][10])
5. `percentile` function ([percentile.c][13])
6. `decimal`, `decimal_cmp`, `decimal_add`, `decimal_sub` and `decimal_mul` functions
([decimal.c][14])
SQLite 3rd party extensions included:
@@ -29,6 +47,9 @@ SQLite 3rd party extensions included:
To ease the step to have working clone locally, the build is committed into
the repository.
Examples of queries involving these extensions can be found in the test suite in
[sqliteExtensions.spec.js][19].
## Build method
Basically it's extended amalgamation and `SQLITE_EXTRA_INIT` concisely
@@ -71,3 +92,10 @@ described in [this message from SQLite Forum][12]:
[10]: https://sqlite.org/src/file/ext/misc/regexp.c
[11]: https://charlesleifer.com/blog/querying-tree-structures-in-sqlite-using-python-and-the-transitive-closure-extension/
[12]: https://sqlite.org/forum/forumpost/6ad7d4f4bebe5e06?raw
[13]: https://sqlite.org/src/file/ext/misc/percentile.c
[14]: https://sqlite.org/src/file/ext/misc/decimal.c
[15]: https://sqlite.org/fts3.html
[16]: https://sqlite.org/json1.html
[17]: https://sqlite.org/contrib/
[18]: https://sqlite.org/contrib//download/extension-functions.c?get=25
[19]: https://github.com/lana-k/sqliteviz/blob/master/tests/lib/database/sqliteExtensions.spec.js

View File

@@ -24,6 +24,8 @@ extension_urls = (
('https://sqlite.org/src/raw/dbfd8543?at=closure.c', 'sqlite3_closure_init'),
('https://sqlite.org/src/raw/5bb2264c?at=uuid.c', 'sqlite3_uuid_init'),
('https://sqlite.org/src/raw/5853b0e5?at=regexp.c', 'sqlite3_regexp_init'),
('https://sqlite.org/src/raw/b9086e22?at=percentile.c', 'sqlite3_percentile_init'),
('https://sqlite.org/src/raw/09f967dc?at=decimal.c', 'sqlite3_decimal_init'),
# Third-party extension
# =====================
('https://github.com/jakethaw/pivot_vtab/raw/08ab0797/pivot_vtab.c', 'sqlite3_pivotvtab_init'),

Binary file not shown.

684
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "sqliteviz",
"version": "0.16.0",
"version": "0.18.0",
"license": "Apache-2.0",
"private": true,
"scripts": {

View File

@@ -0,0 +1,49 @@
<template>
<svg
width="19"
height="18"
viewBox="0 0 19 18"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M5.1626 10.0745L7.56641 10.8831V12.2322L3.68164 10.6501V9.4812L7.56641
7.89917V9.2439L5.1626 10.0745ZM8.99023 13.3H7.93994L10.124 6.35229H11.1787L8.99023
13.3ZM14.1099 10.0613L11.7192 9.24829V7.90356L15.582 9.4856V10.6545L11.7192
12.2366V10.8918L14.1099 10.0613Z"
fill="#A2B1C6"
/>
<path
d="M2.17041 0.0637207H16.2185V1.56372H2.17041V9.30354H0.67041V1.56372C0.67041 0.73872
1.34541 0.0637207 2.17041 0.0637207Z"
fill="#A2B1C6"
/>
<path
d="M17.1704 0.0637207H15.3052V1.56372H17.1704V9.84163H18.6704V1.56372C18.6704 0.73872
17.9954 0.0637207 17.1704 0.0637207Z"
fill="#A2B1C6"
/>
<path
d="M2.17041 17.1098H15.8754V15.6098H2.17041V8.78486H0.67041V15.6098C0.67041 16.4348
1.34541 17.1098 2.17041 17.1098Z"
fill="#A2B1C6"
/>
<path
d="M17.1704 17.1098H15.3052V15.6098H17.1704V8.55939H18.6704V15.6098C18.6704 16.4348
17.9954 17.1098 17.1704 17.1098Z"
fill="#A2B1C6"
/>
<path
fill-rule="evenodd"
clip-rule="evenodd"
d="M18.1197 4.13787H1.76172V3.03787H18.1197V4.13787Z"
fill="#A2B1C6"
/>
</svg>
</template>
<script>
export default {
name: 'HtmlIcon'
}
</script>

View File

@@ -1,5 +1,6 @@
import dereference from 'react-chart-editor/lib/lib/dereference'
import plotly from 'plotly.js'
import { nanoid } from 'nanoid'
export function getOptionsFromDataSources (dataSources) {
if (!dataSources) {
@@ -33,8 +34,43 @@ export async function getImageDataUrl (element, type) {
})
}
export function getChartData (element) {
const chartElement = element.querySelector('.js-plotly-plot')
return {
data: chartElement.data,
layout: chartElement.layout
}
}
export function getHtml (options) {
const chartId = nanoid()
return `
<script src="https://cdn.plot.ly/plotly-latest.js" charset="UTF-8"></script>
<div id="${chartId}"></div>
<script>
const el = document.getElementById("${chartId}")
let timeout
function debounceResize() {
clearTimeout(timeout)
timeout = setTimeout(() => {
var r = el.getBoundingClientRect()
Plotly.relayout(el, {width: r.width, height: r.height})
}, 200)
}
const resizeObserver = new ResizeObserver(debounceResize)
resizeObserver.observe(el)
Plotly.newPlot(el, ${JSON.stringify(options.data)}, ${JSON.stringify(options.layout)})
</script>
`
}
export default {
getOptionsFromDataSources,
getOptionsForSave,
getImageDataUrl
getImageDataUrl,
getHtml,
getChartData
}

View File

@@ -55,6 +55,12 @@ export default {
return chartHelper.getOptionsFromDataSources(this.dataSources)
}
},
created () {
// https://github.com/plotly/plotly.js/issues/4555
plotly.setPlotConfig({
notifyOnLogging: 1
})
},
mounted () {
this.resizeObserver = new ResizeObserver(this.handleResize)
this.resizeObserver.observe(this.$refs.chartContainer)
@@ -99,6 +105,13 @@ export default {
fIo.downloadFromUrl(url, 'chart')
},
saveAsHtml () {
fIo.exportToFile(
chartHelper.getHtml(this.state),
'chart.html',
'text/html'
)
},
async prepareCopy (type = 'png') {
return await chartHelper.getImageDataUrl(this.$refs.plotlyEditor.$el, type)
}

View File

@@ -19,7 +19,7 @@ import $ from 'jquery'
import 'pivottable'
import 'pivottable/dist/pivot.css'
import PivotUi from './PivotUi'
import { getPivotCanvas } from './pivotHelper'
import pivotHelper from './pivotHelper'
import Chart from '@/views/Main/Workspace/Tabs/Tab/DataView/Chart'
import chartHelper from '@/lib/chartHelper'
import Vue from 'vue'
@@ -169,7 +169,7 @@ export default {
} else {
const source = this.viewStandartChart
? await chartHelper.getImageDataUrl(this.$refs.pivotOutput, 'png')
: (await getPivotCanvas(this.$refs.pivotOutput)).toDataURL('image/png')
: (await pivotHelper.getPivotCanvas(this.$refs.pivotOutput)).toDataURL('image/png')
this.$emit('loadingImageCompleted')
fIo.downloadFromUrl(source, 'pivot')
@@ -182,7 +182,7 @@ export default {
} else if (this.viewStandartChart) {
return await chartHelper.getImageDataUrl(this.$refs.pivotOutput, 'png')
} else {
return await getPivotCanvas(this.$refs.pivotOutput)
return await pivotHelper.getPivotCanvas(this.$refs.pivotOutput)
}
},
@@ -193,6 +193,25 @@ export default {
const url = await chartHelper.getImageDataUrl(this.$refs.pivotOutput, 'svg')
fIo.downloadFromUrl(url, 'pivot')
}
},
saveAsHtml () {
if (this.viewCustomChart) {
this.pivotOptions.rendererOptions.customChartComponent.saveAsHtml()
} else if (this.viewStandartChart) {
const chartState = chartHelper.getChartData(this.$refs.pivotOutput)
fIo.exportToFile(
chartHelper.getHtml(chartState),
'chart.html',
'text/html'
)
} else {
fIo.exportToFile(
pivotHelper.getPivotHtml(this.$refs.pivotOutput),
'pivot.html',
'text/html'
)
}
}
}
}

View File

@@ -81,3 +81,40 @@ export async function getPivotCanvas (pivotOutput) {
const tableElement = pivotOutput.querySelector('.pvtTable')
return await html2canvas(tableElement, { logging: false })
}
export function getPivotHtml (pivotOutput) {
return `
<style>
table.pvtTable {
font-family: Arial, sans-serif;
font-size: 12px;
text-align: left;
border-collapse: collapse;
min-width: 100%;
}
table.pvtTable .pvtColLabel {
text-align: center;
}
table.pvtTable .pvtTotalLabel {
text-align: right;
}
table.pvtTable tbody tr td {
color: #506784;
border: 1px solid #DFE8F3;
text-align: right;
}
table.pvtTable thead tr th,
table.pvtTable tbody tr th {
background-color: #506784;
color: #fff;
border: 1px solid #DFE8F3;
}
</style>
${pivotOutput.outerHTML}
`
}
export default {
getPivotCanvas,
getPivotHtml
}

View File

@@ -51,6 +51,13 @@
<export-to-svg-icon />
</icon-button>
<icon-button
tooltip="Save as HTML"
tooltip-position="top-left"
@click="saveAsHtml"
>
<HtmlIcon />
</icon-button>
<icon-button
:loading="copyingImage"
tooltip="Copy visualisation to clipboard"
@@ -81,6 +88,7 @@ import SideToolBar from '../SideToolBar'
import IconButton from '@/components/IconButton'
import ChartIcon from '@/components/svg/chart'
import PivotIcon from '@/components/svg/pivot'
import HtmlIcon from '@/components/svg/html'
import ExportToSvgIcon from '@/components/svg/exportToSvg'
import PngIcon from '@/components/svg/png'
import ClipboardIcon from '@/components/svg/clipboard'
@@ -100,6 +108,7 @@ export default {
PivotIcon,
ExportToSvgIcon,
PngIcon,
HtmlIcon,
ClipboardIcon,
loadingDialog
},
@@ -177,6 +186,9 @@ export default {
saveAsSvg () {
this.$refs.viewComponent.saveAsSvg()
},
saveAsHtml () {
this.$refs.viewComponent.saveAsHtml()
}
}
}

View File

@@ -64,7 +64,40 @@ describe('chartHelper.js', () => {
expect(/^data:image\/png/.test(url)).to.equal(true)
url = await chartHelper.getImageDataUrl(element, 'svg')
console.log()
expect(/^data:image\/svg\+xml/.test(url)).to.equal(true)
})
it('getChartData returns plotly data and layout from element', async () => {
const element = document.createElement('div')
const child = document.createElement('div')
element.append(child)
child.classList.add('js-plotly-plot')
child.data = 'plotly data'
child.layout = 'plotly layout'
const chartData = chartHelper.getChartData(element)
expect(chartData).to.eql({
data: 'plotly data',
layout: 'plotly layout'
})
})
it('getHtml returns valid html', async () => {
const options = {
data: 'plotly data',
layout: 'plotly layout'
}
const html = chartHelper.getHtml(options)
const doc = document.createElement('div')
doc.innerHTML = html
expect(doc.innerHTML).to.equal(html)
expect(doc.children).to.have.lengthOf(3)
expect(doc.children[0].src).to.includes('plotly-latest.js')
expect(doc.children[1].id).to.have.lengthOf(21)
expect(doc.children[2].innerHTML).to.includes(doc.children[1].id)
expect(doc.children[2].innerHTML)
.to.includes('Plotly.newPlot(el, "plotly data", "plotly layout"')
})
})

View File

@@ -269,6 +269,48 @@ describe('SQLite extensions', function () {
})
})
it('supports percentile', async function () {
const actual = await db.execute(`
CREATE TABLE s(x INTEGER);
INSERT INTO s VALUES (15), (20), (35), (40), (50);
SELECT
percentile(x, 5) p5,
percentile(x, 30) p30,
percentile(x, 40) p40,
percentile(x, 50) p50,
percentile(x, 100) p100
FROM s;
`)
expect(actual.values).to.eql({
p5: [16],
p30: [23],
p40: [29],
p50: [35],
p100: [50]
})
})
it('supports decimal', async function () {
const actual = await db.execute(`
select
decimal_add(decimal('0.1'), decimal('0.2')) "add",
decimal_sub(0.2, 0.1) sub,
decimal_mul(power(2, 69), 2) mul,
decimal_cmp(decimal('0.1'), 0.1) cmp_e,
decimal_cmp(decimal('0.1'), decimal('0.099999')) cmp_g,
decimal_cmp(decimal('0.199999'), decimal('0.2')) cmp_l
`)
expect(actual.values).to.eql({
add: ['0.3'],
sub: ['0.1'],
mul: ['1180591620717412000000'],
cmp_e: [0],
cmp_g: [1],
cmp_l: [-1]
})
})
it('supports FTS5', async function () {
const actual = await db.execute(`
CREATE VIRTUAL TABLE email USING fts5(sender, title, body, tokenize = 'porter ascii');
@@ -296,4 +338,96 @@ describe('SQLite extensions', function () {
sender: ['bar@localhost']
})
})
it('supports FTS3', async function () {
const actual = await db.execute(`
CREATE VIRTUAL TABLE email USING fts3(sender, title, body, tokenize = 'porter');
INSERT INTO email VALUES
(
'foo@localhost',
'fts3/4',
'FTS3 and FTS4 are SQLite virtual table modules that allows users to perform '
|| 'full-text searches on a set of documents.'
),
(
'bar@localhost',
'fts4',
'FTS5 is an SQLite virtual table module that provides full-text search '
|| 'functionality to database applications.'
);
SELECT sender
FROM email
WHERE body MATCH '("full-text" NOT document AND (functionality OR table))';
`)
expect(actual.values).to.eql({
sender: ['bar@localhost']
})
})
it('supports FTS4', async function () {
const actual = await db.execute(`
CREATE VIRTUAL TABLE email USING fts4(
sender, title, body, notindexed=sender, tokenize='simple'
);
INSERT INTO email VALUES
(
'foo@localhost',
'fts3/4',
'FTS3 and FTS4 are SQLite virtual table modules that allows users to perform '
|| 'full-text searches on a set of documents.'
),
(
'bar@localhost',
'fts4',
'FTS5 is an SQLite virtual table module that provides full-text search '
|| 'functionality to database applications.'
);
SELECT sender
FROM email
WHERE body MATCH '("full-text" NOT document AND (functionality OR table NOT modules))';
`)
expect(actual.values).to.eql({
sender: ['bar@localhost']
})
})
it('supports JSON1', async function () {
const actual = await db.execute(`
WITH input(filename) AS (
VALUES
('/etc/redis/redis.conf'),
('/run/redis/redis-server.pid'),
('/var/log/redis-server.log')
), tmp AS (
SELECT
filename,
'["' || replace(filename, '/', '", "') || '"]' as filename_array
FROM input
)
SELECT (
SELECT group_concat(ip.value, '/')
FROM json_each(filename_array) ip
WHERE ip.id <= p.id
) AS path
FROM tmp, json_each(filename_array) AS p
WHERE p.id > 1 -- because the filenames start with the separator
`)
expect(actual.values).to.eql({
path: [
'/etc',
'/etc/redis',
'/etc/redis/redis.conf',
'/run',
'/run/redis',
'/run/redis/redis-server.pid',
'/var',
'/var/log',
'/var/log/redis-server.log'
]
})
})
})

View File

@@ -59,6 +59,31 @@ describe('DataView.vue', () => {
expect(pivot.saveAsSvg.calledOnce).to.equal(true)
})
it('method saveAsHtml calls the same method of the current view component', async () => {
const wrapper = mount(DataView)
// Find chart and spy the method
const chart = wrapper.findComponent({ name: 'Chart' }).vm
sinon.spy(chart, 'saveAsHtml')
// Export to html
const htmlBtn = createWrapper(wrapper.findComponent({ name: 'htmlIcon' }).vm.$parent)
await htmlBtn.trigger('click')
expect(chart.saveAsHtml.calledOnce).to.equal(true)
// Switch to pivot
const pivotBtn = createWrapper(wrapper.findComponent({ name: 'pivotIcon' }).vm.$parent)
await pivotBtn.trigger('click')
// Find pivot and spy the method
const pivot = wrapper.findComponent({ name: 'pivot' }).vm
sinon.spy(pivot, 'saveAsHtml')
// Export to svg
await htmlBtn.trigger('click')
expect(pivot.saveAsHtml.calledOnce).to.equal(true)
})
it('shows alert when ClipboardItem is not supported', async () => {
const ClipboardItem = window.ClipboardItem
delete window.ClipboardItem

View File

@@ -5,6 +5,7 @@ import chartHelper from '@/lib/chartHelper'
import fIo from '@/lib/utils/fileIo'
import $ from 'jquery'
import sinon from 'sinon'
import pivotHelper from '@/views/Main/Workspace/Tabs/Tab/DataView/Pivot/pivotHelper'
describe('Pivot.vue', () => {
let container
@@ -271,6 +272,41 @@ describe('Pivot.vue', () => {
expect(chartComponent.saveAsSvg.called).to.equal(true)
})
it('saveAsHtml calls chart method if renderer is Custom Chart', async () => {
const wrapper = mount(Pivot, {
propsData: {
dataSources: {
item: ['foo', 'bar', 'bar', 'bar'],
year: [2021, 2021, 2020, 2020]
},
initOptions: {
rows: ['item'],
cols: ['year'],
colOrder: 'key_a_to_z',
rowOrder: 'key_a_to_z',
aggregatorName: 'Count',
vals: [],
renderer: $.pivotUtilities.renderers['Custom chart'],
rendererName: 'Custom chart',
rendererOptions: {
customChartOptions: {
data: [],
layout: {},
frames: []
}
}
}
},
attachTo: container
})
const chartComponent = wrapper.vm.pivotOptions.rendererOptions.customChartComponent
sinon.stub(chartComponent, 'saveAsHtml')
await wrapper.vm.saveAsHtml()
expect(chartComponent.saveAsHtml.called).to.equal(true)
})
it('saveAsPng calls chart method if renderer is Custom Chart', async () => {
const wrapper = mount(Pivot, {
propsData: {
@@ -333,6 +369,66 @@ describe('Pivot.vue', () => {
expect(chartHelper.getImageDataUrl.calledOnce).to.equal(true)
})
it('saveAsHtml - standart chart', async () => {
sinon.spy(chartHelper, 'getChartData')
sinon.spy(chartHelper, 'getHtml')
const wrapper = mount(Pivot, {
propsData: {
dataSources: {
item: ['foo', 'bar', 'bar', 'bar'],
year: [2021, 2021, 2020, 2020]
},
initOptions: {
rows: ['item'],
cols: ['year'],
colOrder: 'key_a_to_z',
rowOrder: 'key_a_to_z',
aggregatorName: 'Count',
vals: [],
renderer: $.pivotUtilities.renderers['Bar Chart'],
rendererName: 'Bar Chart'
}
},
attachTo: container
})
await wrapper.vm.saveAsHtml()
expect(chartHelper.getChartData.calledOnce).to.equal(true)
const chartData = await chartHelper.getChartData.returnValues[0]
expect(chartHelper.getHtml.calledOnceWith(chartData)).to.equal(true)
})
it('saveAsHtml - table', async () => {
sinon.stub(pivotHelper, 'getPivotHtml')
sinon.stub(fIo, 'exportToFile')
const wrapper = mount(Pivot, {
propsData: {
dataSources: {
item: ['foo', 'bar', 'bar', 'bar'],
year: [2021, 2021, 2020, 2020]
},
initOptions: {
rows: ['item'],
cols: ['year'],
colOrder: 'key_a_to_z',
rowOrder: 'key_a_to_z',
aggregatorName: 'Count',
vals: [],
renderer: $.pivotUtilities.renderers.Table,
rendererName: 'Table'
}
},
attachTo: container
})
await wrapper.vm.saveAsHtml()
expect(pivotHelper.getPivotHtml.calledOnce).to.equal(true)
const html = pivotHelper.getPivotHtml.returnValues[0]
expect(fIo.exportToFile.calledOnceWith(html, 'pivot.html', 'text/html')).to.equal(true)
})
it('saveAsPng - standart chart', async () => {
sinon.stub(chartHelper, 'getImageDataUrl').returns('standat chart data url')
sinon.stub(fIo, 'downloadFromUrl')

View File

@@ -1,5 +1,5 @@
import { expect } from 'chai'
import { _getDataSources, getPivotCanvas }
import { _getDataSources, getPivotCanvas, getPivotHtml }
from '@/views/Main/Workspace/Tabs/Tab/DataView/Pivot/pivotHelper'
describe('pivotHelper.js', () => {
@@ -63,4 +63,19 @@ describe('pivotHelper.js', () => {
expect(await getPivotCanvas(pivotOutput)).to.be.instanceof(HTMLCanvasElement)
})
it('getPivotHtml returns html with styles', async () => {
const pivotOutput = document.createElement('div')
pivotOutput.append('test')
const html = getPivotHtml(pivotOutput)
const doc = document.createElement('div')
doc.innerHTML = html
expect(doc.innerHTML).to.equal(html)
expect(doc.children).to.have.lengthOf(2)
expect(doc.children[0].tagName).to.equal('STYLE')
expect(doc.children[1].tagName).to.equal('DIV')
expect(doc.children[1].innerHTML).to.equal('test')
})
})