SQLite WebAssembly micro-benchmark and new build with a memory leak fix (#70)

lib/sql-js/benchmark/.gitignore

@@ -0,0 +1,4 @@
+/lib/build-*
+/lib/dist
+/build-*-result.json
+/sample.csv

lib/sql-js/benchmark/Dockerfile

@@ -0,0 +1,19 @@
+FROM node:12-buster
+
+RUN set -ex; \
+    echo 'deb http://deb.debian.org/debian unstable main' \
+        > /etc/apt/sources.list.d/unstable.list; \
+    apt-get update; \
+    apt-get install -y -t unstable firefox; \
+    apt-get install -y chromium
+
+WORKDIR /tmp/build
+
+COPY package.json ./
+COPY lib/dist lib/dist
+COPY lib/package.json lib/package.json
+RUN npm install
+
+COPY . .
+
+CMD npm run benchmark

lib/sql-js/benchmark/README.md

@@ -0,0 +1,14 @@
+# SQLite WebAssembly build micro-benchmark
+
+This directory contains a micro-benchmark for evaluating SQLite
+WebAssembly builds performance on typical SQL queries, run from
+`make.sh` script. It can also serve as a smoke test.
+
+The benchmark operates on a set of SQLite WebAssembly builds expected
+in `lib/build-$NAME` directories each containing `sql-wasm.js` and
+`sql-wasm.wasm`. Then it creates a Docker image for each, and runs
+the benchmark in Firefox and Chromium using Karma in the container.
+
+After successful run, the benchmark result of each build is contained
+in `build-$NAME-result.json`. The JSON result files can be analysed
+using `result-analysis.ipynb` Jupyter notebook.

lib/sql-js/benchmark/karma.conf.js

@@ -0,0 +1,52 @@
+module.exports = function (config) {
+  const timeout = 15 * 60 * 1000
+  config.set({
+
+    frameworks: ['mocha'],
+
+    files: [
+      'suite.js',
+      { pattern: 'node_modules/sql.js/dist/sql-wasm.wasm', served: true, included: false },
+      { pattern: 'sample.csv', served: true, included: false }
+    ],
+
+    reporters: ['progress', 'json-to-file'],
+
+    singleRun: true,
+
+    customLaunchers: {
+      ChromiumHeadlessNoSandbox: { base: 'ChromiumHeadless', flags: ['--no-sandbox'] }
+    },
+    browsers: ['ChromiumHeadlessNoSandbox', 'FirefoxHeadless'],
+    concurrency: 1,
+
+    browserDisconnectTimeout: timeout,
+    browserNoActivityTimeout: timeout,
+    captureTimeout: timeout,
+    browserSocketTimeout: timeout,
+    pingTimeout: timeout,
+    client: {
+      captureConsole: true,
+      mocha: { timeout: timeout }
+    },
+
+    logLevel: config.LOG_INFO,
+    browserConsoleLogOptions: { terminal: true, level: config.LOG_INFO },
+
+    preprocessors: { 'suite.js': [ 'webpack' ] },
+    webpack: {
+      mode: 'development',
+      module: {
+        noParse: [ __dirname + '/node_modules/benchmark/benchmark.js' ]
+      },
+      node: { fs: 'empty' }
+    },
+
+    proxies: {
+      '/sql-wasm.wasm': '/base/node_modules/sql.js/dist/sql-wasm.wasm'
+    },
+
+    jsonToFileReporter: { outputPath: '.', fileName: 'suite-result.json' }
+
+  })
+}

lib/sql-js/benchmark/lib/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "sql.js",
+  "main": "./dist/sql-wasm.js",
+  "private": true
+}

lib/sql-js/benchmark/make.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -e
+
+if [ ! -f sample.csv ]; then
+  wget --header="accept-encoding: gzip" -q -O- \
+    https://github.com/plotly/datasets/raw/547090bd/wellspublic.csv \
+    | gunzip -c > sample.csv
+fi
+
+for d in lib/build-* ; do
+  rm -r lib/dist || true
+  cp -r $d lib/dist
+
+  name=$(basename $d)
+  docker build -t sqliteviz/sqljs-benchmark:$name .
+  docker rm sqljs-benchmark-$name 2> /dev/null || true
+  docker run -it --name sqljs-benchmark-$name sqliteviz/sqljs-benchmark:$name
+  docker cp sqljs-benchmark-$name:/tmp/build/suite-result.json ${name}-result.json
+  docker rm sqljs-benchmark-$name
+done
+
+rm -r lib/dist

lib/sql-js/benchmark/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "sqlite-webassembly-microbenchmark",
+  "private": true,
+  "dependencies": {
+    "@babel/core" : "^7.14.8",
+    "babel-loader": "^8.2.2",
+    "benchmark": "^2.1.4",
+    "lodash": "^4.17.4",
+    "papaparse": "^5.3.1",
+    "mocha": "^9.0.3",
+    "karma": "^6.3.4",
+    "karma-chrome-launcher": "^3.1.0",
+    "karma-firefox-launcher": "^2.1.1",
+    "karma-json-to-file-reporter" : "^1.0.1",
+    "karma-mocha": "^2.0.1",
+    "karma-webpack": "^4.0.2",
+    "webpack": "^4.46.0",
+    "sql.js": "file:./lib"
+  },
+  "scripts": {
+    "benchmark": "karma start karma.conf.js"
+  }
+}

lib/sql-js/benchmark/suite.js

@@ -0,0 +1,140 @@
+import benchmark from 'benchmark'
+import initSqlJs from 'sql.js'
+import lodash from 'lodash'
+import Papa from 'papaparse'
+import useragent from 'ua-parser-js'
+
+
+describe('SQLite build benchmark', function () {
+  let parsedCsv
+  let sqlModule
+  let selectDb
+
+  before(async function () {
+    parsedCsv = await parseCsv('http://localhost:9876/base/sample.csv')
+    sqlModule = await initSqlJs()
+
+    selectDb = new sqlModule.Database()
+    importToTable(selectDb, parsedCsv)
+  })
+
+  function benchmarkImport () {
+    const db = new sqlModule.Database()
+    try {
+      importToTable(db, parsedCsv)
+    } finally {
+      db.close()
+    }
+  }
+
+  function benchmarkSelect () {
+    const result = selectDb.exec(`
+      SELECT county, AVG(avg_depth) avg_depth_c
+      FROM (
+        SELECT s.county, s.town, COUNT(*) cnt, AVG(s.DrilledDepth) avg_depth
+        FROM csv_import s
+        JOIN csv_import USING(hole)
+        WHERE s.town IS NOT NULL
+        GROUP BY 1, 2
+        ORDER BY 4 DESC
+      )
+      GROUP BY 1
+      ORDER BY 2 DESC
+    `)
+    console.assert(result.values.length == 56, 'Unexpected size of result set')
+  }
+
+  it('run', async function () {
+    const suite = createSuite()
+    suite.add('import', { initCount: 3, minSamples: 50, fn: benchmarkImport })
+    suite.add('select', { initCount: 3, minSamples: 50, fn: benchmarkSelect })
+    await run(suite)
+  })
+
+})
+
+
+function importToTable (db, parsedCsv, chunkSize = 1024) {
+  const columnListString = parsedCsv.meta.fields.join(', ')
+  db.exec(`CREATE TABLE csv_import(${columnListString})`)
+
+  const params = parsedCsv.meta.fields.map(name => '?').join(', ')
+  const insertStmt = db.prepare(`INSERT INTO csv_import VALUES(${params})`)
+  chunkArray(parsedCsv.data, chunkSize).map(function (chunk) {
+    db.exec('BEGIN')
+    chunk.map(row => insertStmt.run(Object.values(row)))
+    db.exec('COMMIT')
+  })
+}
+
+
+class PromiseWrapper {
+  constructor() {
+    this.promise = new Promise((resolve, reject) => {
+      this.reject = reject
+      this.resolve = resolve
+    })
+  }
+}
+
+function parseCsv(url) {
+  return new Promise((resolve, reject) => {
+    Papa.parse(url, {
+      header: true,
+      download: true,
+      skipEmptyLines: 'greedy',
+      complete: results => resolve(results),
+      error: (error, file) => reject(error)
+    })
+  })
+}
+
+function chunkArray (arr, size) {
+  return arr.reduce(function (result, value, index) {
+    const chunkIndex = Math.floor(index / size)
+
+    if(!(chunkIndex in result)) {
+      result[chunkIndex] = []
+    }
+    result[chunkIndex].push(value)
+
+    return result
+  }, [])
+}
+
+
+function createSuite () {
+  // Combined workaround from:
+  // - https://github.com/bestiejs/benchmark.js/issues/106
+  // - https://github.com/bestiejs/benchmark.js/issues/237
+
+  // Benchmark could not pick up lodash otherwise
+  const bm = benchmark.runInContext({ _: lodash })
+  // Avoid `ReferenceError: Benchmark is not defined` error because Benchmark is assumed
+  // to be in window
+  window.Benchmark = bm
+
+  return new bm.Suite()
+}
+
+function run (suite) {
+  const suiteResult = new PromiseWrapper()
+  suite
+    .on('cycle', function (event) {
+      console.info(String(event.target))
+    })
+    .on('complete', function () {
+      console.log(JSON.stringify({
+        browser: useragent(navigator.userAgent).browser,
+        result: this.filter('successful')
+      }))
+      suiteResult.resolve()
+    })
+    .on('error', function (event) {
+      console.error('Benchmark failed', String(event.target))
+      suiteResult.reject()
+    })
+    .run({async: true})
+
+  return suiteResult.promise
+}