SQLite 3.41.0 and pearson correlation extension function (#106)

* Build SQLite 3.41.0

* Update pivot_vtab

* Add Pearson correlation coefficient function extension, build

* Add an easy way to running test locally without Nodejs

* Use RSS sum to pick top2 processes for the report

* Try previous Ubuntu LTS as a workaround for Firefox worker not starting

.github/workflows/test.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   test:
     name: Run tests
-    runs-on: ubuntu-latest    
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
     - name: Use Node.js

Dockerfile.test

@@ -0,0 +1,24 @@
+# An easy way to run tests locally without Nodejs installed:
+#
+#     docker build -t sqliteviz/test -f Dockerfile.test .
+#
+
+FROM node:12
+
+RUN set -ex; \
+    apt update; \
+    apt install -y chromium firefox-esr; \
+    npm install -g npm@7
+
+WORKDIR /tmp/build
+
+COPY package.json package-lock.json ./
+COPY lib lib
+RUN npm install
+
+COPY . .
+
+RUN set -ex; \
+    sed -i 's/browsers: \[.*\],/browsers: ['"'FirefoxHeadlessTouch'"'],/' karma.conf.js
+
+RUN npm run lint -- --no-fix && npm run test

lib/sql-js/README.md

@@ -43,6 +43,8 @@ SQLite [miscellaneous extensions][3] included:
 SQLite 3rd party extensions included:
 
 1. [pivot_vtab][5] -- a pivot virtual table
+2. `pearson` correlation coefficient function extension from [sqlean][21]
+   (which is part of [squib][20])
 
 To ease the step to have working clone locally, the build is committed into
 the repository.
@@ -99,3 +101,5 @@ described in [this message from SQLite Forum][12]:
 [17]: https://sqlite.org/contrib/
 [18]: https://sqlite.org/contrib//download/extension-functions.c?get=25
 [19]: https://github.com/lana-k/sqliteviz/blob/master/tests/lib/database/sqliteExtensions.spec.js
+[20]: https://github.com/mrwilson/squib/blob/master/pearson.c
+[21]: https://github.com/nalgeon/sqlean/blob/incubator/src/pearson.c

lib/sql-js/benchmark/procpath/top2_rss.sql

@@ -2,7 +2,7 @@ WITH one_time_pid_condition AS (
     SELECT stat_pid
     FROM record
     GROUP BY 1
-    ORDER BY MAX(stat_rss) DESC
+    ORDER BY SUM(stat_rss) DESC
     LIMIT 2
 )
 SELECT

lib/sql-js/configure.py

@@ -8,7 +8,7 @@ from pathlib import Path
 from urllib import request
 
 
-amalgamation_url = 'https://sqlite.org/2022/sqlite-amalgamation-3390300.zip'
+amalgamation_url = 'https://sqlite.org/2023/sqlite-amalgamation-3410000.zip'
 
 # Extension-functions
 # ===================
@@ -28,7 +28,8 @@ extension_urls = (
     ('https://sqlite.org/src/raw/09f967dc?at=decimal.c', 'sqlite3_decimal_init'),
     # Third-party extension
     # =====================
-    ('https://github.com/jakethaw/pivot_vtab/raw/08ab0797/pivot_vtab.c', 'sqlite3_pivotvtab_init'),
+    ('https://github.com/jakethaw/pivot_vtab/raw/9323ef93/pivot_vtab.c', 'sqlite3_pivotvtab_init'),
+    ('https://github.com/nalgeon/sqlean/raw/95e8d21a/src/pearson.c', 'sqlite3_pearson_init'),
 )
 
 sqljs_url = 'https://github.com/sql-js/sql.js/archive/refs/tags/v1.7.0.zip'

tests/lib/database/sqliteExtensions.spec.js

@@ -160,39 +160,39 @@ describe('SQLite extensions', function () {
   it('supports transitive_closure', async function () {
     const actual = await db.execute(`
       CREATE TABLE node(
-      node_id   INTEGER NOT NULL PRIMARY KEY,
+        node_id   INTEGER NOT NULL PRIMARY KEY,
-      parent_id INTEGER,
+        parent_id INTEGER,
-      name      VARCHAR(127),
+        name      VARCHAR(127),
-      FOREIGN KEY (parent_id) REFERENCES node(node_id)
+        FOREIGN KEY (parent_id) REFERENCES node(node_id)
-    );
-    CREATE INDEX node_parent_id_idx ON node(parent_id);
-
-    CREATE VIRTUAL TABLE node_closure USING transitive_closure(
-        tablename    = "node",
-        idcolumn     = "node_id",
-        parentcolumn = "parent_id"
       );
+      CREATE INDEX node_parent_id_idx ON node(parent_id);
 
-    INSERT INTO node VALUES
+      CREATE VIRTUAL TABLE node_closure USING transitive_closure(
-      (1,  NULL, 'tests'),
+          tablename    = "node",
-      (2,  1,    'lib'),
+          idcolumn     = "node_id",
-      (3,  2,    'database'),
+          parentcolumn = "parent_id"
-      (4,  2,    'utils'),
+        );
-      (5,  2,    'storedQueries.spec.js'),
-      (6,  3,    '_sql.spec.js'),
-      (7,  3,    '_statements.spec.js'),
-      (8,  3,    'database.spec.js'),
-      (9,  3,    'sqliteExtensions.spec.js'),
-      (10, 4,    'fileIo.spec.js'),
-      (11, 4,    'time.spec.js');
 
-      SELECT name
+      INSERT INTO node VALUES
-      FROM node
+        (1,  NULL, 'tests'),
-      WHERE node_id IN (
+        (2,  1,    'lib'),
-        SELECT nc.id
+        (3,  2,    'database'),
-        FROM node_closure AS nc
+        (4,  2,    'utils'),
-        WHERE nc.root = 2 AND nc.depth = 2
+        (5,  2,    'storedQueries.spec.js'),
-      );
+        (6,  3,    '_sql.spec.js'),
+        (7,  3,    '_statements.spec.js'),
+        (8,  3,    'database.spec.js'),
+        (9,  3,    'sqliteExtensions.spec.js'),
+        (10, 4,    'fileIo.spec.js'),
+        (11, 4,    'time.spec.js');
+
+        SELECT name
+        FROM node
+        WHERE node_id IN (
+          SELECT nc.id
+          FROM node_closure AS nc
+          WHERE nc.root = 2 AND nc.depth = 2
+        );
     `)
     expect(actual.values).to.eql({
       name: [
@@ -293,7 +293,7 @@ describe('SQLite extensions', function () {
 
   it('supports decimal', async function () {
     const actual = await db.execute(`
-      select
+      SELECT
         decimal_add(decimal('0.1'), decimal('0.2')) "add",
         decimal_sub(0.2, 0.1) sub,
         decimal_mul(power(2, 69), 2) mul,
@@ -430,4 +430,29 @@ describe('SQLite extensions', function () {
       ]
     })
   })
+
+  it('supports pearson', async function () {
+    const actual = await db.execute(`
+      CREATE TABLE dataset(x REAL, y REAL, z REAL);
+      INSERT INTO dataset VALUES
+        (5,3,3.2),  (5,6,4.3),  (5,9,5.4),
+        (10,3,4),   (10,6,3.8), (10,9,3.6),
+        (15,3,4.8), (15,6,4),   (15,9,3.5);
+
+      SELECT
+        pearson(x, x) xx,
+        pearson(x, y) xy,
+        abs(-0.12666 - pearson(x, z)) < 0.00001 xz,
+        pearson(y, x) yx,
+        pearson(y, y) yy,
+        abs(0.10555 - pearson(y, z))  < 0.00001 yz,
+        abs(-0.12666 - pearson(z, x)) < 0.00001 zx,
+        abs(0.10555 - pearson(z, y))  < 0.00001 zy,
+        pearson(z, z) zz
+      FROM dataset;
+    `)
+    expect(actual.values).to.eql({
+      xx: [1], xy: [0], xz: [1], yx: [0], yy: [1], yz: [1], zx: [1], zy: [1], zz: [1]
+    })
+  })
 })