Build sqlitelua for scalar, aggregate & table-valued UDFs in Lua (#118)

* Update base Docker images

* Use performance.now() instead of Date.now() for time promise tests

* Build sqlitelua: user scalar, aggregate & table-valued functions in Lua

Dockerfile.test

@@ -3,7 +3,7 @@
 #     docker build -t sqliteviz/test -f Dockerfile.test .
 #
 
-FROM node:12
+FROM node:12.22-buster
 
 RUN set -ex; \
     apt update; \

lib/sql-js/README.md

@@ -45,6 +45,8 @@ SQLite 3rd party extensions included:
 1. [pivot_vtab][5] -- a pivot virtual table
 2. `pearson` correlation coefficient function extension from [sqlean][21]
    (which is part of [squib][20])
+3. [sqlitelua][22] -- a virtual table `luafunctions` which allows to define custom scalar,
+   aggregate and table-valued functions in Lua
 
 To ease the step to have working clone locally, the build is committed into
 the repository.
@@ -103,3 +105,4 @@ described in [this message from SQLite Forum][12]:
 [19]: https://github.com/lana-k/sqliteviz/blob/master/tests/lib/database/sqliteExtensions.spec.js
 [20]: https://github.com/mrwilson/squib/blob/master/pearson.c
 [21]: https://github.com/nalgeon/sqlean/blob/incubator/src/pearson.c
+[22]: https://github.com/kev82/sqlitelua

lib/sql-js/benchmark/Dockerfile

@@ -1,10 +1,8 @@
-FROM node:14-bullseye
+FROM node:20.14-bookworm
 
 RUN set -ex; \
-    echo 'deb http://deb.debian.org/debian unstable main' \
-        > /etc/apt/sources.list.d/unstable.list; \
     apt-get update; \
-    apt-get install -y -t unstable firefox; \
+    apt-get install -y firefox-esr; \
     apt-get install -y chromium
 
 WORKDIR /tmp/build

lib/sql-js/benchmark/result-analysis.ipynb

@@ -69,6 +69,19 @@
       ],
       "metadata": {}
     },
+    {
+      "cell_type": "code",
+      "source": [
+        "!du -b lib | head -n 2"
+      ],
+      "outputs": [],
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false,
+        "outputHidden": false,
+        "inputHidden": true
+      }
+    },
     {
       "cell_type": "code",
       "source": [
@@ -176,7 +189,7 @@
     },
     "language_info": {
       "name": "python",
-      "version": "3.10.7",
+      "version": "3.10.14",
       "mimetype": "text/x-python",
       "codemirror_mode": {
         "name": "ipython",

lib/sql-js/build.py

@@ -24,6 +24,7 @@ cflags = (
     # Compile-time optimisation
     '-Os',  # reduces the code size about in half comparing to -O2
     '-flto',
+    '-Isrc', '-Isrc/lua',
 )
 emflags = (
     # Base
@@ -61,6 +62,15 @@ def build(src: Path, dst: Path):
         '-c', src / 'extension-functions.c',
         '-o', out / 'extension-functions.o',
     ])
+    logging.info('Building LLVM bitcode for SQLite Lua extension')
+    subprocess.check_call([
+        'emcc',
+        *cflags,
+        '-shared',
+        *(src / 'lua').glob('*.c'),
+        *(src / 'sqlitelua').glob('*.c'),
+        '-o', out / 'sqlitelua.o',
+    ])
 
     logging.info('Building WASM from bitcode')
     subprocess.check_call([
@@ -68,6 +78,7 @@ def build(src: Path, dst: Path):
         *emflags,
         out / 'sqlite3.o',
         out / 'extension-functions.o',
+        out / 'sqlitelua.o',
         '-o', out / 'sql-wasm.js',
     ])
 

lib/sql-js/configure.py

@@ -1,7 +1,9 @@
 import logging
+import re
 import shutil
 import subprocess
 import sys
+import tarfile
 import zipfile
 from io import BytesIO
 from pathlib import Path
@@ -30,8 +32,14 @@ extension_urls = (
     # =====================
     ('https://github.com/jakethaw/pivot_vtab/raw/9323ef93/pivot_vtab.c', 'sqlite3_pivotvtab_init'),
     ('https://github.com/nalgeon/sqlean/raw/95e8d21a/src/pearson.c', 'sqlite3_pearson_init'),
+    # Third-party extension with own dependencies
+    # ===========================================
+    ('https://github.com/kev82/sqlitelua/raw/db479510/src/main.c', 'sqlite3_luafunctions_init'),
 )
 
+lua_url = 'http://www.lua.org/ftp/lua-5.3.5.tar.gz'
+sqlitelua_url = 'https://github.com/kev82/sqlitelua/archive/db479510.zip'
+
 sqljs_url = 'https://github.com/sql-js/sql.js/archive/refs/tags/v1.7.0.zip'
 
 
@@ -59,6 +67,38 @@ def _get_amalgamation(tgt: Path):
             shutil.copyfileobj(fr, fw)
 
 
+def _get_lua(tgt: Path):
+    # Library definitions from lua/Makefile
+    lib_str = '''
+        CORE_O=    lapi.o lcode.o lctype.o ldebug.o ldo.o ldump.o lfunc.o lgc.o llex.o \
+            lmem.o lobject.o lopcodes.o lparser.o lstate.o lstring.o ltable.o \
+            ltm.o lundump.o lvm.o lzio.o
+        LIB_O=    lauxlib.o lbaselib.o lbitlib.o lcorolib.o ldblib.o liolib.o \
+            lmathlib.o loslib.o lstrlib.o ltablib.o lutf8lib.o loadlib.o linit.o
+        LUA_O=    lua.o
+    '''
+    header_only_files = {'lprefix', 'luaconf', 'llimits', 'lualib'}
+    lib_names = set(re.findall(r'(\w+)\.o', lib_str)) | header_only_files
+
+    logging.info('Downloading and extracting Lua %s', lua_url)
+    archive = tarfile.open(fileobj=BytesIO(request.urlopen(lua_url).read()))
+    (tgt / 'lua').mkdir()
+    for tarinfo in archive:
+        tarpath = Path(tarinfo.name)
+        if tarpath.match('src/*') and tarpath.stem in lib_names:
+            with (tgt / 'lua' / tarpath.name).open('wb') as fw:
+                shutil.copyfileobj(archive.extractfile(tarinfo), fw)
+
+    logging.info('Downloading and extracting SQLite Lua extension %s', sqlitelua_url)
+    archive = zipfile.ZipFile(BytesIO(request.urlopen(sqlitelua_url).read()))
+    archive_root_dir = zipfile.Path(archive, archive.namelist()[0])
+    (tgt / 'sqlitelua').mkdir()
+    for zpath in (archive_root_dir / 'src').iterdir():
+        if zpath.name != 'main.c':
+            with zpath.open() as fr, (tgt / 'sqlitelua' / zpath.name).open('wb') as fw:
+                shutil.copyfileobj(fr, fw)
+
+
 def _get_contrib_functions(tgt: Path):
     request.urlretrieve(contrib_functions_url, tgt / 'extension-functions.c')
 
@@ -70,6 +110,7 @@ def _get_extensions(tgt: Path):
         for url, init_fn in extension_urls:
             logging.info('Downloading and appending to amalgamation %s', url)
             with request.urlopen(url) as resp:
+                f.write(b'\n')
                 shutil.copyfileobj(resp, f)
             init_functions.append(init_fn)
 
@@ -90,6 +131,7 @@ def _get_sqljs(tgt: Path):
 def configure(tgt: Path):
     _get_amalgamation(tgt)
     _get_contrib_functions(tgt)
+    _get_lua(tgt)
     _get_extensions(tgt)
     _get_sqljs(tgt)
 

tests/lib/database/sqliteExtensions.spec.js

@@ -455,4 +455,85 @@ describe('SQLite extensions', function () {
       xx: [1], xy: [0], xz: [1], yx: [0], yy: [1], yz: [1], zx: [1], zy: [1], zz: [1]
     })
   })
+
+  it('supports simple Lua functions', async function () {
+    const actual = await db.execute(`
+      INSERT INTO
+        luafunctions(name, src)
+      VALUES
+        ('lua_inline', 'return {"arg"}, {"rv"}, "simple", function(arg) return arg + 1 end'),
+        ('lua_full', '
+        local input = {"arg"}
+        local output = {"rv"}
+
+        local function func(x)
+          return math.sin(math.pi) + x
+        end
+
+        return input, output, "simple", func
+        ');
+
+      SELECT lua_inline(1), lua_full(1) - 1 < 0.000001;
+    `)
+    expect(actual.values).to.eql({ 'lua_inline(1)': [2], 'lua_full(1) - 1 < 0.000001': [1] })
+  })
+
+  it('supports aggregate Lua functions', async function () {
+    const actual = await db.execute(`
+      INSERT INTO
+        luafunctions(name, src)
+      VALUES
+        ('lua_sum', '
+        local inputs = {"item"}
+        local outputs = {"sum"}
+
+        local function func(item)
+          if aggregate_now(item) then
+            return item
+          end
+
+          local sum = 0
+          while true do
+            if aggregate_now(item) then
+              break
+            end
+            sum = sum + item
+            item = coroutine.yield()
+          end
+
+          return sum
+        end
+
+        return inputs, outputs, "aggregate", func
+      ');
+
+      SELECT SUM(value), lua_sum(value) FROM generate_series(1, 10);
+    `)
+    expect(actual.values).to.eql({ 'SUM(value)': [55], 'lua_sum(value)': [55] })
+  })
+
+  it('supports table-valued Lua functions', async function () {
+    const actual = await db.execute(`
+      INSERT INTO
+        luafunctions(name, src)
+      VALUES
+        ('lua_match', '
+        local inputs = {"pattern", "s"}
+        local outputs = {"idx", "elm"}
+
+        local function func(pattern, s)
+          local i = 1
+          for k in s:gmatch(pattern) do
+            coroutine.yield(i, k)
+            i = i + 1
+          end
+        end
+
+        return inputs, outputs, "table", func
+      ');
+
+      SELECT * FROM lua_match('%w+', 'hello world from Lua');
+    `)
+    expect(actual.values).to.eql({ idx: [1, 2, 3, 4], elm: ['hello', 'world', 'from', 'Lua'] })
+  })
 })

tests/lib/utils/time.spec.js

@@ -29,12 +29,12 @@ describe('time.js', () => {
   })
 
   it('sleep resolves after n ms', async () => {
-    let before = Date.now()
+    let before = performance.now()
     await time.sleep(10)
-    expect(Date.now() - before).to.be.least(10)
+    expect(performance.now() - before).to.be.least(10)
 
-    before = Date.now()
+    before = performance.now()
     await time.sleep(30)
-    expect(Date.now() - before).to.be.least(30)
+    expect(performance.now() - before).to.be.least(30)
   })
 })