sqliteviz/lib/sql-js/benchmark/result-analysis.ipynb

In [ ]:

import json
from pathlib import Path

import pandas
from IPython.display import display, IFrame, Markdown
from plotly import express

In [ ]:

render_format = 'svg'
benchmark_path = Path()
df_dict = {}
stat_dict = {}
for p in benchmark_path.glob('build-*-result.json'):
    build_name = p.stem.split('-', 2)[1]
    for browser_data in json.loads(p.read_text()):
        browser_name = f'{browser_data["browser"]["name"]} {browser_data["browser"]["major"]}'
        browser_name = browser_name.lower().replace('chrome headless', 'chromium')
        for result in (r for i, r in browser_data['result'].items() if i.isdigit()):
            key = build_name, browser_name, result['name']
            df_dict[key] = result['stats']['sample']
            stat_dict[key] = result['stats']

min_sample_size = min(len(v) for v in df_dict.values())
df_dict = {k: v[:min_sample_size] for k, v in df_dict.items()}
            
wide_df = pandas.DataFrame(df_dict).reset_index()
df = pandas.melt(
    wide_df, 
    id_vars='index', 
    var_name=['build', 'browser', 'test'], 
    value_name='duration',
)
df = df.rename(columns={'index': 'run'})
df.sort_values(['build', 'run'], inplace=True)

sql.js build comparison

<style> @page { size: 215mm 297mm; margin: 0; } </style>

In [ ]:

!du -b lib | head -n 2

In [ ]:

stat_df = pandas.DataFrame(stat_dict)
stat_df = stat_df.loc[['mean', 'rme']].transpose()
stat_df.index = stat_df.index.set_names(['build', 'browser', 'test'])
stat_df = stat_df.reset_index().sort_values(['test', 'browser'], ascending=False)
for index, row in stat_df.iterrows():
    print('\t'.join([
        row['build'],
        row['browser'],
        row['test'],
        f'{row["mean"]:.2f} s ± {row["rme"]:.1f}%'
    ]))

In [ ]:

fig = express.box(df, x='browser', y='duration', points='all', color='build', facet_row='test')
fig.update_yaxes(matches=None)
fig.show(render_format)

In [ ]:

fig = express.line(
    df, x='run', y='duration', color='build', facet_col='browser', facet_row='test'
)
fig.update_yaxes(matches=None)
fig.show(render_format)

In [ ]:

plot_df = df.groupby(['browser', 'build', 'test'])['duration'].mean().reset_index()
plot_df['pct'] = (
    plot_df
    .groupby(['browser', 'test'])['duration']
    .pct_change()
    .fillna('')
    .map(lambda v: '{:.2%}'.format(v) if v else v)
)
fig = express.bar(
    plot_df, 
    x='browser', 
    y='duration', 
    color='build', 
    text='pct', 
    barmode='overlay', 
    facet_row='test',
)
fig.update_yaxes(matches=None)
fig.show(render_format)

In [ ]:

for p in sorted(benchmark_path.glob('build-*.svg')):
    display(Markdown(p.stem))
    display(IFrame(p, 640, 480))