Compile with the latest ngtcp2

Rewrite docker file so that:

- avoid k8s debian-base
- build h2load as statically as possible

README.rst

@@ -16,6 +16,35 @@ An experimental high level C++ library is also available.
 We have Python bindings of this library, but we do not have full
 code coverage yet.
 
+Running h2load against HTTP/3 server
+------------------------------------
+
+In order to build h2load with HTTP/3 support, you have to build
+ngtcp2, nghttp3 and my patched OpenSSL.
+https://github.com/ngtcp2/ngtcp2#build-from-git describes how to build
+these three software.
+
+To run h2load against HTTP/3 server, specify h3 ALPN with
+``--npn-list`` option like so:
+
+.. code-block:: text
+
+    $ h2load --npn-list h3 https://127.0.0.1:4433
+
+You can use Dockerfile to skip the tedious build steps to manually
+pull and build dependencies.  In order to build Docker image, do this:
+
+.. code-block:: text
+
+    $ cd docker
+    $ docker build -t nghttp2-quic .
+
+Run h2load:
+
+.. code-block:: text
+
+    $ docker run --rm -it --network=host nghttp2-quic /usr/local/bin/h2load --npn-list h3 https://127.0.0.1:4433
+
 Development Status
 ------------------
 

configure.ac

@@ -454,6 +454,29 @@ if test "x${request_libcares}" = "xyes" &&
   AC_MSG_ERROR([libcares was requested (--with-libcares) but not found])
 fi
 
+# ngtcp2 (for src)
+PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.0.0], [have_libngtcp2=yes],
+                  [have_libngtcp2=no])
+if test "x${have_libngtcp2}" = "xno"; then
+  AC_MSG_NOTICE($LIBNGTCP2_PKG_ERRORS)
+fi
+
+# ngtcp2_crypto_openssl (for src)
+PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_OPENSSL],
+                  [libngtcp2_crypto_openssl >= 0.0.0],
+                  [have_libngtcp2_crypto_openssl=yes],
+                  [have_libngtcp2_crypto_openssl=no])
+if test "x${have_libngtcp2_crypto_openssl}" = "xno"; then
+  AC_MSG_NOTICE($LIBNGTCP2_CRYPTO_OPENSSL_PKG_ERRORS)
+fi
+
+# nghttp3 (for src)
+PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.0.0], [have_libnghttp3=yes],
+                  [have_libnghttp3=no])
+if test "x${have_libnghttp3}" = "xno"; then
+  AC_MSG_NOTICE($LIBNGHTTP3_PKT_ERRORS)
+fi
+
 # libevent_openssl (for examples)
 # 2.0.8 is required because we use evconnlistener_set_error_cb()
 have_libevent_openssl=no
@@ -1011,6 +1034,9 @@ AC_MSG_NOTICE([summary of build options:
       Libxml2:        ${have_libxml2} (CFLAGS='${LIBXML2_CFLAGS}' LIBS='${LIBXML2_LIBS}')
       Libev:          ${have_libev} (CFLAGS='${LIBEV_CFLAGS}' LIBS='${LIBEV_LIBS}')
       Libc-ares:      ${have_libcares} (CFLAGS='${LIBCARES_CFLAGS}' LIBS='${LIBCARES_LIBS}')
+      libngtcp2:      ${have_libngtcp2} (CFLAGS='${LIBNGTCP2_CFLAGS}' LIBS='${LIBNGTCP2_LIBS}')
+      libngtcp2_crypto_openssl: ${have_libngtcp2_crypto_openssl} (CFLAGS='${LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS}' LIBS='${LIBNGTCP2_CRYPTO_OPENSSL_LIBS}')
+      libnghttp3:     ${have_libnghttp3} (CFLAGS='${LIBNGHTTP3_CFLAGS}' LIBS='${LIBNGHTTP3_LIBS}')
       Libevent(SSL):  ${have_libevent_openssl} (CFLAGS='${LIBEVENT_OPENSSL_CFLAGS}' LIBS='${LIBEVENT_OPENSSL_LIBS}')
       Jansson:        ${have_jansson} (CFLAGS='${JANSSON_CFLAGS}' LIBS='${JANSSON_LIBS}')
       Jemalloc:       ${have_jemalloc} (CFLAGS='${JEMALLOC_CFLAGS}' LIBS='${JEMALLOC_LIBS}')

docker/Dockerfile

@@ -0,0 +1,36 @@
+FROM debian:10 as build
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        git g++ make binutils autoconf automake autotools-dev libtool \
+        pkg-config \
+        zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison && \
+    git clone --depth 1 -b OpenSSL_1_1_1g-quic-draft-33 https://github.com/tatsuhiro-t/openssl && \
+    cd openssl && ./config enable-tls1_3 --openssldir=/etc/ssl && make -j$(nproc) && make install_sw && cd .. && rm -rf openssl && \
+    git clone --depth 1 https://github.com/ngtcp2/nghttp3 && \
+    cd nghttp3 && autoreconf -i && \
+    ./configure --enable-lib-only && \
+    make -j$(nproc) && make install-strip && cd .. && rm -rf nghttp3 && \
+    git clone --depth 1 https://github.com/ngtcp2/ngtcp2 && \
+    cd ngtcp2 && autoreconf -i && \
+    ./configure && \
+    make -j$(nproc) && make install-strip && cd .. && rm -rf ngtcp2 && \
+    git clone --depth 1 -b quic https://github.com/nghttp2/nghttp2.git && \
+    cd nghttp2 && \
+    git submodule update --init && autoreconf -i && \
+    ./configure --disable-examples --disable-hpack-tools \
+        --disable-python-bindings --with-mruby --with-neverbleed \
+        LIBTOOL_LDFLAGS="-static-libtool-libs" \
+        LIBS="-ldl -pthread" \
+        OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a" \
+        LIBEV_LIBS="-l:libev.a" \
+        JEMALLOC_LIBS="-l:libjemalloc.a" \
+        LIBCARES_LIBS="-l:libcares.a" \
+        ZLIB_LIBS="-l:libz.a" && \
+    make -j$(nproc) install-strip
+
+FROM gcr.io/distroless/cc-debian10
+
+COPY --from=build /usr/local/bin/h2load /usr/local/bin/
+
+ENTRYPOINT ["/usr/local/bin/h2load"]

src/Makefile.am

@@ -46,6 +46,9 @@ AM_CPPFLAGS = \
 	@LIBEV_CFLAGS@ \
 	@OPENSSL_CFLAGS@ \
 	@LIBCARES_CFLAGS@ \
+	@LIBNGHTTP3_CFLAGS@ \
+	@LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS@ \
+	@LIBNGTCP2_CFLAGS@ \
 	@JANSSON_CFLAGS@ \
 	@ZLIB_CFLAGS@ \
 	@DEFS@
@@ -59,6 +62,9 @@ LDADD = $(top_builddir)/lib/libnghttp2.la \
 	@LIBEV_LIBS@ \
 	@OPENSSL_LIBS@ \
 	@LIBCARES_LIBS@ \
+	@LIBNGHTTP3_LIBS@ \
+	@LIBNGTCP2_CRYPTO_OPENSSL_LIBS@ \
+	@LIBNGTCP2_LIBS@ \
 	@SYSTEMD_LIBS@ \
 	@JANSSON_LIBS@ \
 	@ZLIB_LIBS@ \
@@ -97,7 +103,10 @@ h2load_SOURCES = util.cc util.h \
 	tls.cc tls.h \
 	h2load_session.h \
 	h2load_http2_session.cc h2load_http2_session.h \
-	h2load_http1_session.cc h2load_http1_session.h
+	h2load_http1_session.cc h2load_http1_session.h \
+	h2load_http3_session.cc h2load_http3_session.h \
+	h2load_quic.cc h2load_quic.h \
+	quic.cc quic.h
 
 NGHTTPX_SRCS = \
 	util.cc util.h http2.cc http2.h timegm.c timegm.h base64.h \

src/h2load.cc

@@ -34,6 +34,8 @@
 #ifdef HAVE_FCNTL_H
 #  include <fcntl.h>
 #endif // HAVE_FCNTL_H
+#include <sys/mman.h>
+#include <netinet/udp.h>
 
 #include <cstdio>
 #include <cassert>
@@ -48,10 +50,14 @@
 
 #include <openssl/err.h>
 
+#include <ngtcp2/ngtcp2.h>
+
 #include "url-parser/url_parser.h"
 
 #include "h2load_http1_session.h"
 #include "h2load_http2_session.h"
+#include "h2load_http3_session.h"
+#include "h2load_quic.h"
 #include "tls.h"
 #include "http2.h"
 #include "util.h"
@@ -71,9 +77,22 @@ bool recorded(const std::chrono::steady_clock::time_point &t) {
 }
 } // namespace
 
+namespace {
+std::ofstream keylog_file;
+void keylog_callback(const SSL *ssl, const char *line) {
+  keylog_file.write(line, strlen(line));
+  keylog_file.put('\n');
+  keylog_file.flush();
+}
+} // namespace
+
 Config::Config()
     : ciphers(tls::DEFAULT_CIPHER_LIST),
+      tls13_ciphers("TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_"
+                    "CHACHA20_POLY1305_SHA256:TLS_AES_128_CCM_SHA256"),
+      groups("X25519:P-256:P-384:P-521"),
       data_length(-1),
+      data(nullptr),
       addrs(nullptr),
       nreqs(1),
       nclients(1),
@@ -92,6 +111,7 @@ Config::Config()
       encoder_header_table_size(4_k),
       data_fd(-1),
       log_fd(-1),
+      qlog_file_base(),
       port(0),
       default_port(0),
       connect_to_port(0),
@@ -99,7 +119,8 @@ Config::Config()
       timing_script(false),
       base_uri_unix(false),
       unix_addr{},
-      rps(0.) {}
+      rps(0.),
+      no_udp_gso(false) {}
 
 Config::~Config() {
   if (addrs) {
@@ -119,6 +140,10 @@ bool Config::is_rate_mode() const { return (this->rate != 0); }
 bool Config::is_timing_based_mode() const { return (this->duration > 0); }
 bool Config::has_base_uri() const { return (!this->base_uri.empty()); }
 bool Config::rps_enabled() const { return this->rps > 0.0; }
+bool Config::is_quic() const {
+  return !npn_list.empty() &&
+         (npn_list[0] == NGHTTP3_ALPN_H3 || npn_list[0] == "\x5h3-29");
+}
 Config config;
 
 namespace {
@@ -138,7 +163,9 @@ Stats::Stats(size_t req_todo, size_t nclients)
       bytes_head(0),
       bytes_head_decomp(0),
       bytes_body(0),
-      status() {}
+      status(),
+      udp_dgram_recv(0),
+      udp_dgram_sent(0) {}
 
 Stream::Stream() : req_stat{}, status_success(-1) {}
 
@@ -195,8 +222,7 @@ void readcb(struct ev_loop *loop, ev_io *w, int revents) {
     delete client;
     return;
   }
-  writecb(loop, &client->wev, revents);
-  // client->disconnect() and client->fail() may be called
+  client->signal_write();
 }
 } // namespace
 
@@ -409,6 +435,7 @@ Client::Client(uint32_t id, Worker *worker, size_t req_todo)
       cstat{},
       worker(worker),
       ssl(nullptr),
+      quic{},
       next_addr(config.addrs),
       current_addr(nullptr),
       reqidx(0),
@@ -420,6 +447,7 @@ Client::Client(uint32_t id, Worker *worker, size_t req_todo)
       req_done(0),
       id(id),
       fd(-1),
+      local_addr{},
       new_connection_requested(false),
       final(false),
       rps_duration_started(0),
@@ -449,11 +477,18 @@ Client::Client(uint32_t id, Worker *worker, size_t req_todo)
 
   ev_timer_init(&rps_watcher, rps_cb, 0., 0.);
   rps_watcher.data = this;
+
+  ev_timer_init(&quic.pkt_timer, quic_pkt_timeout_cb, 0., 0.);
+  quic.pkt_timer.data = this;
 }
 
 Client::~Client() {
   disconnect();
 
+  if (config.is_quic()) {
+    quic_free();
+  }
+
   if (ssl) {
     SSL_free(ssl);
   }
@@ -466,26 +501,57 @@ int Client::do_read() { return readfn(*this); }
 int Client::do_write() { return writefn(*this); }
 
 int Client::make_socket(addrinfo *addr) {
-  fd = util::create_nonblock_socket(addr->ai_family);
-  if (fd == -1) {
-    return -1;
-  }
-  if (config.scheme == "https") {
-    if (!ssl) {
-      ssl = SSL_new(worker->ssl_ctx);
+  int rv;
+
+  if (config.is_quic()) {
+    fd = util::create_nonblock_udp_socket(addr->ai_family);
+    if (fd == -1) {
+      return -1;
     }
 
-    auto config = worker->config;
-
-    if (!util::numeric_host(config->host.c_str())) {
-      SSL_set_tlsext_host_name(ssl, config->host.c_str());
+    rv = util::bind_any_addr_udp(fd, addr->ai_family);
+    if (rv != 0) {
+      close(fd);
+      fd = -1;
+      return -1;
     }
 
-    SSL_set_fd(ssl, fd);
-    SSL_set_connect_state(ssl);
+    socklen_t addrlen = sizeof(local_addr.su.storage);
+    rv = getsockname(fd, &local_addr.su.sa, &addrlen);
+    if (rv == -1) {
+      return -1;
+    }
+    local_addr.len = addrlen;
+
+    if (quic_init(&local_addr.su.sa, local_addr.len, addr->ai_addr,
+                  addr->ai_addrlen) != 0) {
+      std::cerr << "quic_init failed" << std::endl;
+      return -1;
+    }
+  } else {
+    fd = util::create_nonblock_socket(addr->ai_family);
+    if (fd == -1) {
+      return -1;
+    }
+    if (config.scheme == "https") {
+      if (!ssl) {
+        ssl = SSL_new(worker->ssl_ctx);
+      }
+
+      SSL_set_fd(ssl, fd);
+      SSL_set_connect_state(ssl);
+    }
   }
 
-  auto rv = ::connect(fd, addr->ai_addr, addr->ai_addrlen);
+  if (ssl && !util::numeric_host(config.host.c_str())) {
+    SSL_set_tlsext_host_name(ssl, config.host.c_str());
+  }
+
+  if (config.is_quic()) {
+    return 0;
+  }
+
+  rv = ::connect(fd, addr->ai_addr, addr->ai_addrlen);
   if (rv != 0 && errno != EINPROGRESS) {
     if (ssl) {
       SSL_free(ssl);
@@ -542,13 +608,20 @@ int Client::connect() {
     current_addr = addr;
   }
 
-  writefn = &Client::connected;
-
   ev_io_set(&rev, fd, EV_READ);
   ev_io_set(&wev, fd, EV_WRITE);
 
   ev_io_start(worker->loop, &wev);
 
+  if (config.is_quic()) {
+    ev_io_start(worker->loop, &rev);
+
+    readfn = &Client::read_quic;
+    writefn = &Client::write_quic;
+  } else {
+    writefn = &Client::connected;
+  }
+
   return 0;
 }
 
@@ -603,6 +676,11 @@ void Client::fail() {
 void Client::disconnect() {
   record_client_end_time();
 
+  if (config.is_quic()) {
+    quic_close_connection();
+  }
+
+  ev_timer_stop(worker->loop, &quic.pkt_timer);
   ev_timer_stop(worker->loop, &conn_inactivity_watcher);
   ev_timer_stop(worker->loop, &conn_active_watcher);
   ev_timer_stop(worker->loop, &rps_watcher);
@@ -765,7 +843,12 @@ void Client::report_app_info() {
 }
 
 void Client::terminate_session() {
-  session->terminate();
+  if (config.is_quic()) {
+    quic.close_requested = true;
+  }
+  if (session) {
+    session->terminate();
+  }
   // http1 session needs writecb to tear down session.
   signal_write();
 }
@@ -963,7 +1046,13 @@ int Client::connection_made() {
 
     if (next_proto) {
       auto proto = StringRef{next_proto, next_proto_len};
-      if (util::check_h2_is_selected(proto)) {
+      if (config.is_quic()) {
+        assert(session);
+        if (!util::streq(StringRef{&NGHTTP3_ALPN_H3[1]}, proto) &&
+            !util::streq_l("h3-29", proto)) {
+          return -1;
+        }
+      } else if (util::check_h2_is_selected(proto)) {
         session = std::make_unique<Http2Session>(this);
       } else if (util::streq(NGHTTP2_H1_1, proto)) {
         session = std::make_unique<Http1Session>(this);
@@ -972,6 +1061,9 @@ int Client::connection_made() {
       // Just assign next_proto to selected_proto anyway to show the
       // negotiation result.
       selected_proto = proto.str();
+    } else if (config.is_quic()) {
+      std::cerr << "QUIC requires ALPN negotiation" << std::endl;
+      return -1;
     } else {
       std::cout << "No protocol negotiated. Fallback behaviour may be activated"
                 << std::endl;
@@ -1285,6 +1377,44 @@ int Client::write_tls() {
   return 0;
 }
 
+int Client::write_udp(const sockaddr *addr, socklen_t addrlen,
+                      const uint8_t *data, size_t datalen, size_t gso_size) {
+  iovec msg_iov;
+  msg_iov.iov_base = const_cast<uint8_t *>(data);
+  msg_iov.iov_len = datalen;
+
+  msghdr msg{};
+  msg.msg_name = const_cast<sockaddr *>(addr);
+  msg.msg_namelen = addrlen;
+  msg.msg_iov = &msg_iov;
+  msg.msg_iovlen = 1;
+
+#ifdef UDP_SEGMENT
+  std::array<uint8_t, CMSG_SPACE(sizeof(uint16_t))> msg_ctrl{};
+  if (gso_size && datalen > gso_size) {
+    msg.msg_control = msg_ctrl.data();
+    msg.msg_controllen = msg_ctrl.size();
+
+    auto cm = CMSG_FIRSTHDR(&msg);
+    cm->cmsg_level = SOL_UDP;
+    cm->cmsg_type = UDP_SEGMENT;
+    cm->cmsg_len = CMSG_LEN(sizeof(uint16_t));
+    *(reinterpret_cast<uint16_t *>(CMSG_DATA(cm))) = gso_size;
+  }
+#endif // UDP_SEGMENT
+
+  auto nwrite = sendmsg(fd, &msg, 0);
+  if (nwrite < 0) {
+    std::cerr << "sendto: errno=" << errno << std::endl;
+  } else {
+    ++worker->stats.udp_dgram_sent;
+  }
+
+  ev_io_stop(worker->loop, &wev);
+
+  return 0;
+}
+
 void Client::record_request_time(RequestStat *req_stat) {
   req_stat->request_time = std::chrono::steady_clock::now();
   req_stat->request_wall_time = std::chrono::system_clock::now();
@@ -1403,7 +1533,7 @@ Worker::~Worker() {
 
 void Worker::stop_all_clients() {
   for (auto client : clients) {
-    if (client && client->session) {
+    if (client) {
       client->terminate_session();
     }
   }
@@ -1938,6 +2068,7 @@ Options:
               Default: 1
   -w, --window-bits=<N>
               Sets the stream level initial window size to (2**<N>)-1.
+              For QUIC, <N> is capped to 26 (roughly 64MiB).
               Default: )"
       << config.window_bits << R"(
   -W, --connection-window-bits=<N>
@@ -1948,10 +2079,15 @@ Options:
   -H, --header=<HEADER>
               Add/Override a header to the requests.
   --ciphers=<SUITE>
-              Set allowed  cipher list.  The  format of the  string is
-              described in OpenSSL ciphers(1).
+              Set  allowed cipher  list  for TLSv1.2  or ealier.   The
+              format of the string is described in OpenSSL ciphers(1).
               Default: )"
       << config.ciphers << R"(
+  --tls13-ciphers=<SUITE>
+              Set allowed cipher list for  TLSv1.3.  The format of the
+              string is described in OpenSSL ciphers(1).
+              Default: )"
+      << config.tls13_ciphers << R"(
   -p, --no-tls-proto=<PROTOID>
               Specify ALPN identifier of the  protocol to be used when
               accessing http URI without SSL/TLS.
@@ -2065,11 +2201,24 @@ Options:
               response  time when  using  one worker  thread, but  may
               appear slightly  out of order with  multiple threads due
               to buffering.  Status code is -1 for failed streams.
+  --qlog-file-base=<PATH>
+              Enable qlog output and specify base file name for qlogs.
+              Qlog  is emitted  for each connection.
+              For  a  given  base  name "base", each  output file name
+              becomes  "base.M.N.qlog"  where M is worker ID  and N is
+              client ID (e.g. "base.0.3.qlog").
+              Only effective in QUIC runs.
   --connect-to=<HOST>[:<PORT>]
               Host and port to connect  instead of using the authority
               in <URI>.
   --rps=<N>   Specify request  per second for each  client.  --rps and
               --timing-script-file are mutually exclusive.
+  --groups=<GROUPS>
+              Specify the supported groups.
+              Default: )"
+      << config.groups << R"(
+  --no-udp-gso
+              Disable UDP GSO.
   -v, --verbose
               Output debug information.
   --version   Display version information and exit.
@@ -2097,6 +2246,7 @@ int main(int argc, char **argv) {
 
   std::string datafile;
   std::string logfile;
+  std::string qlog_base;
   bool nreqs_set_manually = false;
   while (1) {
     static int flag = 0;
@@ -2130,6 +2280,10 @@ int main(int argc, char **argv) {
         {"log-file", required_argument, &flag, 10},
         {"connect-to", required_argument, &flag, 11},
         {"rps", required_argument, &flag, 12},
+        {"groups", required_argument, &flag, 13},
+        {"tls13-ciphers", required_argument, &flag, 14},
+        {"no-udp-gso", no_argument, &flag, 15},
+        {"qlog-file-base", required_argument, &flag, 16},
         {nullptr, 0, nullptr, 0}};
     int option_index = 0;
     auto c = getopt_long(argc, argv,
@@ -2380,6 +2534,22 @@ int main(int argc, char **argv) {
         config.rps = v;
         break;
       }
+      case 13:
+        // --groups
+        config.groups = optarg;
+        break;
+      case 14:
+        // --tls13-ciphers
+        config.tls13_ciphers = optarg;
+        break;
+      case 15:
+        // --no-udp-gso
+        config.no_udp_gso = true;
+        break;
+      case 16:
+        // --qlog-file-base
+        qlog_base = optarg;
+        break;
       }
       break;
     default:
@@ -2546,6 +2716,13 @@ int main(int argc, char **argv) {
       exit(EXIT_FAILURE);
     }
     config.data_length = data_stat.st_size;
+    auto addr = mmap(nullptr, config.data_length, PROT_READ, MAP_SHARED,
+                     config.data_fd, 0);
+    if (addr == MAP_FAILED) {
+      std::cerr << "-d: Could not mmap file " << datafile << std::endl;
+      exit(EXIT_FAILURE);
+    }
+    config.data = static_cast<uint8_t *>(addr);
   }
 
   if (!logfile.empty()) {
@@ -2557,6 +2734,16 @@ int main(int argc, char **argv) {
     }
   }
 
+  if (!qlog_base.empty()) {
+    if (!config.is_quic()) {
+      std::cerr
+          << "Warning: --qlog-file-base: only effective in quic, ignoring."
+          << std::endl;
+    } else {
+      config.qlog_file_base = qlog_base;
+    }
+  }
+
   struct sigaction act {};
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, nullptr);
@@ -2576,9 +2763,12 @@ int main(int argc, char **argv) {
   SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY);
   SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);
 
-  if (nghttp2::tls::ssl_ctx_set_proto_versions(
-          ssl_ctx, nghttp2::tls::NGHTTP2_TLS_MIN_VERSION,
-          nghttp2::tls::NGHTTP2_TLS_MAX_VERSION) != 0) {
+  if (config.is_quic()) {
+    SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION);
+    SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION);
+  } else if (nghttp2::tls::ssl_ctx_set_proto_versions(
+                 ssl_ctx, nghttp2::tls::NGHTTP2_TLS_MIN_VERSION,
+                 nghttp2::tls::NGHTTP2_TLS_MAX_VERSION) != 0) {
     std::cerr << "Could not set TLS versions" << std::endl;
     exit(EXIT_FAILURE);
   }
@@ -2590,6 +2780,18 @@ int main(int argc, char **argv) {
     exit(EXIT_FAILURE);
   }
 
+  if (SSL_CTX_set_ciphersuites(ssl_ctx, config.tls13_ciphers.c_str()) == 0) {
+    std::cerr << "SSL_CTX_set_ciphersuites with " << config.tls13_ciphers
+              << " failed: " << ERR_error_string(ERR_get_error(), nullptr)
+              << std::endl;
+    exit(EXIT_FAILURE);
+  }
+
+  if (SSL_CTX_set1_groups_list(ssl_ctx, config.groups.c_str()) != 1) {
+    std::cerr << "SSL_CTX_set1_groups_list failed" << std::endl;
+    exit(EXIT_FAILURE);
+  }
+
 #ifndef OPENSSL_NO_NEXTPROTONEG
   SSL_CTX_set_next_proto_select_cb(ssl_ctx, client_select_next_proto_cb,
                                    nullptr);
@@ -2604,6 +2806,14 @@ int main(int argc, char **argv) {
   SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size());
 #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
 
+  auto keylog_filename = getenv("SSLKEYLOGFILE");
+  if (keylog_filename) {
+    keylog_file.open(keylog_filename, std::ios_base::app);
+    if (keylog_file) {
+      SSL_CTX_set_keylog_callback(ssl_ctx, keylog_callback);
+    }
+  }
+
   std::string user_agent = "h2load nghttp2/" NGHTTP2_VERSION;
   Headers shared_nva;
   shared_nva.emplace_back(":scheme", config.scheme);
@@ -2822,6 +3032,8 @@ int main(int argc, char **argv) {
     stats.bytes_head += s.bytes_head;
     stats.bytes_head_decomp += s.bytes_head_decomp;
     stats.bytes_body += s.bytes_body;
+    stats.udp_dgram_recv += s.udp_dgram_recv;
+    stats.udp_dgram_sent += s.udp_dgram_sent;
 
     for (size_t i = 0; i < stats.status.size(); ++i) {
       stats.status[i] += s.status[i];
@@ -2880,30 +3092,35 @@ traffic: )" << util::utos_funit(stats.bytes_total)
             << util::utos_funit(stats.bytes_head) << "B (" << stats.bytes_head
             << ") headers (space savings " << header_space_savings * 100
             << "%), " << util::utos_funit(stats.bytes_body) << "B ("
-            << stats.bytes_body << R"() data
-                     min         max         mean         sd        +/- sd
+            << stats.bytes_body << R"() data)" << std::endl;
+  if (config.is_quic()) {
+    std::cout << "UDP datagram: " << stats.udp_dgram_sent << " sent, "
+              << stats.udp_dgram_recv << " received" << std::endl;
+  }
+  std::cout
+      << R"(                     min         max         mean         sd        +/- sd
 time for request: )"
-            << std::setw(10) << util::format_duration(ts.request.min) << "  "
-            << std::setw(10) << util::format_duration(ts.request.max) << "  "
-            << std::setw(10) << util::format_duration(ts.request.mean) << "  "
-            << std::setw(10) << util::format_duration(ts.request.sd)
-            << std::setw(9) << util::dtos(ts.request.within_sd) << "%"
-            << "\ntime for connect: " << std::setw(10)
-            << util::format_duration(ts.connect.min) << "  " << std::setw(10)
-            << util::format_duration(ts.connect.max) << "  " << std::setw(10)
-            << util::format_duration(ts.connect.mean) << "  " << std::setw(10)
-            << util::format_duration(ts.connect.sd) << std::setw(9)
-            << util::dtos(ts.connect.within_sd) << "%"
-            << "\ntime to 1st byte: " << std::setw(10)
-            << util::format_duration(ts.ttfb.min) << "  " << std::setw(10)
-            << util::format_duration(ts.ttfb.max) << "  " << std::setw(10)
-            << util::format_duration(ts.ttfb.mean) << "  " << std::setw(10)
-            << util::format_duration(ts.ttfb.sd) << std::setw(9)
-            << util::dtos(ts.ttfb.within_sd) << "%"
-            << "\nreq/s           : " << std::setw(10) << ts.rps.min << "  "
-            << std::setw(10) << ts.rps.max << "  " << std::setw(10)
-            << ts.rps.mean << "  " << std::setw(10) << ts.rps.sd << std::setw(9)
-            << util::dtos(ts.rps.within_sd) << "%" << std::endl;
+      << std::setw(10) << util::format_duration(ts.request.min) << "  "
+      << std::setw(10) << util::format_duration(ts.request.max) << "  "
+      << std::setw(10) << util::format_duration(ts.request.mean) << "  "
+      << std::setw(10) << util::format_duration(ts.request.sd) << std::setw(9)
+      << util::dtos(ts.request.within_sd) << "%"
+      << "\ntime for connect: " << std::setw(10)
+      << util::format_duration(ts.connect.min) << "  " << std::setw(10)
+      << util::format_duration(ts.connect.max) << "  " << std::setw(10)
+      << util::format_duration(ts.connect.mean) << "  " << std::setw(10)
+      << util::format_duration(ts.connect.sd) << std::setw(9)
+      << util::dtos(ts.connect.within_sd) << "%"
+      << "\ntime to 1st byte: " << std::setw(10)
+      << util::format_duration(ts.ttfb.min) << "  " << std::setw(10)
+      << util::format_duration(ts.ttfb.max) << "  " << std::setw(10)
+      << util::format_duration(ts.ttfb.mean) << "  " << std::setw(10)
+      << util::format_duration(ts.ttfb.sd) << std::setw(9)
+      << util::dtos(ts.ttfb.within_sd) << "%"
+      << "\nreq/s           : " << std::setw(10) << ts.rps.min << "  "
+      << std::setw(10) << ts.rps.max << "  " << std::setw(10) << ts.rps.mean
+      << "  " << std::setw(10) << ts.rps.sd << std::setw(9)
+      << util::dtos(ts.rps.within_sd) << "%" << std::endl;
 
   SSL_CTX_free(ssl_ctx);
 

src/h2load.h

@@ -45,11 +45,15 @@
 
 #include <nghttp2/nghttp2.h>
 
+#include <ngtcp2/ngtcp2.h>
+#include <ngtcp2/ngtcp2_crypto.h>
+
 #include <ev.h>
 
 #include <openssl/ssl.h>
 
 #include "http2.h"
+#include "quic.h"
 #include "memchunk.h"
 #include "template.h"
 
@@ -72,8 +76,13 @@ struct Config {
   std::string connect_to_host;
   std::string ifile;
   std::string ciphers;
+  std::string tls13_ciphers;
+  // supported groups (or curves).
+  std::string groups;
   // length of upload data
   int64_t data_length;
+  // memory mapped upload data
+  uint8_t *data;
   addrinfo *addrs;
   size_t nreqs;
   size_t nclients;
@@ -100,6 +109,8 @@ struct Config {
   int data_fd;
   // file descriptor to write per-request stats to.
   int log_fd;
+  // base file name of qlog output files
+  std::string qlog_file_base;
   uint16_t port;
   uint16_t default_port;
   uint16_t connect_to_port;
@@ -116,6 +127,8 @@ struct Config {
   std::vector<std::string> npn_list;
   // The number of request per second for each client.
   double rps;
+  // Disables GSO for UDP connections.
+  bool no_udp_gso;
 
   Config();
   ~Config();
@@ -124,6 +137,7 @@ struct Config {
   bool is_timing_based_mode() const;
   bool has_base_uri() const;
   bool rps_enabled() const;
+  bool is_quic() const;
 };
 
 struct RequestStat {
@@ -220,6 +234,10 @@ struct Stats {
   std::vector<RequestStat> req_stats;
   // The statistics per client
   std::vector<ClientStat> client_stats;
+  // The number of UDP datagrams received.
+  size_t udp_dgram_recv;
+  // The number of UDP datagrams sent.
+  size_t udp_dgram_sent;
 };
 
 enum ClientState { CLIENT_IDLE, CLIENT_CONNECTED };
@@ -309,6 +327,14 @@ struct Client {
   std::function<int(Client &)> readfn, writefn;
   Worker *worker;
   SSL *ssl;
+  struct {
+    ev_timer pkt_timer;
+    ngtcp2_conn *conn;
+    quic::Error last_error;
+    size_t max_pktlen;
+    bool close_requested;
+    FILE *qlog_file;
+  } quic;
   ev_timer request_timeout_watcher;
   addrinfo *next_addr;
   // Address for the current address.  When try_new_connection() is
@@ -332,6 +358,7 @@ struct Client {
   // The client id per worker
   uint32_t id;
   int fd;
+  Address local_addr;
   ev_timer conn_active_watcher;
   ev_timer conn_inactivity_watcher;
   std::string selected_proto;
@@ -419,6 +446,35 @@ struct Client {
   void record_client_end_time();
 
   void signal_write();
+
+  // QUIC
+  int quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
+                const sockaddr *remote_addr, socklen_t remote_addrlen);
+  void quic_free();
+  int read_quic();
+  int write_quic();
+  int write_udp(const sockaddr *addr, socklen_t addrlen, const uint8_t *data,
+                size_t datalen, size_t gso_size);
+  void quic_close_connection();
+
+  int quic_handshake_completed();
+  int quic_recv_stream_data(uint32_t flags, int64_t stream_id,
+                            const uint8_t *data, size_t datalen);
+  int quic_acked_stream_data_offset(int64_t stream_id, size_t datalen);
+  int quic_stream_close(int64_t stream_id, uint64_t app_error_code);
+  int quic_stream_reset(int64_t stream_id, uint64_t app_error_code);
+  int quic_stream_stop_sending(int64_t stream_id, uint64_t app_error_code);
+  int quic_extend_max_local_streams();
+
+  int quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret,
+                  const uint8_t *tx_secret, size_t secretlen);
+  void quic_set_tls_alert(uint8_t alert);
+
+  void quic_write_client_handshake(ngtcp2_crypto_level level,
+                                   const uint8_t *data, size_t datalen);
+  int quic_pkt_timeout();
+  void quic_restart_pkt_timer();
+  void quic_write_qlog(const void *data, size_t datalen);
 };
 
 } // namespace h2load

src/h2load_http3_session.cc

@@ -0,0 +1,427 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2019 nghttp2 contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "h2load_http3_session.h"
+
+#include <iostream>
+
+#include <ngtcp2/ngtcp2.h>
+
+#include "h2load.h"
+
+namespace h2load {
+
+Http3Session::Http3Session(Client *client)
+    : client_(client), conn_(nullptr), npending_request_(0), reqidx_(0) {}
+
+Http3Session::~Http3Session() { nghttp3_conn_del(conn_); }
+
+void Http3Session::on_connect() {}
+
+int Http3Session::submit_request() {
+  if (npending_request_) {
+    ++npending_request_;
+    return 0;
+  }
+
+  auto config = client_->worker->config;
+  reqidx_ = client_->reqidx;
+
+  if (++client_->reqidx == config->nva.size()) {
+    client_->reqidx = 0;
+  }
+
+  auto stream_id = submit_request_internal();
+  if (stream_id < 0) {
+    if (stream_id == NGTCP2_ERR_STREAM_ID_BLOCKED) {
+      ++npending_request_;
+      return 0;
+    }
+    return -1;
+  }
+
+  return 0;
+}
+
+namespace {
+nghttp3_ssize read_data(nghttp3_conn *conn, int64_t stream_id, nghttp3_vec *vec,
+                        size_t veccnt, uint32_t *pflags, void *user_data,
+                        void *stream_user_data) {
+  auto s = static_cast<Http3Session *>(user_data);
+
+  s->read_data(vec, veccnt, pflags);
+
+  return 1;
+}
+} // namespace
+
+void Http3Session::read_data(nghttp3_vec *vec, size_t veccnt,
+                             uint32_t *pflags) {
+  assert(veccnt > 0);
+
+  auto config = client_->worker->config;
+
+  vec[0].base = config->data;
+  vec[0].len = config->data_length;
+  *pflags |= NGHTTP3_DATA_FLAG_EOF;
+}
+
+int64_t Http3Session::submit_request_internal() {
+  int rv;
+  int64_t stream_id;
+
+  auto config = client_->worker->config;
+  auto &nva = config->nva[reqidx_];
+
+  rv = ngtcp2_conn_open_bidi_stream(client_->quic.conn, &stream_id, nullptr);
+  if (rv != 0) {
+    return rv;
+  }
+
+  nghttp3_data_reader dr{};
+  dr.read_data = h2load::read_data;
+
+  rv = nghttp3_conn_submit_request(
+      conn_, stream_id, reinterpret_cast<nghttp3_nv *>(nva.data()), nva.size(),
+      config->data_fd == -1 ? nullptr : &dr, nullptr);
+  if (rv != 0) {
+    return rv;
+  }
+
+  client_->on_request(stream_id);
+  auto req_stat = client_->get_req_stat(stream_id);
+  assert(req_stat);
+  client_->record_request_time(req_stat);
+
+  return stream_id;
+}
+
+int Http3Session::on_read(const uint8_t *data, size_t len) { return -1; }
+
+int Http3Session::on_write() { return -1; }
+
+void Http3Session::terminate() {}
+
+size_t Http3Session::max_concurrent_streams() {
+  return (size_t)client_->worker->config->max_concurrent_streams;
+}
+
+namespace {
+int stream_close(nghttp3_conn *conn, int64_t stream_id, uint64_t app_error_code,
+                 void *user_data, void *stream_user_data) {
+  auto s = static_cast<Http3Session *>(user_data);
+  if (s->stream_close(stream_id, app_error_code) != 0) {
+    return NGHTTP3_ERR_CALLBACK_FAILURE;
+  }
+  return 0;
+}
+} // namespace
+
+int Http3Session::stream_close(int64_t stream_id, uint64_t app_error_code) {
+  if (!ngtcp2_is_bidi_stream(stream_id)) {
+    assert(!ngtcp2_conn_is_local_stream(client_->quic.conn, stream_id));
+    ngtcp2_conn_extend_max_streams_uni(client_->quic.conn, 1);
+  }
+  client_->on_stream_close(stream_id, app_error_code == NGHTTP3_H3_NO_ERROR);
+  return 0;
+}
+
+namespace {
+int recv_data(nghttp3_conn *conn, int64_t stream_id, const uint8_t *data,
+              size_t datalen, void *user_data, void *stream_user_data) {
+  auto s = static_cast<Http3Session *>(user_data);
+  s->recv_data(stream_id, data, datalen);
+  return 0;
+}
+} // namespace
+
+void Http3Session::recv_data(int64_t stream_id, const uint8_t *data,
+                             size_t datalen) {
+  client_->record_ttfb();
+  client_->worker->stats.bytes_body += datalen;
+  consume(stream_id, datalen);
+}
+
+namespace {
+int deferred_consume(nghttp3_conn *conn, int64_t stream_id, size_t nconsumed,
+                     void *user_data, void *stream_user_data) {
+  auto s = static_cast<Http3Session *>(user_data);
+  s->consume(stream_id, nconsumed);
+  return 0;
+}
+} // namespace
+
+void Http3Session::consume(int64_t stream_id, size_t nconsumed) {
+  ngtcp2_conn_extend_max_stream_offset(client_->quic.conn, stream_id,
+                                       nconsumed);
+  ngtcp2_conn_extend_max_offset(client_->quic.conn, nconsumed);
+}
+
+namespace {
+int begin_headers(nghttp3_conn *conn, int64_t stream_id, void *user_data,
+                  void *stream_user_data) {
+  auto s = static_cast<Http3Session *>(user_data);
+  s->begin_headers(stream_id);
+  return 0;
+}
+} // namespace
+
+void Http3Session::begin_headers(int64_t stream_id) {
+  auto payloadlen = nghttp3_conn_get_frame_payload_left(conn_, stream_id);
+  assert(payloadlen > 0);
+
+  client_->worker->stats.bytes_head += payloadlen;
+}
+
+namespace {
+int recv_header(nghttp3_conn *conn, int64_t stream_id, int32_t token,
+                nghttp3_rcbuf *name, nghttp3_rcbuf *value, uint8_t flags,
+                void *user_data, void *stream_user_data) {
+  auto s = static_cast<Http3Session *>(user_data);
+  auto k = nghttp3_rcbuf_get_buf(name);
+  auto v = nghttp3_rcbuf_get_buf(value);
+  s->recv_header(stream_id, &k, &v);
+  return 0;
+}
+} // namespace
+
+void Http3Session::recv_header(int64_t stream_id, const nghttp3_vec *name,
+                               const nghttp3_vec *value) {
+  client_->on_header(stream_id, name->base, name->len, value->base, value->len);
+  client_->worker->stats.bytes_head_decomp += name->len + value->len;
+}
+
+namespace {
+int send_stop_sending(nghttp3_conn *conn, int64_t stream_id,
+                      uint64_t app_error_code, void *user_data,
+                      void *stream_user_data) {
+  auto s = static_cast<Http3Session *>(user_data);
+  if (s->send_stop_sending(stream_id, app_error_code) != 0) {
+    return NGHTTP3_ERR_CALLBACK_FAILURE;
+  }
+  return 0;
+}
+} // namespace
+
+int Http3Session::send_stop_sending(int64_t stream_id,
+                                    uint64_t app_error_code) {
+  auto rv = ngtcp2_conn_shutdown_stream_read(client_->quic.conn, stream_id,
+                                             app_error_code);
+  if (rv != 0) {
+    std::cerr << "ngtcp2_conn_shutdown_stream_read: " << ngtcp2_strerror(rv)
+              << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int Http3Session::close_stream(int64_t stream_id, uint64_t app_error_code) {
+  auto rv = nghttp3_conn_close_stream(conn_, stream_id, app_error_code);
+  switch (rv) {
+  case 0:
+    return 0;
+  case NGHTTP3_ERR_STREAM_NOT_FOUND:
+    if (!ngtcp2_is_bidi_stream(stream_id)) {
+      assert(!ngtcp2_conn_is_local_stream(client_->quic.conn, stream_id));
+      ngtcp2_conn_extend_max_streams_uni(client_->quic.conn, 1);
+    }
+    return 0;
+  default:
+    return -1;
+  }
+}
+
+int Http3Session::shutdown_stream_read(int64_t stream_id) {
+  auto rv = nghttp3_conn_shutdown_stream_read(conn_, stream_id);
+  if (rv != 0) {
+    return -1;
+  }
+  return 0;
+}
+
+int Http3Session::extend_max_local_streams() {
+  auto config = client_->worker->config;
+
+  for (; npending_request_; --npending_request_) {
+    auto stream_id = submit_request_internal();
+    if (stream_id < 0) {
+      if (stream_id == NGTCP2_ERR_STREAM_ID_BLOCKED) {
+        return 0;
+      }
+      return -1;
+    }
+
+    if (++reqidx_ == config->nva.size()) {
+      reqidx_ = 0;
+    }
+  }
+
+  return 0;
+}
+
+int Http3Session::init_conn() {
+  int rv;
+
+  assert(conn_ == nullptr);
+
+  if (ngtcp2_conn_get_max_local_streams_uni(client_->quic.conn) < 3) {
+    return -1;
+  }
+
+  nghttp3_callbacks callbacks{
+      nullptr, // acked_stream_data
+      h2load::stream_close,
+      h2load::recv_data,
+      h2load::deferred_consume,
+      h2load::begin_headers,
+      h2load::recv_header,
+      nullptr, // end_headers
+      nullptr, // begin_trailers
+      h2load::recv_header,
+      nullptr, // end_trailers
+      h2load::send_stop_sending,
+  };
+
+  auto config = client_->worker->config;
+
+  nghttp3_settings settings;
+  nghttp3_settings_default(&settings);
+  settings.qpack_max_table_capacity = config->header_table_size;
+  settings.qpack_blocked_streams = 100;
+
+  auto mem = nghttp3_mem_default();
+
+  rv = nghttp3_conn_client_new(&conn_, &callbacks, &settings, mem, this);
+  if (rv != 0) {
+    std::cerr << "nghttp3_conn_client_new: " << nghttp3_strerror(rv)
+              << std::endl;
+    return -1;
+  }
+
+  int64_t ctrl_stream_id;
+
+  rv = ngtcp2_conn_open_uni_stream(client_->quic.conn, &ctrl_stream_id, NULL);
+  if (rv != 0) {
+    std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv)
+              << std::endl;
+    return -1;
+  }
+
+  rv = nghttp3_conn_bind_control_stream(conn_, ctrl_stream_id);
+  if (rv != 0) {
+    std::cerr << "nghttp3_conn_bind_control_stream: " << nghttp3_strerror(rv)
+              << std::endl;
+    return -1;
+  }
+
+  int64_t qpack_enc_stream_id, qpack_dec_stream_id;
+
+  rv = ngtcp2_conn_open_uni_stream(client_->quic.conn, &qpack_enc_stream_id,
+                                   NULL);
+  if (rv != 0) {
+    std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv)
+              << std::endl;
+    return -1;
+  }
+
+  rv = ngtcp2_conn_open_uni_stream(client_->quic.conn, &qpack_dec_stream_id,
+                                   NULL);
+  if (rv != 0) {
+    std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv)
+              << std::endl;
+    return -1;
+  }
+
+  rv = nghttp3_conn_bind_qpack_streams(conn_, qpack_enc_stream_id,
+                                       qpack_dec_stream_id);
+  if (rv != 0) {
+    std::cerr << "nghttp3_conn_bind_qpack_streams: " << nghttp3_strerror(rv)
+              << std::endl;
+    return -1;
+  }
+
+  return 0;
+}
+
+ssize_t Http3Session::read_stream(uint32_t flags, int64_t stream_id,
+                                  const uint8_t *data, size_t datalen) {
+  auto nconsumed = nghttp3_conn_read_stream(
+      conn_, stream_id, data, datalen, flags & NGTCP2_STREAM_DATA_FLAG_FIN);
+  if (nconsumed < 0) {
+    std::cerr << "nghttp3_conn_read_stream: " << nghttp3_strerror(nconsumed)
+              << std::endl;
+    client_->quic.last_error = quic::err_application(nconsumed);
+    return -1;
+  }
+  return nconsumed;
+}
+
+ssize_t Http3Session::write_stream(int64_t &stream_id, int &fin,
+                                   nghttp3_vec *vec, size_t veccnt) {
+  auto sveccnt =
+      nghttp3_conn_writev_stream(conn_, &stream_id, &fin, vec, veccnt);
+  if (sveccnt < 0) {
+    client_->quic.last_error = quic::err_application(sveccnt);
+    return -1;
+  }
+  return sveccnt;
+}
+
+int Http3Session::block_stream(int64_t stream_id) {
+  auto rv = nghttp3_conn_block_stream(conn_, stream_id);
+  if (rv != 0) {
+    client_->quic.last_error = quic::err_application(rv);
+    return -1;
+  }
+  return 0;
+}
+
+int Http3Session::shutdown_stream_write(int64_t stream_id) {
+  auto rv = nghttp3_conn_shutdown_stream_write(conn_, stream_id);
+  if (rv != 0) {
+    client_->quic.last_error = quic::err_application(rv);
+    return -1;
+  }
+  return 0;
+}
+
+int Http3Session::add_write_offset(int64_t stream_id, size_t ndatalen) {
+  auto rv = nghttp3_conn_add_write_offset(conn_, stream_id, ndatalen);
+  if (rv != 0) {
+    client_->quic.last_error = quic::err_application(rv);
+    return -1;
+  }
+  return 0;
+}
+
+int Http3Session::add_ack_offset(int64_t stream_id, size_t datalen) {
+  auto rv = nghttp3_conn_add_ack_offset(conn_, stream_id, datalen);
+  if (rv != 0) {
+    client_->quic.last_error = quic::err_application(rv);
+    return -1;
+  }
+  return 0;
+}
+
+} // namespace h2load

src/h2load_http3_session.h

@@ -0,0 +1,81 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2019 nghttp2 contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef H2LOAD_HTTP3_SESSION_H
+#define H2LOAD_HTTP3_SESSION_H
+
+#include "h2load_session.h"
+
+#include <nghttp3/nghttp3.h>
+
+namespace h2load {
+
+struct Client;
+
+class Http3Session : public Session {
+public:
+  Http3Session(Client *client);
+  virtual ~Http3Session();
+  virtual void on_connect();
+  virtual int submit_request();
+  virtual int on_read(const uint8_t *data, size_t len);
+  virtual int on_write();
+  virtual void terminate();
+  virtual size_t max_concurrent_streams();
+
+  int init_conn();
+  int stream_close(int64_t stream_id, uint64_t app_error_code);
+  void recv_data(int64_t stream_id, const uint8_t *data, size_t datalen);
+  void consume(int64_t stream_id, size_t nconsumed);
+  void begin_headers(int64_t stream_id);
+  void recv_header(int64_t stream_id, const nghttp3_vec *name,
+                   const nghttp3_vec *value);
+  int send_stop_sending(int64_t stream_id, uint64_t app_error_code);
+
+  int close_stream(int64_t stream_id, uint64_t app_error_code);
+  int shutdown_stream_read(int64_t stream_id);
+  int extend_max_local_streams();
+  int64_t submit_request_internal();
+
+  ssize_t read_stream(uint32_t flags, int64_t stream_id, const uint8_t *data,
+                      size_t datalen);
+  ssize_t write_stream(int64_t &stream_id, int &fin, nghttp3_vec *vec,
+                       size_t veccnt);
+  int block_stream(int64_t stream_id);
+  int shutdown_stream_write(int64_t stream_id);
+  int add_write_offset(int64_t stream_id, size_t ndatalen);
+  int add_ack_offset(int64_t stream_id, size_t datalen);
+
+  void read_data(nghttp3_vec *vec, size_t veccnt, uint32_t *pflags);
+
+private:
+  Client *client_;
+  nghttp3_conn *conn_;
+  size_t npending_request_;
+  size_t reqidx_;
+};
+
+} // namespace h2load
+
+#endif // H2LOAD_HTTP3_SESSION_H

src/h2load_quic.cc

@@ -0,0 +1,679 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2019 nghttp2 contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "h2load_quic.h"
+
+#include <netinet/udp.h>
+
+#include <iostream>
+
+#include <ngtcp2/ngtcp2_crypto_openssl.h>
+
+#include <openssl/err.h>
+
+#include "h2load_http3_session.h"
+
+namespace h2load {
+
+namespace {
+auto randgen = util::make_mt19937();
+} // namespace
+
+namespace {
+int handshake_completed(ngtcp2_conn *conn, void *user_data) {
+  auto c = static_cast<Client *>(user_data);
+
+  if (c->quic_handshake_completed() != 0) {
+    return NGTCP2_ERR_CALLBACK_FAILURE;
+  }
+
+  return 0;
+}
+} // namespace
+
+int Client::quic_handshake_completed() { return connection_made(); }
+
+namespace {
+int recv_stream_data(ngtcp2_conn *conn, uint32_t flags, int64_t stream_id,
+                     uint64_t offset, const uint8_t *data, size_t datalen,
+                     void *user_data, void *stream_user_data) {
+  auto c = static_cast<Client *>(user_data);
+  if (c->quic_recv_stream_data(flags, stream_id, data, datalen) != 0) {
+    // TODO Better to do this gracefully rather than
+    // NGTCP2_ERR_CALLBACK_FAILURE.  Perhaps, call
+    // ngtcp2_conn_write_application_close() ?
+    return NGTCP2_ERR_CALLBACK_FAILURE;
+  }
+  return 0;
+}
+} // namespace
+
+int Client::quic_recv_stream_data(uint32_t flags, int64_t stream_id,
+                                  const uint8_t *data, size_t datalen) {
+  if (worker->current_phase == Phase::MAIN_DURATION) {
+    worker->stats.bytes_total += datalen;
+  }
+
+  auto s = static_cast<Http3Session *>(session.get());
+  auto nconsumed = s->read_stream(flags, stream_id, data, datalen);
+  if (nconsumed == -1) {
+    return -1;
+  }
+
+  ngtcp2_conn_extend_max_stream_offset(quic.conn, stream_id, nconsumed);
+  ngtcp2_conn_extend_max_offset(quic.conn, nconsumed);
+
+  return 0;
+}
+
+namespace {
+int acked_stream_data_offset(ngtcp2_conn *conn, int64_t stream_id,
+                             uint64_t offset, uint64_t datalen, void *user_data,
+                             void *stream_user_data) {
+  auto c = static_cast<Client *>(user_data);
+  if (c->quic_acked_stream_data_offset(stream_id, datalen) != 0) {
+    return NGTCP2_ERR_CALLBACK_FAILURE;
+  }
+  return 0;
+}
+} // namespace
+
+int Client::quic_acked_stream_data_offset(int64_t stream_id, size_t datalen) {
+  auto s = static_cast<Http3Session *>(session.get());
+  if (s->add_ack_offset(stream_id, datalen) != 0) {
+    return -1;
+  }
+  return 0;
+}
+
+namespace {
+int stream_close(ngtcp2_conn *conn, int64_t stream_id, uint64_t app_error_code,
+                 void *user_data, void *stream_user_data) {
+  auto c = static_cast<Client *>(user_data);
+  if (c->quic_stream_close(stream_id, app_error_code) != 0) {
+    return -1;
+  }
+  return 0;
+}
+} // namespace
+
+int Client::quic_stream_close(int64_t stream_id, uint64_t app_error_code) {
+  auto s = static_cast<Http3Session *>(session.get());
+  if (s->close_stream(stream_id, app_error_code == 0 ? NGHTTP3_H3_NO_ERROR
+                                                     : app_error_code) != 0) {
+    return -1;
+  }
+  return 0;
+}
+
+namespace {
+int stream_reset(ngtcp2_conn *conn, int64_t stream_id, uint64_t final_size,
+                 uint64_t app_error_code, void *user_data,
+                 void *stream_user_data) {
+  auto c = static_cast<Client *>(user_data);
+  if (c->quic_stream_reset(stream_id, app_error_code) != 0) {
+    return -1;
+  }
+  return 0;
+}
+} // namespace
+
+int Client::quic_stream_reset(int64_t stream_id, uint64_t app_error_code) {
+  auto s = static_cast<Http3Session *>(session.get());
+  if (s->shutdown_stream_read(stream_id) != 0) {
+    return -1;
+  }
+  return 0;
+}
+
+namespace {
+int stream_stop_sending(ngtcp2_conn *conn, int64_t stream_id,
+                        uint64_t app_error_code, void *user_data,
+                        void *stream_user_data) {
+  auto c = static_cast<Client *>(user_data);
+  if (c->quic_stream_stop_sending(stream_id, app_error_code) != 0) {
+    return -1;
+  }
+  return 0;
+}
+} // namespace
+
+int Client::quic_stream_stop_sending(int64_t stream_id,
+                                     uint64_t app_error_code) {
+  auto s = static_cast<Http3Session *>(session.get());
+  if (s->shutdown_stream_read(stream_id) != 0) {
+    return -1;
+  }
+  return 0;
+}
+
+namespace {
+int extend_max_local_streams_bidi(ngtcp2_conn *conn, uint64_t max_streams,
+                                  void *user_data) {
+  auto c = static_cast<Client *>(user_data);
+
+  if (c->quic_extend_max_local_streams() != 0) {
+    return NGTCP2_ERR_CALLBACK_FAILURE;
+  }
+
+  return 0;
+}
+} // namespace
+
+int Client::quic_extend_max_local_streams() {
+  auto s = static_cast<Http3Session *>(session.get());
+  if (s->extend_max_local_streams() != 0) {
+    return NGTCP2_ERR_CALLBACK_FAILURE;
+  }
+  return 0;
+}
+
+namespace {
+int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token,
+                          size_t cidlen, void *user_data) {
+  auto dis = std::uniform_int_distribution<uint8_t>(
+      0, std::numeric_limits<uint8_t>::max());
+  auto f = [&dis]() { return dis(randgen); };
+
+  std::generate_n(cid->data, cidlen, f);
+  cid->datalen = cidlen;
+  std::generate_n(token, NGTCP2_STATELESS_RESET_TOKENLEN, f);
+
+  return 0;
+}
+} // namespace
+
+namespace {
+void debug_log_printf(void *user_data, const char *fmt, ...) {
+  va_list ap;
+
+  va_start(ap, fmt);
+  vfprintf(stderr, fmt, ap);
+  va_end(ap);
+
+  fprintf(stderr, "\n");
+}
+} // namespace
+
+namespace {
+void generate_cid(ngtcp2_cid &dest) {
+  auto dis = std::uniform_int_distribution<uint8_t>(
+      0, std::numeric_limits<uint8_t>::max());
+  dest.datalen = 8;
+  std::generate_n(dest.data, dest.datalen, [&dis]() { return dis(randgen); });
+}
+} // namespace
+
+namespace {
+ngtcp2_tstamp timestamp(struct ev_loop *loop) {
+  return ev_now(loop) * NGTCP2_SECONDS;
+}
+} // namespace
+
+namespace {
+int set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
+                           const uint8_t *rx_secret, const uint8_t *tx_secret,
+                           size_t secret_len) {
+  auto c = static_cast<Client *>(SSL_get_app_data(ssl));
+
+  if (c->quic_on_key(
+          ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level),
+          rx_secret, tx_secret, secret_len) != 0) {
+    return 0;
+  }
+
+  return 1;
+}
+} // namespace
+
+namespace {
+int add_handshake_data(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
+                       const uint8_t *data, size_t len) {
+  auto c = static_cast<Client *>(SSL_get_app_data(ssl));
+  c->quic_write_client_handshake(
+      ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level), data, len);
+  return 1;
+}
+} // namespace
+
+namespace {
+int flush_flight(SSL *ssl) { return 1; }
+} // namespace
+
+namespace {
+int send_alert(SSL *ssl, enum ssl_encryption_level_t level, uint8_t alert) {
+  auto c = static_cast<Client *>(SSL_get_app_data(ssl));
+  c->quic_set_tls_alert(alert);
+  return 1;
+}
+} // namespace
+
+namespace {
+auto quic_method = SSL_QUIC_METHOD{
+    set_encryption_secrets,
+    add_handshake_data,
+    flush_flight,
+    send_alert,
+};
+} // namespace
+
+// qlog write callback -- excerpted from ngtcp2/examples/client_base.cc
+namespace {
+void qlog_write_cb(void *user_data, uint32_t flags, const void *data,
+                   size_t datalen) {
+  auto c = static_cast<Client *>(user_data);
+  c->quic_write_qlog(data, datalen);
+}
+} // namespace
+
+void Client::quic_write_qlog(const void *data, size_t datalen) {
+  assert(quic.qlog_file != nullptr);
+  fwrite(data, 1, datalen, quic.qlog_file);
+}
+
+int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
+                      const sockaddr *remote_addr, socklen_t remote_addrlen) {
+  int rv;
+
+  if (!ssl) {
+    ssl = SSL_new(worker->ssl_ctx);
+
+    SSL_set_app_data(ssl, this);
+    SSL_set_connect_state(ssl);
+    SSL_set_quic_method(ssl, &quic_method);
+  }
+
+  switch (remote_addr->sa_family) {
+  case AF_INET:
+    quic.max_pktlen = NGTCP2_MAX_PKTLEN_IPV4;
+    break;
+  case AF_INET6:
+    quic.max_pktlen = NGTCP2_MAX_PKTLEN_IPV6;
+    break;
+  default:
+    return -1;
+  }
+
+  auto callbacks = ngtcp2_callbacks{
+      ngtcp2_crypto_client_initial_cb,
+      nullptr, // recv_client_initial
+      ngtcp2_crypto_recv_crypto_data_cb,
+      h2load::handshake_completed,
+      nullptr, // recv_version_negotiation
+      ngtcp2_crypto_encrypt_cb,
+      ngtcp2_crypto_decrypt_cb,
+      ngtcp2_crypto_hp_mask_cb,
+      h2load::recv_stream_data,
+      h2load::acked_stream_data_offset,
+      nullptr, // stream_open
+      h2load::stream_close,
+      nullptr, // recv_stateless_reset
+      ngtcp2_crypto_recv_retry_cb,
+      h2load::extend_max_local_streams_bidi,
+      nullptr, // extend_max_local_streams_uni
+      nullptr, // rand
+      get_new_connection_id,
+      nullptr, // remove_connection_id
+      ngtcp2_crypto_update_key_cb,
+      nullptr, // path_validation
+      nullptr, // select_preferred_addr
+      h2load::stream_reset,
+      nullptr, // extend_max_remote_streams_bidi
+      nullptr, // extend_max_remote_streams_uni
+      nullptr, // extend_max_stream_data
+      nullptr, // dcid_status
+      nullptr, // handshake_confirmed
+      nullptr, // recv_new_token
+      ngtcp2_crypto_delete_crypto_aead_ctx_cb,
+      ngtcp2_crypto_delete_crypto_cipher_ctx_cb,
+      nullptr, // recv_datagram
+      nullptr, // ack_datagram
+      nullptr, // lost_datagram
+      nullptr, // get_path_challenge_data
+      h2load::stream_stop_sending,
+  };
+
+  ngtcp2_cid scid, dcid;
+  generate_cid(scid);
+  generate_cid(dcid);
+
+  auto config = worker->config;
+
+  ngtcp2_settings settings;
+  ngtcp2_settings_default(&settings);
+  if (config->verbose) {
+    settings.log_printf = debug_log_printf;
+  }
+  settings.initial_ts = timestamp(worker->loop);
+  if (!config->qlog_file_base.empty()) {
+    assert(quic.qlog_file == nullptr);
+    auto path = config->qlog_file_base;
+    path += '.';
+    path += util::utos(worker->id);
+    path += '.';
+    path += util::utos(id);
+    path += ".qlog";
+    quic.qlog_file = fopen(path.c_str(), "w");
+    if (quic.qlog_file == nullptr) {
+      std::cerr << "Failed to open a qlog file: " << path << std::endl;
+      return -1;
+    }
+    settings.qlog.write = qlog_write_cb;
+  }
+
+  ngtcp2_transport_params params;
+  ngtcp2_transport_params_default(&params);
+  auto max_stream_data =
+      std::min((1 << 26) - 1, (1 << config->window_bits) - 1);
+  params.initial_max_stream_data_bidi_local = max_stream_data;
+  params.initial_max_stream_data_uni = max_stream_data;
+  params.initial_max_data = (1 << config->connection_window_bits) - 1;
+  params.initial_max_streams_bidi = 0;
+  params.initial_max_streams_uni = 100;
+  params.max_idle_timeout = 30 * NGTCP2_SECONDS;
+
+  auto path = ngtcp2_path{
+      {local_addrlen, const_cast<sockaddr *>(local_addr)},
+      {remote_addrlen, const_cast<sockaddr *>(remote_addr)},
+  };
+
+  assert(config->npn_list.size());
+
+  uint32_t quic_version;
+
+  if (config->npn_list[0] == NGHTTP3_ALPN_H3) {
+    quic_version = NGTCP2_PROTO_VER_V1;
+  } else {
+    quic_version = NGTCP2_PROTO_VER_MIN;
+  }
+
+  rv = ngtcp2_conn_client_new(&quic.conn, &dcid, &scid, &path, quic_version,
+                              &callbacks, &settings, &params, nullptr, this);
+  if (rv != 0) {
+    return -1;
+  }
+
+  ngtcp2_conn_set_tls_native_handle(quic.conn, ssl);
+
+  return 0;
+}
+
+void Client::quic_free() {
+  ngtcp2_conn_del(quic.conn);
+  if (quic.qlog_file != nullptr) {
+    fclose(quic.qlog_file);
+    quic.qlog_file = nullptr;
+  }
+}
+
+void Client::quic_close_connection() {
+  if (!quic.conn) {
+    return;
+  }
+
+  std::array<uint8_t, 1500> buf;
+  ngtcp2_ssize nwrite;
+  ngtcp2_path_storage ps;
+  ngtcp2_path_storage_zero(&ps);
+
+  switch (quic.last_error.type) {
+  case quic::ErrorType::TransportVersionNegotiation:
+    return;
+  case quic::ErrorType::Transport:
+    nwrite = ngtcp2_conn_write_connection_close(
+        quic.conn, &ps.path, nullptr, buf.data(), quic.max_pktlen,
+        quic.last_error.code, timestamp(worker->loop));
+    break;
+  case quic::ErrorType::Application:
+    nwrite = ngtcp2_conn_write_application_close(
+        quic.conn, &ps.path, nullptr, buf.data(), quic.max_pktlen,
+        quic.last_error.code, timestamp(worker->loop));
+    break;
+  default:
+    assert(0);
+  }
+
+  if (nwrite < 0) {
+    return;
+  }
+
+  write_udp(reinterpret_cast<sockaddr *>(ps.path.remote.addr),
+            ps.path.remote.addrlen, buf.data(), nwrite, 0);
+}
+
+int Client::quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret,
+                        const uint8_t *tx_secret, size_t secretlen) {
+  if (ngtcp2_crypto_derive_and_install_rx_key(quic.conn, nullptr, nullptr,
+                                              nullptr, level, rx_secret,
+                                              secretlen) != 0) {
+    std::cerr << "ngtcp2_crypto_derive_and_install_rx_key() failed"
+              << std::endl;
+    return -1;
+  }
+
+  if (ngtcp2_crypto_derive_and_install_tx_key(quic.conn, nullptr, nullptr,
+                                              nullptr, level, tx_secret,
+                                              secretlen) != 0) {
+    std::cerr << "ngtcp2_crypto_derive_and_install_tx_key() failed"
+              << std::endl;
+    return -1;
+  }
+
+  if (level == NGTCP2_CRYPTO_LEVEL_APPLICATION) {
+    auto s = std::make_unique<Http3Session>(this);
+    if (s->init_conn() == -1) {
+      return -1;
+    }
+    session = std::move(s);
+  }
+
+  return 0;
+}
+
+void Client::quic_set_tls_alert(uint8_t alert) {
+  quic.last_error = quic::err_transport_tls(alert);
+}
+
+void Client::quic_write_client_handshake(ngtcp2_crypto_level level,
+                                         const uint8_t *data, size_t datalen) {
+  assert(level < 2);
+
+  ngtcp2_conn_submit_crypto_data(quic.conn, level, data, datalen);
+}
+
+void quic_pkt_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
+  auto c = static_cast<Client *>(w->data);
+
+  if (c->quic_pkt_timeout() != 0) {
+    c->fail();
+    c->worker->free_client(c);
+    delete c;
+    return;
+  }
+}
+
+int Client::quic_pkt_timeout() {
+  int rv;
+  auto now = timestamp(worker->loop);
+
+  rv = ngtcp2_conn_handle_expiry(quic.conn, now);
+  if (rv != 0) {
+    quic.last_error = quic::err_transport(NGTCP2_ERR_INTERNAL);
+    return -1;
+  }
+
+  return write_quic();
+}
+
+void Client::quic_restart_pkt_timer() {
+  auto expiry = ngtcp2_conn_get_expiry(quic.conn);
+  auto now = timestamp(worker->loop);
+  auto t = expiry > now ? static_cast<ev_tstamp>(expiry - now) / NGTCP2_SECONDS
+                        : 1e-9;
+  quic.pkt_timer.repeat = t;
+  ev_timer_again(worker->loop, &quic.pkt_timer);
+}
+
+int Client::read_quic() {
+  std::array<uint8_t, 65536> buf;
+  sockaddr_union su;
+  socklen_t addrlen = sizeof(su);
+  int rv;
+  size_t pktcnt = 0;
+  ngtcp2_pkt_info pi{};
+
+  for (;;) {
+    auto nread =
+        recvfrom(fd, buf.data(), buf.size(), MSG_DONTWAIT, &su.sa, &addrlen);
+    if (nread == -1) {
+      return 0;
+    }
+
+    assert(quic.conn);
+
+    ++worker->stats.udp_dgram_recv;
+
+    auto path = ngtcp2_path{
+        {local_addr.len, &local_addr.su.sa},
+        {addrlen, &su.sa},
+    };
+
+    rv = ngtcp2_conn_read_pkt(quic.conn, &path, &pi, buf.data(), nread,
+                              timestamp(worker->loop));
+    if (rv != 0) {
+      std::cerr << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv) << std::endl;
+      return -1;
+    }
+
+    if (++pktcnt == 100) {
+      break;
+    }
+  }
+
+  return 0;
+}
+
+int Client::write_quic() {
+  ev_io_stop(worker->loop, &wev);
+
+  if (quic.close_requested) {
+    return -1;
+  }
+
+  std::array<nghttp3_vec, 16> vec;
+  size_t pktcnt = 0;
+  size_t max_pktcnt =
+#ifdef UDP_SEGMENT
+      worker->config->no_udp_gso
+          ? 1
+          : std::min(static_cast<size_t>(10),
+                     static_cast<size_t>(64_k / quic.max_pktlen));
+#else  // !UDP_SEGMENT
+      1;
+#endif // !UDP_SEGMENT
+  std::array<uint8_t, 64_k> buf;
+  uint8_t *bufpos = buf.data();
+  ngtcp2_path_storage ps;
+
+  ngtcp2_path_storage_zero(&ps);
+
+  auto s = static_cast<Http3Session *>(session.get());
+
+  for (;;) {
+    int64_t stream_id = -1;
+    int fin = 0;
+    ssize_t sveccnt = 0;
+
+    if (session && ngtcp2_conn_get_max_data_left(quic.conn)) {
+      sveccnt = s->write_stream(stream_id, fin, vec.data(), vec.size());
+      if (sveccnt == -1) {
+        return -1;
+      }
+    }
+
+    ngtcp2_ssize ndatalen;
+    auto v = vec.data();
+    auto vcnt = static_cast<size_t>(sveccnt);
+
+    uint32_t flags = NGTCP2_WRITE_STREAM_FLAG_MORE;
+    if (fin) {
+      flags |= NGTCP2_WRITE_STREAM_FLAG_FIN;
+    }
+
+    auto nwrite = ngtcp2_conn_writev_stream(
+        quic.conn, &ps.path, nullptr, bufpos, quic.max_pktlen, &ndatalen, flags,
+        stream_id, reinterpret_cast<const ngtcp2_vec *>(v), vcnt,
+        timestamp(worker->loop));
+    if (nwrite < 0) {
+      switch (nwrite) {
+      case NGTCP2_ERR_STREAM_DATA_BLOCKED:
+        assert(ndatalen == -1);
+        if (s->block_stream(stream_id) != 0) {
+          return -1;
+        }
+        continue;
+      case NGTCP2_ERR_STREAM_SHUT_WR:
+        assert(ndatalen == -1);
+        if (s->shutdown_stream_write(stream_id) != 0) {
+          return -1;
+        }
+        continue;
+      case NGTCP2_ERR_WRITE_MORE:
+        assert(ndatalen >= 0);
+        if (s->add_write_offset(stream_id, ndatalen) != 0) {
+          return -1;
+        }
+        continue;
+      }
+
+      quic.last_error = quic::err_transport(nwrite);
+      return -1;
+    } else if (ndatalen >= 0 && s->add_write_offset(stream_id, ndatalen) != 0) {
+      return -1;
+    }
+
+    quic_restart_pkt_timer();
+
+    if (nwrite == 0) {
+      if (bufpos - buf.data()) {
+        write_udp(ps.path.remote.addr, ps.path.remote.addrlen, buf.data(),
+                  bufpos - buf.data(), quic.max_pktlen);
+      }
+      return 0;
+    }
+
+    bufpos += nwrite;
+
+    // Assume that the path does not change.
+    if (++pktcnt == max_pktcnt ||
+        static_cast<size_t>(nwrite) < quic.max_pktlen) {
+      write_udp(ps.path.remote.addr, ps.path.remote.addrlen, buf.data(),
+                bufpos - buf.data(), quic.max_pktlen);
+      signal_write();
+      return 0;
+    }
+  }
+}
+
+} // namespace h2load

src/h2load_quic.h

@@ -0,0 +1,38 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2019 nghttp2 contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef H2LOAD_QUIC_H
+#define H2LOAD_QUIC_H
+
+#include "nghttp2_config.h"
+
+#include <ev.h>
+
+#include "h2load.h"
+
+namespace h2load {
+void quic_pkt_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents);
+} // namespace h2load
+
+#endif // H2LOAD_QUIC_H

src/quic.cc

@@ -0,0 +1,56 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2019 nghttp2 contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "quic.h"
+
+#include <cassert>
+
+#include <ngtcp2/ngtcp2.h>
+#include <nghttp3/nghttp3.h>
+
+#include "template.h"
+
+using namespace nghttp2;
+
+namespace quic {
+
+Error err_transport(int liberr) {
+  if (liberr == NGTCP2_ERR_RECV_VERSION_NEGOTIATION) {
+    return {ErrorType::TransportVersionNegotiation, 0};
+  }
+  return {ErrorType::Transport,
+          ngtcp2_err_infer_quic_transport_error_code(liberr)};
+}
+
+Error err_transport_tls(int alert) {
+  return {ErrorType::Transport, ngtcp2_err_infer_quic_transport_error_code(
+                                    NGTCP2_CRYPTO_ERROR | alert)};
+}
+
+Error err_application(int liberr) {
+  return {ErrorType::Application,
+          nghttp3_err_infer_quic_app_error_code(liberr)};
+}
+
+} // namespace quic

src/quic.h

@@ -0,0 +1,54 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2019 nghttp2 contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef QUIC_H
+#define QUIC_H
+
+#include "nghttp2_config.h"
+
+#include "stdint.h"
+
+namespace quic {
+
+enum class ErrorType {
+  Transport,
+  TransportVersionNegotiation,
+  Application,
+};
+
+struct Error {
+  Error(ErrorType type, uint64_t code) : type(type), code(code) {}
+  Error() : type(ErrorType::Transport), code(0) {}
+
+  ErrorType type;
+  uint64_t code;
+};
+
+Error err_transport(int liberr);
+Error err_transport_tls(int alert);
+Error err_application(int liberr);
+
+} // namespace quic
+
+#endif // QUIC_H

src/util.cc

@@ -945,6 +945,56 @@ int create_nonblock_socket(int family) {
   return fd;
 }
 
+int create_nonblock_udp_socket(int family) {
+#ifdef SOCK_NONBLOCK
+  auto fd = socket(family, SOCK_DGRAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
+
+  if (fd == -1) {
+    return -1;
+  }
+#else  // !SOCK_NONBLOCK
+  auto fd = socket(family, SOCK_DGRAM, 0);
+
+  if (fd == -1) {
+    return -1;
+  }
+
+  make_socket_nonblocking(fd);
+  make_socket_closeonexec(fd);
+#endif // !SOCK_NONBLOCK
+
+  return fd;
+}
+
+int bind_any_addr_udp(int fd, int family) {
+  addrinfo hints{};
+  addrinfo *res, *rp;
+  int rv;
+
+  hints.ai_family = family;
+  hints.ai_socktype = SOCK_DGRAM;
+  hints.ai_flags = AI_PASSIVE;
+
+  rv = getaddrinfo(nullptr, "0", &hints, &res);
+  if (rv != 0) {
+    return -1;
+  }
+
+  for (rp = res; rp; rp = rp->ai_next) {
+    if (bind(fd, rp->ai_addr, rp->ai_addrlen) != -1) {
+      break;
+    }
+  }
+
+  freeaddrinfo(res);
+
+  if (!rp) {
+    return -1;
+  }
+
+  return 0;
+}
+
 bool check_socket_connected(int fd) {
   int error;
   socklen_t len = sizeof(error);

src/util.h

@@ -626,6 +626,9 @@ int make_socket_nonblocking(int fd);
 int make_socket_nodelay(int fd);
 
 int create_nonblock_socket(int family);
+int create_nonblock_udp_socket(int family);
+
+int bind_any_addr_udp(int fd, int family);
 
 bool check_socket_connected(int fd);