Update man pages

The overall meaning is the same. I added a paragraph to introduce nghttp2_hd_inflate_hd(), and added an explanation of using NGHTTP2_NV_FLAG_NO_INDEX for security sensitive headers.

configure.ac

@@ -25,7 +25,7 @@ dnl Do not change user variables!
 dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html
 
 AC_PREREQ(2.61)
-AC_INIT([nghttp2], [1.2.0], [t-tujikawa@users.sourceforge.net])
+AC_INIT([nghttp2], [1.2.1], [t-tujikawa@users.sourceforge.net])
 AC_CONFIG_AUX_DIR([.])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS([config.h])
@@ -48,7 +48,7 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 dnl See versioning rule:
 dnl  http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
 AC_SUBST(LT_CURRENT, 14)
-AC_SUBST(LT_REVISION, 7)
+AC_SUBST(LT_REVISION, 8)
 AC_SUBST(LT_AGE, 0)
 
 major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"`

doc/h2load.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "H2LOAD" "1" "August 09, 2015" "1.2.0" "nghttp2"
+.TH "H2LOAD" "1" "August 14, 2015" "1.2.1" "nghttp2"
 .SH NAME
 h2load \- HTTP/2 benchmarking tool
 .
@@ -196,9 +196,9 @@ The number of requests failed, including HTTP level failures
 .TP
 .B errored
 The number of requests failed, except for HTTP level failures.
-status code.  This is the subset of the number reported in
-\fBfailed\fP and most likely the network level failures or stream
-was reset by RST_STREAM.
+This is the subset of the number reported in \fBfailed\fP and most
+likely the network level failures or stream was reset by
+RST_STREAM.
 .UNINDENT
 .TP
 .B status codes

doc/h2load.1.rst

@@ -154,9 +154,9 @@ requests
     (non-successful HTTP status code).
   errored
     The number of requests failed, except for HTTP level failures.
-    status code.  This is the subset of the number reported in
-    ``failed`` and most likely the network level failures or stream
-    was reset by RST_STREAM.
+    This is the subset of the number reported in ``failed`` and most
+    likely the network level failures or stream was reset by
+    RST_STREAM.
 
 status codes
   The number of status code h2load received.

doc/h2load.h2r

@@ -16,9 +16,9 @@ requests
     (non-successful HTTP status code).
   errored
     The number of requests failed, except for HTTP level failures.
-    status code.  This is the subset of the number reported in
-    ``failed`` and most likely the network level failures or stream
-    was reset by RST_STREAM.
+    This is the subset of the number reported in ``failed`` and most
+    likely the network level failures or stream was reset by
+    RST_STREAM.
 
 status codes
   The number of status code h2load received.

doc/nghttp.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTP" "1" "August 09, 2015" "1.2.0" "nghttp2"
+.TH "NGHTTP" "1" "August 14, 2015" "1.2.1" "nghttp2"
 .SH NAME
 nghttp \- HTTP/2 experimental client
 .

doc/nghttpd.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTPD" "1" "August 09, 2015" "1.2.0" "nghttp2"
+.TH "NGHTTPD" "1" "August 14, 2015" "1.2.1" "nghttp2"
 .SH NAME
 nghttpd \- HTTP/2 experimental server
 .

doc/nghttpx.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTPX" "1" "August 09, 2015" "1.2.0" "nghttp2"
+.TH "NGHTTPX" "1" "August 14, 2015" "1.2.1" "nghttp2"
 .SH NAME
 nghttpx \- HTTP/2 experimental proxy
 .
@@ -1037,6 +1037,9 @@ translated into Python.
 The script file is usually installed under
 \fB$(prefix)/share/nghttp2/\fP directory.  The actual path to script can
 be customized using \fI\%\-\-fetch\-ocsp\-response\-file\fP option.
+.sp
+If OCSP query is failed, previous OCSP response, if any, is continued
+to be used.
 .SH TLS SESSION RESUMPTION
 .sp
 nghttpx supports TLS session resumption through both session ID and

doc/nghttpx.1.rst

@@ -943,6 +943,9 @@ The script file is usually installed under
 ``$(prefix)/share/nghttp2/`` directory.  The actual path to script can
 be customized using :option:`--fetch-ocsp-response-file` option.
 
+If OCSP query is failed, previous OCSP response, if any, is continued
+to be used.
+
 TLS SESSION RESUMPTION
 ----------------------
 

doc/nghttpx.h2r

@@ -96,6 +96,9 @@ The script file is usually installed under
 ``$(prefix)/share/nghttp2/`` directory.  The actual path to script can
 be customized using :option:`--fetch-ocsp-response-file` option.
 
+If OCSP query is failed, previous OCSP response, if any, is continued
+to be used.
+
 TLS SESSION RESUMPTION
 ----------------------
 

doc/sources/tutorial-hpack.rst

@@ -1,118 +1,129 @@
 Tutorial: HPACK API
 ===================
 
-In this tutorial, we describe basic use of HPACK API in nghttp2
-library.  We briefly describe APIs for deflating and inflating header
-fields.  The example of using these APIs are presented as complete
-source code `deflate.c`_.
+In this tutorial, we describe basic use of nghttp2's HPACK API.  We
+briefly describe the APIs for deflating and inflating header fields.
+The full example of using these APIs, `deflate.c`_, is attached at the
+end of this page. It also resides in the examples directory in the
+archive or repository.
 
 Deflating (encoding) headers
 ----------------------------
 
-First we need to initialize :type:`nghttp2_hd_deflater` object using
-`nghttp2_hd_deflate_new()` function::
+First we need to initialize a :type:`nghttp2_hd_deflater` object using
+the `nghttp2_hd_deflate_new()` function::
 
     int nghttp2_hd_deflate_new(nghttp2_hd_deflater **deflater_ptr,
                                size_t deflate_hd_table_bufsize_max);
 
-This function allocates :type:`nghttp2_hd_deflater` object and
-initializes it and assigns its pointer to ``*deflater_ptr`` passed by
-parameter.  The *deflate_hd_table_bufsize_max* is the upper bound of
-header table size the deflater will use.  This will limit the memory
-usage in deflater object for dynamic header table.  If you doubt, just
+This function allocates a :type:`nghttp2_hd_deflater` object,
+initializes it, and assigns its pointer to ``*deflater_ptr``. The
+*deflate_hd_table_bufsize_max* is the upper bound of header table size
+the deflater will use.  This will limit the memory usage by the
+deflater object for the dynamic header table.  If in doubt, just
 specify 4096 here, which is the default upper bound of dynamic header
 table buffer size.
 
-To encode header fields, `nghttp2_hd_deflate_hd()` function::
+To encode header fields, use the `nghttp2_hd_deflate_hd()` function::
 
     ssize_t nghttp2_hd_deflate_hd(nghttp2_hd_deflater *deflater,
                                   uint8_t *buf, size_t buflen,
                                   const nghttp2_nv *nva, size_t nvlen);
 
 The *deflater* is the deflater object initialized by
-`nghttp2_hd_deflate_new()` function described above.  The *buf* is a
-pointer to buffer to store encoded byte string.  The *buflen* is
-capacity of *buf*.  The *nva* is a pointer to :type:`nghttp2_nv`,
-which is an array of header fields to deflate.  The *nvlen* is the
-number of header fields which *nva* contains.
+`nghttp2_hd_deflate_new()` described above. The encoded byte string is
+written to the buffer *buf*, which has length *buflen*.  The *nva* is
+a pointer to an array of headers fields, each of type
+:type:`nghttp2_nv`.  *nvlen* is the number of header fields which
+*nva* contains.
 
 It is important to initialize and assign all members of
-:type:`nghttp2_nv`.  If a header field should not be inserted in
-dynamic header table for a security reason, set
-:macro:`NGHTTP2_NV_FLAG_NO_INDEX` flag in :member:`nghttp2_nv.flags`.
+:type:`nghttp2_nv`. For security sensitive header fields (such as
+cookies), set the :macro:`NGHTTP2_NV_FLAG_NO_INDEX` flag in
+:member:`nghttp2_nv.flags`.  Setting this flag prevents recovery of
+sensitive header fields by compression based attacks: This is achieved
+by not inserting the header field into the dynamic header table.
 
 `nghttp2_hd_deflate_hd()` processes all headers given in *nva*.  The
 *nva* must include all request or response header fields to be sent in
 one HEADERS (or optionally following (multiple) CONTINUATION
 frame(s)).  The *buf* must have enough space to store the encoded
-result.  Otherwise, the function will fail.  To estimate the upper
-bound of encoded result, use `nghttp2_hd_deflate_bound()` function::
+result, otherwise the function will fail.  To estimate the upper bound
+of the encoded result length, use `nghttp2_hd_deflate_bound()`::
 
     size_t nghttp2_hd_deflate_bound(nghttp2_hd_deflater *deflater,
                                     const nghttp2_nv *nva, size_t nvlen);
 
-Pass this function with the same parameters *deflater*, *nva* and
-*nvlen* which will be passed to `nghttp2_hd_deflate_hd()`.
+Pass this function the same parameters (*deflater*, *nva*, and
+*nvlen*) which will be passed to `nghttp2_hd_deflate_hd()`.
 
-The subsequent call of `nghttp2_hd_deflate_hd()` will use current
-encoder state and perform differential encoding which is the
-fundamental compression gain for HPACK.
+Subsequent calls to `nghttp2_hd_deflate_hd()` will use the current
+encoder state and perform differential encoding, which yields HPAC's
+fundamental compression gain.
 
-Once `nghttp2_hd_deflate_hd()` fails, it cannot be undone and its
-further call with the same deflater object shall fail.  So it is very
-important to use `nghttp2_hd_deflate_bound()` to know the required
-size of buffer.
+If `nghttp2_hd_deflate_hd()` fails, the failure is fatal and any
+further calls with the same deflater object will fail.  Thus it's very
+important to use `nghttp2_hd_deflate_bound()` to determine the
+required size of the output buffer.
 
-To delete :type:`nghttp2_hd_deflater` object, use `nghttp2_hd_deflate_del()`
-function.
+To delete a :type:`nghttp2_hd_deflater` object, use the
+`nghttp2_hd_deflate_del()` function.
 
 Inflating (decoding) headers
 ----------------------------
 
-We use :type:`nghttp2_hd_inflater` object to inflate compressed header
-data.  To initialize the object, use `nghttp2_hd_inflate_new()`::
+A :type:`nghttp2_hd_inflater` object is used to inflate compressed
+header data.  To initialize the object, use
+`nghttp2_hd_inflate_new()`::
 
     int nghttp2_hd_inflate_new(nghttp2_hd_inflater **inflater_ptr);
 
-To inflate header data, use `nghttp2_hd_inflate_hd()` function::
+To inflate header data, use `nghttp2_hd_inflate_hd()`::
 
     ssize_t nghttp2_hd_inflate_hd(nghttp2_hd_inflater *inflater,
                                   nghttp2_nv *nv_out, int *inflate_flags,
                                   uint8_t *in, size_t inlen, int in_final);
 
+`nghttp2_hd_inflate_hd()` reads a stream of bytes and outputs a single
+header field at a time. Multiple calls are normally required to read a
+full stream of bytes and output all of the header fields.
+
 The *inflater* is the inflater object initialized above.  The *nv_out*
-is a pointer to :type:`nghttp2_nv` to store the result.  The *in* is a
-pointer to input data and *inlen* is its length.  The caller is not
-required to specify whole deflated header data to *in* at once.  It
-can call this function multiple times for portion of the data in
-streaming way.  If *in_final* is nonzero, it tells the function that
-the passed data is the final sequence of deflated header data.  The
-*inflate_flags* is output parameter and successful call of this
-function stores a set of flags in it.  It will be described later.
+is a pointer to a :type:`nghttp2_nv` into which one header field may
+be stored.  The *in* is a pointer to input data, and *inlen* is its
+length.  The caller is not required to specify the whole deflated
+header data via *in* at once: Instead it can call this function
+multiple times as additional data bytes become available.  If
+*in_final* is nonzero, it tells the function that the passed data is
+the final sequence of deflated header data.
+
+The *inflate_flags* is an output parameter; on success the function
+sets it to a bitset of flags.  It will be described later.
 
 This function returns when each header field is inflated.  When this
-happens, the function sets :macro:`NGHTTP2_HD_INFLATE_EMIT` flag to
-*inflate_flag* parameter and header field is stored in *nv_out*.  The
-return value indicates the number of data read from *in* to processed
-so far.  It may be less than *inlen*.  The caller should call the
-function repeatedly until all data are processed by adjusting *in* and
-*inlen* with the processed bytes.
+happens, the function sets the :macro:`NGHTTP2_HD_INFLATE_EMIT` flag
+in *inflate_flags*, and a header field is stored in *nv_out*.  The
+return value indicates the number of bytes read from *in* processed so
+far, which may be less than *inlen*.  The caller should call the
+function repeatedly until all bytes are processed. Processed bytes
+should be removed from *in*, and *inlen* should be adjusted
+appropriately.
 
 If *in_final* is nonzero and all given data was processed, the
-function sets :macro:`NGHTTP2_HD_INFLATE_FINAL` flag to
-*inflate_flag*.  If the caller sees this flag set, call
+function sets the :macro:`NGHTTP2_HD_INFLATE_FINAL` flag in
+*inflate_flags*.  When you see this flag set, call the
 `nghttp2_hd_inflate_end_headers()` function.
 
-If *in_final* is zero and :macro:`NGHTTP2_HD_INFLATE_EMIT` flag is not
-set, it indicates that all given data was processed.  The caller is
-required to pass subsequent data.
+If *in_final* is zero and the :macro:`NGHTTP2_HD_INFLATE_EMIT` flag is
+not set, it indicates that all given data was processed.  The caller
+is required to pass additional data.
 
 It is important to note that the function may produce one or more
 header fields even if *inlen* is 0 when *in_final* is nonzero, due to
 differential encoding.
 
-The example use of `nghttp2_hd_inflate_hd()` is shown in
+Example usage of `nghttp2_hd_inflate_hd()` is shown in the
 `inflate_header_block()` function in `deflate.c`_.
 
-To delete :type:`nghttp2_hd_inflater` object, use `nghttp2_hd_inflate_del()`
-function.
+Finally, to delete a :type:`nghttp2_hd_inflater` object, use
+`nghttp2_hd_inflate_del()`.

lib/nghttp2_stream.c

@@ -989,8 +989,14 @@ int nghttp2_stream_dep_insert_subtree(nghttp2_stream *dep_stream,
     return 0;
   }
 
+  /* If dep_stream has stream whose dpri is NGHTTP2_DPRI_TOP in its
+     subtree, parent stream already accounted dep_stream->weight in
+     its sum_norest_weight */
+  if (dep_stream->sum_norest_weight == 0) {
+    stream_update_dep_sum_norest_weight(dep_stream->dep_prev,
+                                        dep_stream->weight);
+  }
   dep_stream->sum_norest_weight = stream->weight;
-  stream_update_dep_sum_norest_weight(dep_stream->dep_prev, dep_stream->weight);
 
   rv = stream_update_dep_queue_top(stream, session);
   if (rv != 0) {

src/asio_server_tls_context.cc

@@ -61,11 +61,13 @@ configure_tls_context_easy(boost::system::error_code &ec,
 
   SSL_CTX_set_cipher_list(ctx, ssl::DEFAULT_CIPHER_LIST);
 
+#ifndef OPENSSL_NO_EC
   auto ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
   if (ecdh) {
     SSL_CTX_set_tmp_ecdh(ctx, ecdh);
     EC_KEY_free(ecdh);
   }
+#endif /* OPENSSL_NO_EC */
 
   SSL_CTX_set_next_protos_advertised_cb(
       ctx,

src/h2load.cc

@@ -737,11 +737,11 @@ Worker::Worker(uint32_t id, SSL_CTX *ssl_ctx, size_t req_todo, size_t nclients,
   auto nreqs_per_client = req_todo / nclients;
   auto nreqs_rem = req_todo % nclients;
 
-  if (config->is_rate_mode()) {
-    // create timer that will go off every second
-    ev_timer_init(&timeout_watcher, second_timeout_w_cb, 0., 1.);
-    timeout_watcher.data = this;
-  } else {
+  // create timer that will go off every second
+  ev_timer_init(&timeout_watcher, second_timeout_w_cb, 0., 1.);
+  timeout_watcher.data = this;
+
+  if (!config->is_rate_mode()) {
     for (size_t i = 0; i < nclients; ++i) {
       auto req_todo = nreqs_per_client;
       if (nreqs_rem > 0) {
@@ -754,6 +754,8 @@ Worker::Worker(uint32_t id, SSL_CTX *ssl_ctx, size_t req_todo, size_t nclients,
 }
 
 Worker::~Worker() {
+  ev_timer_stop(loop, &timeout_watcher);
+
   // first clear clients so that io watchers are stopped before
   // destructing ev_loop.
   clients.clear();
@@ -1017,6 +1019,8 @@ namespace {
 void print_help(std::ostream &out) {
   print_usage(out);
 
+  auto config = Config();
+
   out << R"(
   <URI>       Specify URI to access.   Multiple URIs can be specified.
               URIs are used  in this order for each  client.  All URIs
@@ -1113,6 +1117,7 @@ int main(int argc, char **argv) {
   OPENSSL_config(nullptr);
 
   std::string datafile;
+  bool nreqs_set_manually = false;
   while (1) {
     static int flag = 0;
     static option long_options[] = {
@@ -1142,6 +1147,7 @@ int main(int argc, char **argv) {
     switch (c) {
     case 'n':
       config.nreqs = strtoul(optarg, nullptr, 10);
+      nreqs_set_manually = true;
       break;
     case 'c':
       config.nclients = strtoul(optarg, nullptr, 10);
@@ -1320,6 +1326,12 @@ int main(int argc, char **argv) {
       exit(EXIT_FAILURE);
     }
 
+    if (nreqs_set_manually && config.rate > config.nreqs) {
+      std::cerr << "-r, -n: the connection rate must be smaller than or equal "
+                   "to the number of requests." << std::endl;
+      exit(EXIT_FAILURE);
+    }
+
     if (config.nconns != 0 && config.nconns < config.nthreads) {
       std::cerr
           << "-C, -t: the total number of connections must be greater than "
@@ -1327,6 +1339,13 @@ int main(int argc, char **argv) {
           << "to the number of threads." << std::endl;
       exit(EXIT_FAILURE);
     }
+
+    if (config.nconns == 0 && !nreqs_set_manually) {
+      std::cerr
+          << "-r: the rate option must be used with either the -n option "
+             "or the -C option." << std::endl;
+      exit(EXIT_FAILURE);
+    }
   }
 
   if (!datafile.empty()) {
@@ -1666,9 +1685,10 @@ int main(int argc, char **argv) {
   double rps = 0;
   int64_t bps = 0;
   if (duration.count() > 0) {
-    auto secd = static_cast<double>(duration.count()) / (1000 * 1000);
-    rps = stats.req_success / secd;
-    bps = stats.bytes_total / secd;
+    auto secd = std::chrono::duration_cast<
+        std::chrono::duration<double, std::chrono::seconds::period>>(duration);
+    rps = stats.req_success / secd.count();
+    bps = stats.bytes_total / secd.count();
   }
 
   std::cout << R"(

src/memchunk.h

@@ -117,6 +117,31 @@ template <typename T> struct Pool {
 template <typename Memchunk> struct Memchunks {
   Memchunks(Pool<Memchunk> *pool)
       : pool(pool), head(nullptr), tail(nullptr), len(0) {}
+  Memchunks(const Memchunks &) = delete;
+  Memchunks(Memchunks &&other)
+      : pool(other.pool), head(other.head), tail(other.head), len(other.len) {
+    // keep other.pool
+    other.head = other.tail = nullptr;
+    other.len = 0;
+  }
+  Memchunks &operator=(const Memchunks &) = delete;
+  Memchunks &operator=(Memchunks &&other) {
+    if (this == &other) {
+      return *this;
+    }
+
+    reset();
+
+    pool = other.pool;
+    head = other.head;
+    tail = other.tail;
+    len = other.len;
+
+    other.head = other.tail = nullptr;
+    other.len = 0;
+
+    return *this;
+  }
   ~Memchunks() {
     if (!pool) {
       return;
@@ -223,15 +248,142 @@ template <typename Memchunk> struct Memchunks {
     return i;
   }
   size_t rleft() const { return len; }
+  void reset() {
+    for (auto m = head; m;) {
+      auto next = m->next;
+      pool->recycle(m);
+      m = next;
+    }
+    len = 0;
+    head = tail = nullptr;
+  }
 
   Pool<Memchunk> *pool;
   Memchunk *head, *tail;
   size_t len;
 };
 
+// Wrapper around Memchunks to offer "peeking" functionality.
+template <typename Memchunk> struct PeekMemchunks {
+  PeekMemchunks(Pool<Memchunk> *pool)
+      : memchunks(pool), cur(nullptr), cur_pos(nullptr), cur_last(nullptr),
+        len(0), peeking(true) {}
+  PeekMemchunks(const PeekMemchunks &) = delete;
+  PeekMemchunks(PeekMemchunks &&other)
+      : memchunks(std::move(other.memchunks)), cur(other.cur),
+        cur_pos(other.cur_pos), cur_last(other.cur_last), len(other.len),
+        peeking(other.peeking) {
+    other.reset();
+  }
+  PeekMemchunks &operator=(const PeekMemchunks &) = delete;
+  PeekMemchunks &operator=(PeekMemchunks &&other) {
+    if (this == &other) {
+      return *this;
+    }
+
+    memchunks = std::move(other.memchunks);
+    cur = other.cur;
+    cur_pos = other.cur_pos;
+    cur_last = other.cur_last;
+    len = other.len;
+    peeking = other.peeking;
+
+    other.reset();
+
+    return *this;
+  }
+  size_t append(const void *src, size_t count) {
+    count = memchunks.append(src, count);
+    len += count;
+    return count;
+  }
+  size_t remove(void *dest, size_t count) {
+    if (!peeking) {
+      count = memchunks.remove(dest, count);
+      len -= count;
+      return count;
+    }
+
+    if (count == 0 || len == 0) {
+      return 0;
+    }
+
+    if (!cur) {
+      cur = memchunks.head;
+      cur_pos = cur->pos;
+    }
+
+    // cur_last could be updated in append
+    cur_last = cur->last;
+
+    if (cur_pos == cur_last) {
+      assert(cur->next);
+      cur = cur->next;
+    }
+
+    auto first = static_cast<uint8_t *>(dest);
+    auto last = first + count;
+
+    for (;;) {
+      auto n = std::min(last - first, cur_last - cur_pos);
+
+      first = std::copy_n(cur_pos, n, first);
+      cur_pos += n;
+      len -= n;
+
+      if (first == last) {
+        break;
+      }
+      assert(cur_pos == cur_last);
+      if (!cur->next) {
+        break;
+      }
+      cur = cur->next;
+      cur_pos = cur->pos;
+      cur_last = cur->last;
+    }
+    return first - static_cast<uint8_t *>(dest);
+  }
+  size_t rleft() const { return len; }
+  size_t rleft_buffered() const { return memchunks.rleft(); }
+  void disable_peek(bool drain) {
+    if (!peeking) {
+      return;
+    }
+    if (drain) {
+      auto n = rleft_buffered() - rleft();
+      memchunks.drain(n);
+      assert(len == memchunks.rleft());
+    } else {
+      len = memchunks.rleft();
+    }
+    cur = nullptr;
+    cur_pos = cur_last = nullptr;
+    peeking = false;
+  }
+  void reset() {
+    memchunks.reset();
+    cur = nullptr;
+    cur_pos = cur_last = nullptr;
+    len = 0;
+    peeking = true;
+  }
+  Memchunks<Memchunk> memchunks;
+  // Pointer to the Memchunk currently we are reading/writing.
+  Memchunk *cur;
+  // Region inside cur, we have processed to cur_pos.
+  uint8_t *cur_pos, *cur_last;
+  // This is the length we have left unprocessed.  len <=
+  // memchunk.rleft() must hold.
+  size_t len;
+  // true if peeking is enabled.  Initially it is true.
+  bool peeking;
+};
+
 using Memchunk16K = Memchunk<16_k>;
 using MemchunkPool = Pool<Memchunk16K>;
 using DefaultMemchunks = Memchunks<Memchunk16K>;
+using DefaultPeekMemchunks = PeekMemchunks<Memchunk16K>;
 
 #define DEFAULT_WR_IOVCNT 16
 

src/memchunk_test.cc

@@ -84,6 +84,7 @@ void test_pool_recycle(void) {
 using Memchunk16 = Memchunk<16>;
 using MemchunkPool16 = Pool<Memchunk16>;
 using Memchunks16 = Memchunks<Memchunk16>;
+using PeekMemchunks16 = PeekMemchunks<Memchunk16>;
 
 void test_memchunks_append(void) {
   MemchunkPool16 pool;
@@ -196,4 +197,144 @@ void test_memchunks_recycle(void) {
   CU_ASSERT(nullptr == m->next);
 }
 
+void test_memchunks_reset(void) {
+  MemchunkPool16 pool;
+  Memchunks16 chunks(&pool);
+
+  std::array<uint8_t, 32> b{};
+
+  chunks.append(b.data(), b.size());
+
+  CU_ASSERT(32 == chunks.rleft());
+
+  chunks.reset();
+
+  CU_ASSERT(0 == chunks.rleft());
+  CU_ASSERT(nullptr == chunks.head);
+  CU_ASSERT(nullptr == chunks.tail);
+
+  auto m = pool.freelist;
+
+  CU_ASSERT(nullptr != m);
+  CU_ASSERT(nullptr != m->next);
+  CU_ASSERT(nullptr == m->next->next);
+}
+
+void test_peek_memchunks_append(void) {
+  MemchunkPool16 pool;
+  PeekMemchunks16 pchunks(&pool);
+
+  std::array<uint8_t, 32> b{{
+      '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4',
+      '5', '6', '7', '8', '9', '0', '1',
+  }},
+      d;
+
+  pchunks.append(b.data(), b.size());
+
+  CU_ASSERT(32 == pchunks.rleft());
+  CU_ASSERT(32 == pchunks.rleft_buffered());
+
+  CU_ASSERT(0 == pchunks.remove(nullptr, 0));
+
+  CU_ASSERT(32 == pchunks.rleft());
+  CU_ASSERT(32 == pchunks.rleft_buffered());
+
+  CU_ASSERT(12 == pchunks.remove(d.data(), 12));
+
+  CU_ASSERT(std::equal(std::begin(b), std::begin(b) + 12, std::begin(d)));
+
+  CU_ASSERT(20 == pchunks.rleft());
+  CU_ASSERT(32 == pchunks.rleft_buffered());
+
+  CU_ASSERT(20 == pchunks.remove(d.data(), d.size()));
+
+  CU_ASSERT(std::equal(std::begin(b) + 12, std::end(b), std::begin(d)));
+
+  CU_ASSERT(0 == pchunks.rleft());
+  CU_ASSERT(32 == pchunks.rleft_buffered());
+}
+
+void test_peek_memchunks_disable_peek_drain(void) {
+  MemchunkPool16 pool;
+  PeekMemchunks16 pchunks(&pool);
+
+  std::array<uint8_t, 32> b{{
+      '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4',
+      '5', '6', '7', '8', '9', '0', '1',
+  }},
+      d;
+
+  pchunks.append(b.data(), b.size());
+
+  CU_ASSERT(12 == pchunks.remove(d.data(), 12));
+
+  pchunks.disable_peek(true);
+
+  CU_ASSERT(!pchunks.peeking);
+  CU_ASSERT(20 == pchunks.rleft());
+  CU_ASSERT(20 == pchunks.rleft_buffered());
+
+  CU_ASSERT(20 == pchunks.remove(d.data(), d.size()));
+
+  CU_ASSERT(std::equal(std::begin(b) + 12, std::end(b), std::begin(d)));
+
+  CU_ASSERT(0 == pchunks.rleft());
+  CU_ASSERT(0 == pchunks.rleft_buffered());
+}
+
+void test_peek_memchunks_disable_peek_no_drain(void) {
+  MemchunkPool16 pool;
+  PeekMemchunks16 pchunks(&pool);
+
+  std::array<uint8_t, 32> b{{
+      '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4',
+      '5', '6', '7', '8', '9', '0', '1',
+  }},
+      d;
+
+  pchunks.append(b.data(), b.size());
+
+  CU_ASSERT(12 == pchunks.remove(d.data(), 12));
+
+  pchunks.disable_peek(false);
+
+  CU_ASSERT(!pchunks.peeking);
+  CU_ASSERT(32 == pchunks.rleft());
+  CU_ASSERT(32 == pchunks.rleft_buffered());
+
+  CU_ASSERT(32 == pchunks.remove(d.data(), d.size()));
+
+  CU_ASSERT(std::equal(std::begin(b), std::end(b), std::begin(d)));
+
+  CU_ASSERT(0 == pchunks.rleft());
+  CU_ASSERT(0 == pchunks.rleft_buffered());
+}
+
+void test_peek_memchunks_reset(void) {
+  MemchunkPool16 pool;
+  PeekMemchunks16 pchunks(&pool);
+
+  std::array<uint8_t, 32> b{{
+      '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4',
+      '5', '6', '7', '8', '9', '0', '1',
+  }},
+      d;
+
+  pchunks.append(b.data(), b.size());
+
+  CU_ASSERT(12 == pchunks.remove(d.data(), 12));
+
+  pchunks.disable_peek(true);
+  pchunks.reset();
+
+  CU_ASSERT(0 == pchunks.rleft());
+  CU_ASSERT(0 == pchunks.rleft_buffered());
+
+  CU_ASSERT(nullptr == pchunks.cur);
+  CU_ASSERT(nullptr == pchunks.cur_pos);
+  CU_ASSERT(nullptr == pchunks.cur_last);
+  CU_ASSERT(pchunks.peeking);
+}
+
 } // namespace nghttp2

src/memchunk_test.h

@@ -36,6 +36,11 @@ void test_memchunks_append(void);
 void test_memchunks_drain(void);
 void test_memchunks_riovec(void);
 void test_memchunks_recycle(void);
+void test_memchunks_reset(void);
+void test_peek_memchunks_append(void);
+void test_peek_memchunks_disable_peek_drain(void);
+void test_peek_memchunks_disable_peek_no_drain(void);
+void test_peek_memchunks_reset(void);
 
 } // namespace nghttp2
 

src/shrpx-unittest.cc

@@ -172,7 +172,16 @@ int main(int argc, char *argv[]) {
       !CU_add_test(pSuite, "memchunk_drain", nghttp2::test_memchunks_drain) ||
       !CU_add_test(pSuite, "memchunk_riovec", nghttp2::test_memchunks_riovec) ||
       !CU_add_test(pSuite, "memchunk_recycle",
-                   nghttp2::test_memchunks_recycle)) {
+                   nghttp2::test_memchunks_recycle) ||
+      !CU_add_test(pSuite, "memchunk_reset", nghttp2::test_memchunks_reset) ||
+      !CU_add_test(pSuite, "peek_memchunk_append",
+                   nghttp2::test_peek_memchunks_append) ||
+      !CU_add_test(pSuite, "peek_memchunk_disable_peek_drain",
+                   nghttp2::test_peek_memchunks_disable_peek_drain) ||
+      !CU_add_test(pSuite, "peek_memchunk_disable_peek_no_drain",
+                   nghttp2::test_peek_memchunks_disable_peek_no_drain) ||
+      !CU_add_test(pSuite, "peek_memchunk_reset",
+                   nghttp2::test_peek_memchunks_reset)) {
     CU_cleanup_registry();
     return CU_get_error();
   }

src/shrpx.cc

@@ -775,7 +775,7 @@ void memcached_get_ticket_key_cb(struct ev_loop *loop, ev_timer *w,
       auto key = TicketKey();
       key.cipher = get_config()->tls_ticket_key_cipher;
       key.hmac = EVP_sha256();
-      key.hmac_keylen = EVP_MD_size(key.hmac);
+      key.hmac_keylen = hmac_keylen;
 
       std::copy_n(p, key.data.name.size(), key.data.name.data());
       p += key.data.name.size();

src/shrpx_client_handler.cc

@@ -358,7 +358,8 @@ int ClientHandler::upstream_http1_connhd_read() {
 
 ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
                              const char *ipaddr, const char *port)
-    : conn_(worker->get_loop(), fd, ssl, get_config()->upstream_write_timeout,
+    : conn_(worker->get_loop(), fd, ssl, worker->get_mcpool(),
+            get_config()->upstream_write_timeout,
             get_config()->upstream_read_timeout, get_config()->write_rate,
             get_config()->write_burst, get_config()->read_rate,
             get_config()->read_burst, writecb, readcb, timeoutcb, this),
@@ -849,6 +850,4 @@ ev_io *ClientHandler::get_wev() { return &conn_.wev; }
 
 Worker *ClientHandler::get_worker() const { return worker_; }
 
-Connection *ClientHandler::get_connection() { return &conn_; }
-
 } // namespace shrpx

src/shrpx_client_handler.h

@@ -130,8 +130,6 @@ public:
   void signal_write();
   ev_io *get_wev();
 
-  Connection *get_connection();
-
 private:
   Connection conn_;
   ev_timer reneg_shutdown_timer_;

src/shrpx_connection.cc

@@ -40,12 +40,13 @@ using namespace nghttp2;
 
 namespace shrpx {
 Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl,
-                       ev_tstamp write_timeout, ev_tstamp read_timeout,
-                       size_t write_rate, size_t write_burst, size_t read_rate,
-                       size_t read_burst, IOCb writecb, IOCb readcb,
-                       TimerCb timeoutcb, void *data)
-    : tls{}, wlimit(loop, &wev, write_rate, write_burst),
-      rlimit(loop, &rev, read_rate, read_burst, ssl), writecb(writecb),
+                       MemchunkPool *mcpool, ev_tstamp write_timeout,
+                       ev_tstamp read_timeout, size_t write_rate,
+                       size_t write_burst, size_t read_rate, size_t read_burst,
+                       IOCb writecb, IOCb readcb, TimerCb timeoutcb, void *data)
+    : tls{DefaultMemchunks(mcpool), DefaultPeekMemchunks(mcpool)},
+      wlimit(loop, &wev, write_rate, write_burst),
+      rlimit(loop, &rev, read_rate, read_burst, this), writecb(writecb),
       readcb(readcb), timeoutcb(timeoutcb), loop(loop), data(data), fd(fd) {
 
   ev_io_init(&wev, writecb, fd, EV_WRITE);
@@ -77,22 +78,18 @@ Connection::~Connection() {
 }
 
 void Connection::disconnect() {
-  ev_timer_stop(loop, &rt);
-  ev_timer_stop(loop, &wt);
-
-  rlimit.stopw();
-  wlimit.stopw();
-
   if (tls.ssl) {
     SSL_set_shutdown(tls.ssl, SSL_RECEIVED_SHUTDOWN);
     ERR_clear_error();
 
     if (tls.cached_session) {
       SSL_SESSION_free(tls.cached_session);
+      tls.cached_session = nullptr;
     }
 
     if (tls.cached_session_lookup_req) {
       tls.cached_session_lookup_req->canceled = true;
+      tls.cached_session_lookup_req = nullptr;
     }
 
     // To reuse SSL/TLS session, we have to shutdown, and don't free
@@ -102,7 +99,15 @@ void Connection::disconnect() {
       tls.ssl = nullptr;
     }
 
-    tls = {tls.ssl};
+    tls.wbuf.reset();
+    tls.rbuf.reset();
+    tls.last_write_idle = 0.;
+    tls.warmup_writelen = 0;
+    tls.last_writelen = 0;
+    tls.last_readlen = 0;
+    tls.handshake_state = 0;
+    tls.initial_handshake_done = false;
+    tls.reneg_started = false;
   }
 
   if (fd != -1) {
@@ -110,24 +115,19 @@ void Connection::disconnect() {
     close(fd);
     fd = -1;
   }
+
+  // Stop watchers here because they could be activated in
+  // SSL_shutdown().
+  ev_timer_stop(loop, &rt);
+  ev_timer_stop(loop, &wt);
+
+  rlimit.stopw();
+  wlimit.stopw();
 }
 
-namespace {
-void allocate_buffer(Connection *conn) {
-  conn->tls.rb = make_unique<Buffer<16_k>>();
-  conn->tls.wb = make_unique<Buffer<16_k>>();
-}
-} // namespace
+void Connection::prepare_client_handshake() { SSL_set_connect_state(tls.ssl); }
 
-void Connection::prepare_client_handshake() {
-  SSL_set_connect_state(tls.ssl);
-  allocate_buffer(this);
-}
-
-void Connection::prepare_server_handshake() {
-  SSL_set_accept_state(tls.ssl);
-  allocate_buffer(this);
-}
+void Connection::prepare_server_handshake() { SSL_set_accept_state(tls.ssl); }
 
 // BIO implementation is inspired by openldap implementation:
 // http://www.openldap.org/devel/cvsweb.cgi/~checkout~/libraries/libldap/tls_o.c
@@ -138,27 +138,26 @@ int shrpx_bio_write(BIO *b, const char *buf, int len) {
   }
 
   auto conn = static_cast<Connection *>(b->ptr);
-  auto &wb = conn->tls.wb;
+  auto &wbuf = conn->tls.wbuf;
 
   BIO_clear_retry_flags(b);
 
   if (conn->tls.initial_handshake_done) {
     // After handshake finished, send |buf| of length |len| to the
     // socket directly.
-    if (wb && wb->rleft()) {
-      auto nwrite = conn->write_clear(wb->pos, wb->rleft());
+    if (wbuf.rleft()) {
+      std::array<struct iovec, 4> iov;
+      auto iovcnt = wbuf.riovec(iov.data(), iov.size());
+      auto nwrite = conn->writev_clear(iov.data(), iovcnt);
       if (nwrite < 0) {
         return -1;
       }
 
-      wb->drain(nwrite);
-      if (wb->rleft()) {
+      wbuf.drain(nwrite);
+      if (wbuf.rleft()) {
         BIO_set_retry_write(b);
         return -1;
       }
-
-      // Here delete TLS write buffer
-      wb.reset();
     }
     auto nwrite = conn->write_clear(buf, len);
     if (nwrite < 0) {
@@ -173,16 +172,9 @@ int shrpx_bio_write(BIO *b, const char *buf, int len) {
     return nwrite;
   }
 
-  auto nwrite = std::min(static_cast<size_t>(len), wb->wleft());
+  wbuf.append(buf, len);
 
-  if (nwrite == 0) {
-    BIO_set_retry_write(b);
-    return -1;
-  }
-
-  wb->write(buf, nwrite);
-
-  return nwrite;
+  return len;
 }
 } // namespace
 
@@ -193,11 +185,11 @@ int shrpx_bio_read(BIO *b, char *buf, int len) {
   }
 
   auto conn = static_cast<Connection *>(b->ptr);
-  auto &rb = conn->tls.rb;
+  auto &rbuf = conn->tls.rbuf;
 
   BIO_clear_retry_flags(b);
 
-  if (conn->tls.initial_handshake_done && !rb) {
+  if (conn->tls.initial_handshake_done && rbuf.rleft() == 0) {
     auto nread = conn->read_clear(buf, len);
     if (nread < 0) {
       return -1;
@@ -209,22 +201,12 @@ int shrpx_bio_read(BIO *b, char *buf, int len) {
     return nread;
   }
 
-  auto nread = std::min(static_cast<size_t>(len), rb->rleft());
-
-  if (nread == 0) {
-    if (conn->tls.initial_handshake_done) {
-      rb.reset();
-    }
-
+  if (rbuf.rleft() == 0) {
     BIO_set_retry_read(b);
     return -1;
   }
 
-  std::copy_n(rb->pos, nread, buf);
-
-  rb->drain(nread);
-
-  return nread;
+  return rbuf.remove(buf, len);
 }
 } // namespace
 
@@ -287,42 +269,59 @@ void Connection::set_ssl(SSL *ssl) {
   bio->ptr = this;
   SSL_set_bio(tls.ssl, bio, bio);
   SSL_set_app_data(tls.ssl, this);
-  rlimit.set_ssl(tls.ssl);
 }
 
+namespace {
+// We should buffer at least full encrypted TLS record here.
+// Theoretically, peer can send client hello in several TLS records,
+// which could exeed this limit, but it is not portable, and we don't
+// have to handle such exotic behaviour.
+bool read_buffer_full(DefaultPeekMemchunks &rbuf) {
+  return rbuf.rleft_buffered() >= 20_k;
+}
+} // namespace
+
 int Connection::tls_handshake() {
   wlimit.stopw();
   ev_timer_stop(loop, &wt);
 
-  auto nread = read_clear(tls.rb->last, tls.rb->wleft());
-  if (nread < 0) {
-    if (LOG_ENABLED(INFO)) {
-      LOG(INFO) << "tls: handshake read error";
+  if (ev_is_active(&rev)) {
+    std::array<uint8_t, 8_k> buf;
+    auto nread = read_clear(buf.data(), buf.size());
+    if (nread < 0) {
+      if (LOG_ENABLED(INFO)) {
+        LOG(INFO) << "tls: handshake read error";
+      }
+      return -1;
+    }
+    tls.rbuf.append(buf.data(), nread);
+    if (read_buffer_full(tls.rbuf)) {
+      rlimit.stopw();
     }
-    return -1;
   }
-  tls.rb->write(nread);
 
   switch (tls.handshake_state) {
   case TLS_CONN_WAIT_FOR_SESSION_CACHE:
-    if (tls.rb->wleft() == 0) {
-      // Input buffer is full.  Disable read until cache is returned
-      rlimit.stopw();
-      ev_timer_stop(loop, &rt);
-    }
     return SHRPX_ERR_INPROGRESS;
   case TLS_CONN_GOT_SESSION_CACHE: {
-    // Use the same trick invented by @kazuho in h2o project
-    tls.wb->reset();
-    tls.rb->pos = tls.rb->begin();
+    // Use the same trick invented by @kazuho in h2o project.
+
+    // Discard all outgoing data.
+    tls.wbuf.reset();
+    // Rewind buffered incoming data to replay client hello.
+    tls.rbuf.disable_peek(false);
 
     auto ssl_ctx = SSL_get_SSL_CTX(tls.ssl);
+    auto ssl_opts = SSL_get_options(tls.ssl);
     SSL_free(tls.ssl);
 
-    auto ssl = ssl::create_server_ssl(ssl_ctx, nullptr);
+    auto ssl = ssl::create_ssl(ssl_ctx);
     if (!ssl) {
       return -1;
     }
+    if (ssl_opts & SSL_OP_NO_TICKET) {
+      SSL_set_options(ssl, SSL_OP_NO_TICKET);
+    }
 
     set_ssl(ssl);
 
@@ -342,6 +341,13 @@ int Connection::tls_handshake() {
     auto err = SSL_get_error(tls.ssl, rv);
     switch (err) {
     case SSL_ERROR_WANT_READ:
+      if (read_buffer_full(tls.rbuf)) {
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "tls: handshake message is too large";
+        }
+        return -1;
+      }
+      break;
     case SSL_ERROR_WANT_WRITE:
       break;
     default:
@@ -359,20 +365,33 @@ int Connection::tls_handshake() {
     return SHRPX_ERR_INPROGRESS;
   }
 
-  if (tls.wb->rleft()) {
-    auto nwrite = write_clear(tls.wb->pos, tls.wb->rleft());
+  if (tls.wbuf.rleft()) {
+    // First write indicates that resumption stuff has done.
+    if (tls.handshake_state != TLS_CONN_WRITE_STARTED) {
+      tls.handshake_state = TLS_CONN_WRITE_STARTED;
+      // If peek has already disabled, this is noop.
+      tls.rbuf.disable_peek(true);
+    }
+    std::array<struct iovec, 4> iov;
+    auto iovcnt = tls.wbuf.riovec(iov.data(), iov.size());
+    auto nwrite = writev_clear(iov.data(), iovcnt);
     if (nwrite < 0) {
       if (LOG_ENABLED(INFO)) {
         LOG(INFO) << "tls: handshake write error";
       }
       return -1;
     }
-    tls.wb->drain(nwrite);
+    tls.wbuf.drain(nwrite);
+
+    if (tls.wbuf.rleft()) {
+      wlimit.startw();
+      ev_timer_again(loop, &wt);
+    }
   }
 
-  if (tls.wb->rleft()) {
-    wlimit.startw();
-    ev_timer_again(loop, &wt);
+  if (!read_buffer_full(tls.rbuf)) {
+    // We may have stopped reading
+    rlimit.startw();
   }
 
   if (rv != 1) {
@@ -384,6 +403,15 @@ int Connection::tls_handshake() {
 
   tls.initial_handshake_done = true;
 
+  // We have to start read watcher, since later stage of code expects
+  // this.
+  rlimit.startw();
+
+  // We may have whole request in tls.rbuf.  This means that we don't
+  // get notified further read event.  This is especially true for
+  // HTTP/1.1.
+  handle_tls_pending_read();
+
   if (LOG_ENABLED(INFO)) {
     LOG(INFO) << "SSL/TLS handshake completed";
     if (SSL_session_reused(tls.ssl)) {
@@ -463,8 +491,8 @@ ssize_t Connection::write_tls(const void *data, size_t len) {
       return SHRPX_ERR_NETWORK;
     case SSL_ERROR_WANT_WRITE:
       tls.last_writelen = len;
-      wlimit.startw();
-      ev_timer_again(loop, &wt);
+      // starting write watcher and timer is done in write_clear via
+      // bio.
       return 0;
     default:
       if (LOG_ENABLED(INFO)) {

src/shrpx_connection.h

@@ -35,7 +35,7 @@
 
 #include "shrpx_rate_limit.h"
 #include "shrpx_error.h"
-#include "buffer.h"
+#include "memchunk.h"
 
 namespace shrpx {
 
@@ -46,9 +46,12 @@ enum {
   TLS_CONN_WAIT_FOR_SESSION_CACHE,
   TLS_CONN_GOT_SESSION_CACHE,
   TLS_CONN_CANCEL_SESSION_CACHE,
+  TLS_CONN_WRITE_STARTED,
 };
 
 struct TLSConnection {
+  DefaultMemchunks wbuf;
+  DefaultPeekMemchunks rbuf;
   SSL *ssl;
   SSL_SESSION *cached_session;
   MemcachedRequest *cached_session_lookup_req;
@@ -61,8 +64,6 @@ struct TLSConnection {
   int handshake_state;
   bool initial_handshake_done;
   bool reneg_started;
-  std::unique_ptr<Buffer<16_k>> rb;
-  std::unique_ptr<Buffer<16_k>> wb;
 };
 
 template <typename T> using EVCb = void (*)(struct ev_loop *, T *, int);
@@ -71,10 +72,10 @@ using IOCb = EVCb<ev_io>;
 using TimerCb = EVCb<ev_timer>;
 
 struct Connection {
-  Connection(struct ev_loop *loop, int fd, SSL *ssl, ev_tstamp write_timeout,
-             ev_tstamp read_timeout, size_t write_rate, size_t write_burst,
-             size_t read_rate, size_t read_burst, IOCb writecb, IOCb readcb,
-             TimerCb timeoutcb, void *data);
+  Connection(struct ev_loop *loop, int fd, SSL *ssl, MemchunkPool *mcpool,
+             ev_tstamp write_timeout, ev_tstamp read_timeout, size_t write_rate,
+             size_t write_burst, size_t read_rate, size_t read_burst,
+             IOCb writecb, IOCb readcb, TimerCb timeoutcb, void *data);
   ~Connection();
 
   void disconnect();

src/shrpx_http2_session.cc

@@ -144,7 +144,8 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) {
 Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
                            ConnectBlocker *connect_blocker, Worker *worker,
                            size_t group, size_t idx)
-    : conn_(loop, -1, nullptr, get_config()->downstream_write_timeout,
+    : conn_(loop, -1, nullptr, worker->get_mcpool(),
+            get_config()->downstream_write_timeout,
             get_config()->downstream_read_timeout, 0, 0, 0, 0, writecb, readcb,
             timeoutcb, this),
       worker_(worker), connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx),
@@ -323,7 +324,7 @@ int Http2Session::initiate_connection() {
       // We are establishing TLS connection.  If conn_.tls.ssl, we may
       // reuse the previous session.
       if (!conn_.tls.ssl) {
-        auto ssl = ssl::create_client_ssl(ssl_ctx_);
+        auto ssl = ssl::create_ssl(ssl_ctx_);
         if (!ssl) {
           return -1;
         }

src/shrpx_http_downstream_connection.cc

@@ -112,7 +112,7 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) {
 HttpDownstreamConnection::HttpDownstreamConnection(
     DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop)
     : DownstreamConnection(dconn_pool),
-      conn_(loop, -1, nullptr, get_config()->downstream_write_timeout,
+      conn_(loop, -1, nullptr, nullptr, get_config()->downstream_write_timeout,
             get_config()->downstream_read_timeout, 0, 0, 0, 0, connectcb,
             readcb, timeoutcb, this),
       ioctrl_(&conn_.rlimit), response_htp_{0}, group_(group), addr_idx_(0),

src/shrpx_memcached_connection.cc

@@ -92,7 +92,7 @@ constexpr ev_tstamp read_timeout = 10.;
 
 MemcachedConnection::MemcachedConnection(const Address *addr,
                                          struct ev_loop *loop)
-    : conn_(loop, -1, nullptr, write_timeout, read_timeout, 0, 0, 0, 0,
+    : conn_(loop, -1, nullptr, nullptr, write_timeout, read_timeout, 0, 0, 0, 0,
             connectcb, readcb, timeoutcb, this),
       parse_state_{}, addr_(addr), sendsum_(0), connected_(false) {}
 
@@ -403,6 +403,7 @@ int MemcachedConnection::parse_packet() {
   return 0;
 }
 
+#undef DEFAULT_WR_IOVCNT
 #define DEFAULT_WR_IOVCNT 128
 
 #if defined(IOV_MAX) && IOV_MAX < DEFAULT_WR_IOVCNT

src/shrpx_rate_limit.cc

@@ -26,6 +26,8 @@
 
 #include <limits>
 
+#include "shrpx_connection.h"
+
 namespace shrpx {
 
 namespace {
@@ -36,9 +38,9 @@ void regencb(struct ev_loop *loop, ev_timer *w, int revents) {
 } // namespace
 
 RateLimit::RateLimit(struct ev_loop *loop, ev_io *w, size_t rate, size_t burst,
-                     SSL *ssl)
-    : w_(w), loop_(loop), ssl_(ssl), rate_(rate), burst_(burst), avail_(burst),
-      startw_req_(false) {
+                     Connection *conn)
+    : w_(w), loop_(loop), conn_(conn), rate_(rate), burst_(burst),
+      avail_(burst), startw_req_(false) {
   ev_timer_init(&t_, regencb, 0., 1.);
   t_.data = this;
   if (rate_ > 0) {
@@ -97,7 +99,8 @@ void RateLimit::stopw() {
 }
 
 void RateLimit::handle_tls_pending_read() {
-  if (!ssl_ || SSL_pending(ssl_) == 0) {
+  if (!conn_ || !conn_->tls.ssl ||
+      (SSL_pending(conn_->tls.ssl) == 0 && conn_->tls.rbuf.rleft() == 0)) {
     return;
   }
 
@@ -106,6 +109,4 @@ void RateLimit::handle_tls_pending_read() {
   ev_feed_event(loop_, w_, EV_READ);
 }
 
-void RateLimit::set_ssl(SSL *ssl) { ssl_ = ssl; }
-
 } // namespace shrpx

src/shrpx_rate_limit.h

@@ -33,28 +33,30 @@
 
 namespace shrpx {
 
+struct Connection;
+
 class RateLimit {
 public:
-  // We need |ssl| object to check that it has unread decrypted bytes.
+  // We need |conn| object to check that it has unread bytes for TLS
+  // connection.
   RateLimit(struct ev_loop *loop, ev_io *w, size_t rate, size_t burst,
-            SSL *ssl = nullptr);
+            Connection *conn = nullptr);
   ~RateLimit();
   size_t avail() const;
   void drain(size_t n);
   void regen();
   void startw();
   void stopw();
-  // Feeds event if ssl_ object has unread decrypted bytes.  This is
-  // required since it is buffered in ssl_ object, io event is not
-  // generated unless new incoming data is received.
+  // Feeds event if conn_->tls object has unread bytes.  This is
+  // required since it is buffered in conn_->tls object, io event is
+  // not generated unless new incoming data is received.
   void handle_tls_pending_read();
-  void set_ssl(SSL *ssl);
 
 private:
   ev_timer t_;
   ev_io *w_;
   struct ev_loop *loop_;
-  SSL *ssl_;
+  Connection *conn_;
   size_t rate_;
   size_t burst_;
   size_t avail_;

src/shrpx_ssl.cc

@@ -134,7 +134,8 @@ int ssl_pem_passwd_cb(char *buf, int size, int rwflag, void *user_data) {
 
 namespace {
 int servername_callback(SSL *ssl, int *al, void *arg) {
-  auto handler = static_cast<ClientHandler *>(SSL_get_app_data(ssl));
+  auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
+  auto handler = static_cast<ClientHandler *>(conn->data);
   auto worker = handler->get_worker();
   auto cert_tree = worker->get_cert_lookup_tree();
   if (cert_tree) {
@@ -190,7 +191,8 @@ constexpr char MEMCACHED_SESSION_CACHE_KEY_PREFIX[] =
 
 namespace {
 int tls_session_new_cb(SSL *ssl, SSL_SESSION *session) {
-  auto handler = static_cast<ClientHandler *>(SSL_get_app_data(ssl));
+  auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
+  auto handler = static_cast<ClientHandler *>(conn->data);
   auto worker = handler->get_worker();
   auto dispatcher = worker->get_session_cache_memcached_dispatcher();
 
@@ -236,10 +238,10 @@ int tls_session_new_cb(SSL *ssl, SSL_SESSION *session) {
 namespace {
 SSL_SESSION *tls_session_get_cb(SSL *ssl, unsigned char *id, int idlen,
                                 int *copy) {
-  auto handler = static_cast<ClientHandler *>(SSL_get_app_data(ssl));
+  auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
+  auto handler = static_cast<ClientHandler *>(conn->data);
   auto worker = handler->get_worker();
   auto dispatcher = worker->get_session_cache_memcached_dispatcher();
-  auto conn = handler->get_connection();
 
   if (conn->tls.cached_session) {
     if (LOG_ENABLED(INFO)) {
@@ -309,7 +311,8 @@ SSL_SESSION *tls_session_get_cb(SSL *ssl, unsigned char *id, int idlen,
 namespace {
 int ticket_key_cb(SSL *ssl, unsigned char *key_name, unsigned char *iv,
                   EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc) {
-  auto handler = static_cast<ClientHandler *>(SSL_get_app_data(ssl));
+  auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
+  auto handler = static_cast<ClientHandler *>(conn->data);
   auto worker = handler->get_worker();
   auto ticket_keys = worker->get_ticket_keys();
 
@@ -385,7 +388,6 @@ void info_callback(const SSL *ssl, int where, int ret) {
   if (where & SSL_CB_HANDSHAKE_START) {
     auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
     if (conn && conn->tls.initial_handshake_done) {
-      // We only set SSL_get_app_data for ClientHandler for now.
       auto handler = static_cast<ClientHandler *>(conn->data);
       if (LOG_ENABLED(INFO)) {
         CLOG(INFO, handler) << "TLS renegotiation started";
@@ -695,7 +697,6 @@ SSL_CTX *create_ssl_client_context() {
   return ssl_ctx;
 }
 
-namespace {
 SSL *create_ssl(SSL_CTX *ssl_ctx) {
   auto ssl = SSL_new(ssl_ctx);
   if (!ssl) {
@@ -706,23 +707,6 @@ SSL *create_ssl(SSL_CTX *ssl_ctx) {
 
   return ssl;
 }
-} // namespace
-
-SSL *create_server_ssl(SSL_CTX *ssl_ctx, Worker *worker) {
-  auto ssl = create_ssl(ssl_ctx);
-  if (!ssl) {
-    return nullptr;
-  }
-
-  // Disable TLS session ticket if we don't have working ticket keys.
-  if (worker && !worker->get_ticket_keys()) {
-    SSL_set_options(ssl, SSL_OP_NO_TICKET);
-  }
-
-  return ssl;
-}
-
-SSL *create_client_ssl(SSL_CTX *ssl_ctx) { return create_ssl(ssl_ctx); }
 
 ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr,
                                  int addrlen) {
@@ -746,10 +730,15 @@ ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr,
   SSL *ssl = nullptr;
   auto ssl_ctx = worker->get_sv_ssl_ctx();
   if (ssl_ctx) {
-    ssl = create_server_ssl(ssl_ctx, worker);
+    ssl = create_ssl(ssl_ctx);
     if (!ssl) {
       return nullptr;
     }
+    // Disable TLS session ticket if we don't have working ticket
+    // keys.
+    if (!worker->get_ticket_keys()) {
+      SSL_set_options(ssl, SSL_OP_NO_TICKET);
+    }
   }
 
   return new ClientHandler(worker, fd, ssl, host, service);

src/shrpx_ssl.h

@@ -172,8 +172,7 @@ SSL_CTX *setup_client_ssl_context();
 // this function returns nullptr.
 CertLookupTree *create_cert_lookup_tree();
 
-SSL *create_server_ssl(SSL_CTX *ssl_ctx, Worker *worker);
-SSL *create_client_ssl(SSL_CTX *ssl_ctx);
+SSL *create_ssl(SSL_CTX *ssl_ctx);
 
 } // namespace ssl
 

src/util.cc

@@ -1158,7 +1158,7 @@ uint64_t get_uint64(const uint8_t *data) {
   n += static_cast<uint64_t>(data[1]) << 48;
   n += static_cast<uint64_t>(data[2]) << 40;
   n += static_cast<uint64_t>(data[3]) << 32;
-  n += data[4] << 24;
+  n += static_cast<uint64_t>(data[4]) << 24;
   n += data[5] << 16;
   n += data[6] << 8;
   n += data[7];

src/util_test.cc

@@ -394,12 +394,22 @@ void test_util_localtime_date(void) {
 }
 
 void test_util_get_uint64(void) {
-  auto v = std::array<unsigned char, 8>{
-      {0x01, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xab, 0xbc}};
+  {
+    auto v = std::array<unsigned char, 8>{
+        {0x01, 0x12, 0x34, 0x56, 0xff, 0x9a, 0xab, 0xbc}};
 
-  auto n = util::get_uint64(v.data());
+    auto n = util::get_uint64(v.data());
 
-  CU_ASSERT(0x01123456789aabbcULL == n);
+    CU_ASSERT(0x01123456ff9aabbcULL == n);
+  }
+  {
+    auto v = std::array<unsigned char, 8>{
+        {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
+
+    auto n = util::get_uint64(v.data());
+
+    CU_ASSERT(0xffffffffffffffffULL == n);
+  }
 }
 
 } // namespace shrpx