cache-digest: Truncate hash from MSB

clang + libc++ does not work, it still requires libc++_shared.so
runtime even if -lstdc++ is used, which supposed to link with static
version of libc++.

CMakeLists.txt

@@ -24,13 +24,13 @@
 
 cmake_minimum_required(VERSION 3.0)
 # XXX using 1.8.90 instead of 1.9.0-DEV
-project(nghttp2 VERSION 1.11.1)
+project(nghttp2 VERSION 1.13.90)
 
 # See versioning rule:
 #  http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
-set(LT_CURRENT  22)
+set(LT_CURRENT  23)
 set(LT_REVISION 0)
-set(LT_AGE      8)
+set(LT_AGE      9)
 
 set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 include(Version)

README.rst

@@ -176,6 +176,23 @@ To compile the source code, gcc >= 4.8.3 or clang >= 3.4 is required.
    applications were not built, then using ``--enable-app`` may find
    that cause, such as the missing dependency.
 
+Notes for building on Windows (MSVC)
+------------------------------------
+
+The easiest way to build native Windows nghttp2 dll is use `cmake
+<https://cmake.org/>`_.  The free version of `Visual C++ Build Tools
+<http://landinghub.visualstudio.com/visual-cpp-build-tools>`_ works
+fine.
+
+1. Install cmake for windows
+2. Open "Visual C++ ... Native Build Tool Command Prompt", and inside
+   nghttp2 directly, run ``cmake``.
+3. Then run ``cmake --build`` to build library.
+4. nghttp2.dll, nghttp2.lib, nghttp2.exp are placed under lib directory.
+
+Note that the above steps most likely produce nghttp2 library only.
+No bundled applications are compiled.
+
 Notes for building on Windows (Mingw/Cygwin)
 --------------------------------------------
 

android-config

@@ -39,9 +39,8 @@ PATH="$TOOLCHAIN"/bin:"$PATH"
     --without-libxml2 \
     --disable-python-bindings \
     --disable-examples \
-    --enable-werror \
-    CC="$TOOLCHAIN"/bin/clang \
-    CXX="$TOOLCHAIN"/bin/clang++ \
+    CC="$TOOLCHAIN"/bin/arm-linux-androideabi-gcc \
+    CXX="$TOOLCHAIN"/bin/arm-linux-androideabi-g++ \
     CPPFLAGS="-fPIE -I$PREFIX/include" \
     PKG_CONFIG_LIBDIR="$PREFIX/lib/pkgconfig" \
     LDFLAGS="-fPIE -pie -L$PREFIX/lib"

configure.ac

@@ -25,7 +25,7 @@ dnl Do not change user variables!
 dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html
 
 AC_PREREQ(2.61)
-AC_INIT([nghttp2], [1.11.1], [t-tujikawa@users.sourceforge.net])
+AC_INIT([nghttp2], [1.14.0-DEV], [t-tujikawa@users.sourceforge.net])
 AC_CONFIG_AUX_DIR([.])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS([config.h])
@@ -44,9 +44,9 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
 dnl See versioning rule:
 dnl  http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
-AC_SUBST(LT_CURRENT, 22)
+AC_SUBST(LT_CURRENT, 23)
 AC_SUBST(LT_REVISION, 0)
-AC_SUBST(LT_AGE, 8)
+AC_SUBST(LT_AGE, 9)
 
 major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"`
 minor=`echo $PACKAGE_VERSION |cut -d. -f2 | sed -e "s/[^0-9]//g"`

doc/Makefile.am

@@ -57,6 +57,7 @@ APIDOCS= \
 	nghttp2_option_new.rst \
 	nghttp2_option_set_builtin_recv_extension_type.rst \
 	nghttp2_option_set_max_reserved_remote_streams.rst \
+	nghttp2_option_set_max_send_header_block_length.rst \
 	nghttp2_option_set_no_auto_ping_ack.rst \
 	nghttp2_option_set_no_auto_window_update.rst \
 	nghttp2_option_set_no_http_messaging.rst \
@@ -127,6 +128,7 @@ APIDOCS= \
 	nghttp2_session_server_new.rst \
 	nghttp2_session_server_new2.rst \
 	nghttp2_session_server_new3.rst \
+	nghttp2_session_set_local_window_size.rst \
 	nghttp2_session_set_next_stream_id.rst \
 	nghttp2_session_set_stream_user_data.rst \
 	nghttp2_session_terminate_session.rst \

doc/_exts/sphinxcontrib/rubydomain.py

@@ -15,6 +15,7 @@ from docutils import nodes
 from docutils.parsers.rst import directives
 
 from sphinx import addnodes
+from sphinx import version_info
 from sphinx.roles import XRefRole
 from sphinx.locale import l_, _
 from sphinx.domains import Domain, ObjType, Index
@@ -231,8 +232,8 @@ class RubyObject(ObjectDescription):
 
         indextext = self.get_index_text(modname, name_cls)
         if indextext:
-            self.indexnode['entries'].append(('single', indextext,
-                                              fullname, fullname))
+            self.indexnode['entries'].append(
+                _make_index('single', indextext, fullname, fullname))
 
     def before_content(self):
         # needed for automatic qualification of members (reset in subclasses)
@@ -415,11 +416,19 @@ class RubyModule(Directive):
         # modindex currently
         if not noindex:
             indextext = _('%s (module)') % modname
-            inode = addnodes.index(entries=[('single', indextext,
-                                             'module-' + modname, modname)])
+            inode = addnodes.index(entries=[_make_index(
+                'single', indextext, 'module-' + modname, modname)])
             ret.append(inode)
         return ret
 
+def _make_index(entrytype, entryname, target, ignored, key=None):
+    # Sphinx 1.4 introduced backward incompatible changes, it now
+    # requires 5 tuples.  Last one is categorization key.  See
+    # http://www.sphinx-doc.org/en/stable/extdev/nodes.html#sphinx.addnodes.index
+    if version_info >= (1, 4, 0, '', 0):
+        return (entrytype, entryname, target, ignored, key)
+    else:
+        return (entrytype, entryname, target, ignored)
 
 class RubyCurrentModule(Directive):
     """

doc/bash_completion/nghttpx

@@ -8,7 +8,7 @@ _nghttpx()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--worker-read-rate --include --frontend-http2-dump-response-header --tls-ticket-key-file --verify-client-cacert --max-response-header-fields --backend-request-buffer --max-request-header-fields --backend-http2-connection-window-bits --conf --backend-http2-max-concurrent-streams --worker-write-burst --npn-list --fetch-ocsp-response-file --no-via --tls-session-cache-memcached-cert-file --no-http2-cipher-black-list --mruby-file --no-server-push --stream-read-timeout --tls-ticket-key-memcached --forwarded-for --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --rlimit-nofile --tls-ticket-key-memcached-cert-file --ocsp-update-interval --forwarded-by --tls-session-cache-memcached-private-key-file --error-page --backend-write-timeout --tls-dyn-rec-warmup-threshold --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --worker-read-burst --dh-param-file --accesslog-format --errorlog-syslog --request-header-field-buffer --errorlog-file --frontend-http2-max-concurrent-streams --frontend-write-timeout --tls-ticket-key-cipher --read-burst --backend --insecure --log-level --host-rewrite --tls-proto-list --tls-ticket-key-memcached-interval --frontend-http2-setting-timeout --worker-frontend-connections --syslog-facility --fastopen --no-location-rewrite --tls-session-cache-memcached --no-ocsp --backend-response-buffer --workers --add-forwarded --frontend-http2-window-bits --worker-write-rate --add-request-header --backend-http2-settings-timeout --subcert --no-kqueue --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --user --add-x-forwarded-for --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --backend-connections-per-host --backend-http2-window-bits --response-header-field-buffer --tls-ticket-key-memcached-address-family --padding --tls-session-cache-memcached-address-family --stream-write-timeout --cacert --tls-ticket-key-memcached-private-key-file --backend-address-family --version --add-response-header --backend-read-timeout --frontend --accesslog-file --http2-proxy --client-private-key-file --client-cert-file --accept-proxy-protocol --tls-dyn-rec-idle-timeout --verify-client --read-rate --backend-connections-per-frontend --strip-incoming-forwarded ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--worker-read-rate --include --frontend-http2-dump-response-header --tls-ticket-key-file --verify-client-cacert --max-response-header-fields --backend-request-buffer --max-request-header-fields --backend-http2-connection-window-bits --conf --backend-http2-max-concurrent-streams --worker-write-burst --npn-list --fetch-ocsp-response-file --no-via --tls-session-cache-memcached-cert-file --no-http2-cipher-black-list --mruby-file --no-server-push --stream-read-timeout --tls-ticket-key-memcached --forwarded-for --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --rlimit-nofile --tls-ticket-key-memcached-cert-file --ocsp-update-interval --forwarded-by --tls-session-cache-memcached-private-key-file --error-page --backend-write-timeout --tls-dyn-rec-warmup-threshold --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --worker-read-burst --dh-param-file --accesslog-format --errorlog-syslog --request-header-field-buffer --api-max-request-body --errorlog-file --frontend-http2-max-concurrent-streams --frontend-write-timeout --tls-ticket-key-cipher --read-burst --backend --insecure --backend-max-backoff --log-level --host-rewrite --tls-proto-list --tls-ticket-key-memcached-interval --frontend-http2-setting-timeout --worker-frontend-connections --syslog-facility --fastopen --no-location-rewrite --tls-session-cache-memcached --no-ocsp --backend-response-buffer --workers --add-forwarded --frontend-http2-window-bits --worker-write-rate --add-request-header --backend-http2-settings-timeout --subcert --no-kqueue --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --user --add-x-forwarded-for --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --backend-connections-per-host --backend-http2-window-bits --response-header-field-buffer --tls-ticket-key-memcached-address-family --padding --tls-session-cache-memcached-address-family --stream-write-timeout --cacert --tls-ticket-key-memcached-private-key-file --backend-address-family --version --add-response-header --backend-read-timeout --frontend --accesslog-file --http2-proxy --client-private-key-file --client-cert-file --accept-proxy-protocol --tls-dyn-rec-idle-timeout --verify-client --read-rate --backend-connections-per-frontend --strip-incoming-forwarded ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/h2load.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "H2LOAD" "1" "May 29, 2016" "1.11.1" "nghttp2"
+.TH "H2LOAD" "1" "Jul 21, 2016" "1.13.0" "nghttp2"
 .SH NAME
 h2load \- HTTP/2 benchmarking tool
 .
@@ -138,7 +138,9 @@ Default: \fBh2c\fP
 .TP
 .B \-d, \-\-data=<PATH>
 Post FILE to  server.  The request method  is changed to
-POST.
+POST.   For  http/1.1 connection,  if  \fI\%\-d\fP  is used,  the
+maximum number of in\-flight pipelined requests is set to
+1.
 .UNINDENT
 .INDENT 0.0
 .TP
@@ -149,7 +151,7 @@ representing the  number of  connections to be  made per
 rate period.   The maximum  number of connections  to be
 made  is  given  in  \fI\%\-c\fP   option.   This  rate  will  be
 distributed among  threads as  evenly as  possible.  For
-example,  with   \fB\-t2\fP  and   \fB\-r4\fP,  each  thread   gets  2
+example,  with   \fI\%\-t\fP2  and   \fI\%\-r\fP4,  each  thread   gets  2
 connections per period.  When the rate is 0, the program
 will run  as it  normally does, creating  connections at
 whatever variable rate it  wants.  The default value for
@@ -412,7 +414,7 @@ performance.  To set smaller flow control window, use \fI\%\-w\fP and
 window size described in HTTP/2 and SPDY protocol specification.
 .SH SEE ALSO
 .sp
-\fInghttp(1)\fP, \fInghttpd(1)\fP, \fInghttpx(1)\fP
+\fBnghttp(1)\fP, \fBnghttpd(1)\fP, \fBnghttpx(1)\fP
 .SH AUTHOR
 Tatsuhiro Tsujikawa
 .SH COPYRIGHT

doc/h2load.1.rst

@@ -108,7 +108,9 @@ OPTIONS
 .. option:: -d, --data=<PATH>
 
     Post FILE to  server.  The request method  is changed to
-    POST.
+    POST.   For  http/1.1 connection,  if  :option:`-d`  is used,  the
+    maximum number of in-flight pipelined requests is set to
+    1.
 
 .. option:: -r, --rate=<N>
 
@@ -118,7 +120,7 @@ OPTIONS
     rate period.   The maximum  number of connections  to be
     made  is  given  in  :option:`-c`   option.   This  rate  will  be
     distributed among  threads as  evenly as  possible.  For
-    example,  with   :option:`-t2`  and   :option:`\-r4`,  each  thread   gets  2
+    example,  with   :option:`-t`\2  and   :option:`-r`\4,  each  thread   gets  2
     connections per period.  When the rate is 0, the program
     will run  as it  normally does, creating  connections at
     whatever variable rate it  wants.  The default value for

doc/nghttp.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTP" "1" "May 29, 2016" "1.11.1" "nghttp2"
+.TH "NGHTTP" "1" "Jul 21, 2016" "1.13.0" "nghttp2"
 .SH NAME
 nghttp \- HTTP/2 client
 .
@@ -300,7 +300,7 @@ stream 11 with the weight 12.  The other resources (e.g., icon) depend
 on stream 11 with the weight 2.
 .SH SEE ALSO
 .sp
-\fInghttpd(1)\fP, \fInghttpx(1)\fP, \fIh2load(1)\fP
+\fBnghttpd(1)\fP, \fBnghttpx(1)\fP, \fBh2load(1)\fP
 .SH AUTHOR
 Tatsuhiro Tsujikawa
 .SH COPYRIGHT

doc/nghttp.1.rst

@@ -208,7 +208,9 @@ implementation.
 
 When connection is established, nghttp sends 5 PRIORITY frames to idle
 streams 3, 5, 7, 9 and 11 to create "anchor" nodes in dependency
-tree::
+tree:
+
+.. code-block:: text
 
                       +-----+
                       |id=0 |

doc/nghttp.h2r

@@ -12,7 +12,9 @@ implementation.
 
 When connection is established, nghttp sends 5 PRIORITY frames to idle
 streams 3, 5, 7, 9 and 11 to create "anchor" nodes in dependency
-tree::
+tree:
+
+.. code-block:: text
 
                       +-----+
                       |id=0 |

doc/nghttpd.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTPD" "1" "May 29, 2016" "1.11.1" "nghttp2"
+.TH "NGHTTPD" "1" "Jul 21, 2016" "1.13.0" "nghttp2"
 .SH NAME
 nghttpd \- HTTP/2 server
 .
@@ -209,7 +209,7 @@ The <SIZE> argument is an integer and an optional unit (e.g., 10K is
 10 * 1024).  Units are K, M and G (powers of 1024).
 .SH SEE ALSO
 .sp
-\fInghttp(1)\fP, \fInghttpx(1)\fP, \fIh2load(1)\fP
+\fBnghttp(1)\fP, \fBnghttpx(1)\fP, \fBh2load(1)\fP
 .SH AUTHOR
 Tatsuhiro Tsujikawa
 .SH COPYRIGHT

doc/nghttpx.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTPX" "1" "May 29, 2016" "1.11.1" "nghttp2"
+.TH "NGHTTPX" "1" "Jul 21, 2016" "1.13.0" "nghttp2"
 .SH NAME
 nghttpx \- HTTP/2 proxy
 .
@@ -120,12 +120,13 @@ together forming  load balancing  group.
 Several parameters <PARAM> are accepted after <PATTERN>.
 The  parameters are  delimited  by  ";".  The  available
 parameters       are:      "proto=<PROTO>",       "tls",
-"sni=<SNI_HOST>",   "fall=<N>",  and   "rise=<N>".   The
-parameter consists  of keyword, and  optionally followed
-by "=" and value.  For example, the parameter "proto=h2"
-consists  of the  keyword "proto"  and value  "h2".  The
-parameter "tls"  consists of  the keyword  "tls" without
-value.  Each parameter is described as follows.
+"sni=<SNI_HOST>",     "fall=<N>",    "rise=<N>",     and
+"affinity=<METHOD>".  The parameter consists of keyword,
+and optionally followed by  "=" and value.  For example,
+the parameter "proto=h2" consists of the keyword "proto"
+and  value "h2".   The parameter  "tls" consists  of the
+keyword   "tls"  without   value.   Each   parameter  is
+described as follows.
 .sp
 The backend application protocol  can be specified using
 optional  "proto"   parameter,  and   in  the   form  of
@@ -160,6 +161,20 @@ eligible  for load  balancing target.   If <N>  is 0,  a
 backend  is permanently  offline, once  it goes  in that
 state, and this is the default behaviour.
 .sp
+The     session     affinity    is     enabled     using
+"affinity=<METHOD>"  parameter.   If  "ip" is  given  in
+<METHOD>, client  IP based session affinity  is enabled.
+If  "none" is  given  in <METHOD>,  session affinity  is
+disabled, and this is the default.  The session affinity
+is enabled per  <PATTERN>.  If at least  one backend has
+"affinity" parameter,  and its  <METHOD> is  not "none",
+session  affinity is  enabled  for  all backend  servers
+sharing  the  same  <PATTERN>.   It is  advised  to  set
+"affinity"  parameter  to   all  backend  explicitly  if
+session affinity  is desired.  The session  affinity may
+break if one of the backend gets unreachable, or backend
+settings are reloaded or replaced by API.
+.sp
 Since ";" and ":" are  used as delimiter, <PATTERN> must
 not  contain these  characters.  Since  ";" has  special
 meaning in shell, the option value must be quoted.
@@ -168,7 +183,7 @@ Default: \fB127.0.0.1,80\fP
 .UNINDENT
 .INDENT 0.0
 .TP
-.B \-f, \-\-frontend=(<HOST>,<PORT>|unix:<PATH>)[;no\-tls]
+.B \-f, \-\-frontend=(<HOST>,<PORT>|unix:<PATH>)[[;PARAM]...]
 Set  frontend  host and  port.   If  <HOST> is  \(aq*\(aq,  it
 assumes  all addresses  including  both  IPv4 and  IPv6.
 UNIX domain  socket can  be specified by  prefixing path
@@ -176,9 +191,25 @@ name  with  "unix:" (e.g.,  unix:/var/run/nghttpx.sock).
 This  option can  be used  multiple times  to listen  to
 multiple addresses.
 .sp
+This option  can take  0 or  more parameters,  which are
+described  below.   Note   that  "api"  and  "healthmon"
+parameters are mutually exclusive.
+.sp
 Optionally, TLS  can be disabled by  specifying "no\-tls"
 parameter.  TLS is enabled by default.
 .sp
+To  make this  frontend as  API endpoint,  specify "api"
+parameter.   This   is  disabled  by  default.    It  is
+important  to  limit the  access  to  the API  frontend.
+Otherwise, someone  may change  the backend  server, and
+break your services,  or expose confidential information
+to the outside the world.
+.sp
+To  make  this  frontend  as  health  monitor  endpoint,
+specify  "healthmon"  parameter.   This is  disabled  by
+default.  Any  requests which come through  this address
+are replied with 200 HTTP status, without no body.
+.sp
 Default: \fB*,3000\fP
 .UNINDENT
 .INDENT 0.0
@@ -455,6 +486,20 @@ backend server.
 .sp
 Default: \fB10s\fP
 .UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-backend\-max\-backoff=<DURATION>
+Specify  maximum backoff  interval.  This  is used  when
+doing health  check against offline backend  (see "fail"
+parameter  in \fI\%\-\-backend\fP  option).   It is  also used  to
+limit  the  maximum   interval  to  temporarily  disable
+backend  when nghttpx  failed to  connect to  it.  These
+intervals are calculated  using exponential backoff, and
+consecutive failed attempts increase the interval.  This
+option caps its maximum value.
+.sp
+Default: \fB2m\fP
+.UNINDENT
 .SS SSL/TLS
 .INDENT 0.0
 .TP
@@ -1078,6 +1123,14 @@ originally  generates  HTTP  error status  code  <CODE>.
 HTTP  status  code.  If  error  status  code comes  from
 backend server, the custom error pages are not used.
 .UNINDENT
+.SS API
+.INDENT 0.0
+.TP
+.B \-\-api\-max\-request\-body=<SIZE>
+Set the maximum size of request body for API request.
+.sp
+Default: \fB16K\fP
+.UNINDENT
 .SS Debug
 .INDENT 0.0
 .TP
@@ -1682,9 +1735,64 @@ App.new
 .fi
 .UNINDENT
 .UNINDENT
+.SH API ENDPOINTS
+.sp
+nghttpx exposes API endpoints to manipulate it via HTTP based API.  By
+default, API endpoint is disabled.  To enable it, add a dedicated
+frontend for API using \fI\%\-\-frontend\fP option with "api"
+parameter.  All requests which come from this frontend address, will
+be treated as API request.
+.sp
+The response is normally JSON dictionary, and at least includes the
+following keys:
+.INDENT 0.0
+.TP
+.B status
+The status of the request processing.  The following values are
+defined:
+.INDENT 7.0
+.TP
+.B Success
+The request was successful.
+.TP
+.B Failure
+The request was failed.  No change has been made.
+.UNINDENT
+.TP
+.B code
+HTTP status code
+.UNINDENT
+.sp
+We wrote "normally", since nghttpx may return ordinal HTML response in
+some cases where the error has occurred before reaching API endpoint
+(e.g., header field is too large).
+.sp
+The following section describes available API endpoints.
+.SS PUT /api/v1beta1/backendconfig
+.sp
+This API replaces the current backend server settings with the
+requested ones.  The request method should be PUT, but POST is also
+acceptable.  The request body must be nghttpx configuration file
+format.  For configuration file format, see \fI\%FILES\fP section.  The
+line separator inside the request body must be single LF (0x0A).
+Currently, only \fI\%backend\fP option is parsed, the
+others are simply ignored.  The semantics of this API is replace the
+current backend with the backend options in request body.  Describe
+the desired set of backend severs, and nghttpx makes it happen.  If
+there is no \fI\%backend\fP option is found in request
+body, the current set of backend is replaced with the \fI\%backend\fP option\(aqs default value, which is \fB127.0.0.1,80\fP\&.
+.sp
+The replacement is done instantly without breaking existing
+connections or requests.  It also avoids any process creation as is
+the case with hot swapping with signals.
+.sp
+The one limitation is that only numeric IP address is allowd in
+\fI\%backend\fP in request body while non numeric
+hostname is allowed in command\-line or configuration file is read
+using \fI\%\-\-conf\fP\&.
 .SH SEE ALSO
 .sp
-\fInghttp(1)\fP, \fInghttpd(1)\fP, \fIh2load(1)\fP
+\fBnghttp(1)\fP, \fBnghttpd(1)\fP, \fBh2load(1)\fP
 .SH AUTHOR
 Tatsuhiro Tsujikawa
 .SH COPYRIGHT

doc/nghttpx.1.rst

@@ -104,12 +104,13 @@ Connections
     Several parameters <PARAM> are accepted after <PATTERN>.
     The  parameters are  delimited  by  ";".  The  available
     parameters       are:      "proto=<PROTO>",       "tls",
-    "sni=<SNI_HOST>",   "fall=<N>",  and   "rise=<N>".   The
-    parameter consists  of keyword, and  optionally followed
-    by "=" and value.  For example, the parameter "proto=h2"
-    consists  of the  keyword "proto"  and value  "h2".  The
-    parameter "tls"  consists of  the keyword  "tls" without
-    value.  Each parameter is described as follows.
+    "sni=<SNI_HOST>",     "fall=<N>",    "rise=<N>",     and
+    "affinity=<METHOD>".  The parameter consists of keyword,
+    and optionally followed by  "=" and value.  For example,
+    the parameter "proto=h2" consists of the keyword "proto"
+    and  value "h2".   The parameter  "tls" consists  of the
+    keyword   "tls"  without   value.   Each   parameter  is
+    described as follows.
 
     The backend application protocol  can be specified using
     optional  "proto"   parameter,  and   in  the   form  of
@@ -144,6 +145,20 @@ Connections
     backend  is permanently  offline, once  it goes  in that
     state, and this is the default behaviour.
 
+    The     session     affinity    is     enabled     using
+    "affinity=<METHOD>"  parameter.   If  "ip" is  given  in
+    <METHOD>, client  IP based session affinity  is enabled.
+    If  "none" is  given  in <METHOD>,  session affinity  is
+    disabled, and this is the default.  The session affinity
+    is enabled per  <PATTERN>.  If at least  one backend has
+    "affinity" parameter,  and its  <METHOD> is  not "none",
+    session  affinity is  enabled  for  all backend  servers
+    sharing  the  same  <PATTERN>.   It is  advised  to  set
+    "affinity"  parameter  to   all  backend  explicitly  if
+    session affinity  is desired.  The session  affinity may
+    break if one of the backend gets unreachable, or backend
+    settings are reloaded or replaced by API.
+
     Since ";" and ":" are  used as delimiter, <PATTERN> must
     not  contain these  characters.  Since  ";" has  special
     meaning in shell, the option value must be quoted.
@@ -151,7 +166,7 @@ Connections
 
     Default: ``127.0.0.1,80``
 
-.. option:: -f, --frontend=(<HOST>,<PORT>|unix:<PATH>)[;no-tls]
+.. option:: -f, --frontend=(<HOST>,<PORT>|unix:<PATH>)[[;PARAM]...]
 
     Set  frontend  host and  port.   If  <HOST> is  '\*',  it
     assumes  all addresses  including  both  IPv4 and  IPv6.
@@ -160,9 +175,25 @@ Connections
     This  option can  be used  multiple times  to listen  to
     multiple addresses.
 
+    This option  can take  0 or  more parameters,  which are
+    described  below.   Note   that  "api"  and  "healthmon"
+    parameters are mutually exclusive.
+
     Optionally, TLS  can be disabled by  specifying "no-tls"
     parameter.  TLS is enabled by default.
 
+    To  make this  frontend as  API endpoint,  specify "api"
+    parameter.   This   is  disabled  by  default.    It  is
+    important  to  limit the  access  to  the API  frontend.
+    Otherwise, someone  may change  the backend  server, and
+    break your services,  or expose confidential information
+    to the outside the world.
+
+    To  make  this  frontend  as  health  monitor  endpoint,
+    specify  "healthmon"  parameter.   This is  disabled  by
+    default.  Any  requests which come through  this address
+    are replied with 200 HTTP status, without no body.
+
 
     Default: ``*,3000``
 
@@ -414,6 +445,19 @@ Timeout
 
     Default: ``10s``
 
+.. option:: --backend-max-backoff=<DURATION>
+
+    Specify  maximum backoff  interval.  This  is used  when
+    doing health  check against offline backend  (see "fail"
+    parameter  in :option:`--backend`  option).   It is  also used  to
+    limit  the  maximum   interval  to  temporarily  disable
+    backend  when nghttpx  failed to  connect to  it.  These
+    intervals are calculated  using exponential backoff, and
+    consecutive failed attempts increase the interval.  This
+    option caps its maximum value.
+
+    Default: ``2m``
+
 
 SSL/TLS
 ~~~~~~~
@@ -972,6 +1016,16 @@ HTTP
     backend server, the custom error pages are not used.
 
 
+API
+~~~
+
+.. option:: --api-max-request-body=<SIZE>
+
+    Set the maximum size of request body for API request.
+
+    Default: ``16K``
+
+
 Debug
 ~~~~~
 
@@ -1145,7 +1199,7 @@ backend server and extracts URI-reference with parameter
 and pushes those URIs to the frontend client. Here is a sample Link
 header field to initiate server push:
 
-.. code-block:: http
+.. code-block:: text
 
   Link: </fonts/font.woff>; rel=preload
   Link: </css/theme.css>; rel=preload
@@ -1522,6 +1576,62 @@ addresses:
 
     App.new
 
+API ENDPOINTS
+-------------
+
+nghttpx exposes API endpoints to manipulate it via HTTP based API.  By
+default, API endpoint is disabled.  To enable it, add a dedicated
+frontend for API using :option:`--frontend` option with "api"
+parameter.  All requests which come from this frontend address, will
+be treated as API request.
+
+The response is normally JSON dictionary, and at least includes the
+following keys:
+
+status
+  The status of the request processing.  The following values are
+  defined:
+
+  Success
+    The request was successful.
+
+  Failure
+    The request was failed.  No change has been made.
+
+code
+  HTTP status code
+
+We wrote "normally", since nghttpx may return ordinal HTML response in
+some cases where the error has occurred before reaching API endpoint
+(e.g., header field is too large).
+
+The following section describes available API endpoints.
+
+PUT /api/v1beta1/backendconfig
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This API replaces the current backend server settings with the
+requested ones.  The request method should be PUT, but POST is also
+acceptable.  The request body must be nghttpx configuration file
+format.  For configuration file format, see `FILES`_ section.  The
+line separator inside the request body must be single LF (0x0A).
+Currently, only :option:`backend <--backend>` option is parsed, the
+others are simply ignored.  The semantics of this API is replace the
+current backend with the backend options in request body.  Describe
+the desired set of backend severs, and nghttpx makes it happen.  If
+there is no :option:`backend <--backend>` option is found in request
+body, the current set of backend is replaced with the :option:`backend
+<--backend>` option's default value, which is ``127.0.0.1,80``.
+
+The replacement is done instantly without breaking existing
+connections or requests.  It also avoids any process creation as is
+the case with hot swapping with signals.
+
+The one limitation is that only numeric IP address is allowd in
+:option:`backend <--backend>` in request body while non numeric
+hostname is allowed in command-line or configuration file is read
+using :option:`--conf`.
+
 SEE ALSO
 --------
 

doc/nghttpx.h2r

@@ -85,7 +85,7 @@ backend server and extracts URI-reference with parameter
 and pushes those URIs to the frontend client. Here is a sample Link
 header field to initiate server push:
 
-.. code-block:: http
+.. code-block:: text
 
   Link: </fonts/font.woff>; rel=preload
   Link: </css/theme.css>; rel=preload
@@ -462,6 +462,62 @@ addresses:
 
     App.new
 
+API ENDPOINTS
+-------------
+
+nghttpx exposes API endpoints to manipulate it via HTTP based API.  By
+default, API endpoint is disabled.  To enable it, add a dedicated
+frontend for API using :option:`--frontend` option with "api"
+parameter.  All requests which come from this frontend address, will
+be treated as API request.
+
+The response is normally JSON dictionary, and at least includes the
+following keys:
+
+status
+  The status of the request processing.  The following values are
+  defined:
+
+  Success
+    The request was successful.
+
+  Failure
+    The request was failed.  No change has been made.
+
+code
+  HTTP status code
+
+We wrote "normally", since nghttpx may return ordinal HTML response in
+some cases where the error has occurred before reaching API endpoint
+(e.g., header field is too large).
+
+The following section describes available API endpoints.
+
+PUT /api/v1beta1/backendconfig
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This API replaces the current backend server settings with the
+requested ones.  The request method should be PUT, but POST is also
+acceptable.  The request body must be nghttpx configuration file
+format.  For configuration file format, see `FILES`_ section.  The
+line separator inside the request body must be single LF (0x0A).
+Currently, only :option:`backend <--backend>` option is parsed, the
+others are simply ignored.  The semantics of this API is replace the
+current backend with the backend options in request body.  Describe
+the desired set of backend severs, and nghttpx makes it happen.  If
+there is no :option:`backend <--backend>` option is found in request
+body, the current set of backend is replaced with the :option:`backend
+<--backend>` option's default value, which is ``127.0.0.1,80``.
+
+The replacement is done instantly without breaking existing
+connections or requests.  It also avoids any process creation as is
+the case with hot swapping with signals.
+
+The one limitation is that only numeric IP address is allowd in
+:option:`backend <--backend>` in request body while non numeric
+hostname is allowed in command-line or configuration file is read
+using :option:`--conf`.
+
 SEE ALSO
 --------
 

doc/sources/building-android-binary.rst

@@ -17,19 +17,16 @@ installed in the following way.  First, let us introduce
 under ``$ANDROID_HOME/toolchain``.  An user can freely choose the path
 for ``ANDROID_HOME``.  For example, to install toolchain under
 ``$ANDROID_HOME/toolchain``, do this in the the directory where NDK is
-unpacked::
+unpacked:
 
-    $ build/tools/make-standalone-toolchain.sh \
-      --install-dir=$ANDROID_HOME/toolchain \
-      --toolchain=arm-linux-androideabi-4.9 \
-      --llvm-version=3.5 \
-      --platform=android-16
+.. code-block:: text
 
-The additional flag ``--system=linux-x86_64`` may be required if you
-are using x86_64 system.
+    $ build/tools/make_standalone_toolchain.py \
+      --arch arm --api 16 --stl gnustl
+      --install-dir $ANDROID_HOME/toolchain
 
-The platform level is not important here because we don't use Android
-specific C/C++ API.
+The API level (``--api``) is not important here because we don't use
+Android specific C/C++ API.
 
 The dependent libraries, such as OpenSSL and libev should be built
 with the toolchain and installed under ``$ANDROID_HOME/usr/local``.
@@ -45,7 +42,9 @@ spdylay as well.
 
 Before running ``android-config`` and ``android-make``,
 ``ANDROID_HOME`` environment variable must be set to point to the
-correct path.  Also add ``$ANDROID_HOME/toolchain/bin`` to ``PATH``::
+correct path.  Also add ``$ANDROID_HOME/toolchain/bin`` to ``PATH``:
+
+.. code-block:: text
 
     $ export PATH=$PATH:$ANDROID_HOME/toolchain/bin
 
@@ -159,6 +158,8 @@ then ``android-make`` to compile nghttp2 source files.
 
 If all went well, application binaries, such as nghttpx, are created
 under src directory.  Strip debugging information from the binary
-using the following command::
+using the following command:
+
+.. code-block:: text
 
     $ arm-linux-androideabi-strip src/nghttpx

doc/sources/nghttpx-howto.rst

@@ -48,12 +48,16 @@ explicitly.
 The backend is supposed to be Web server.  For example, to make
 nghttpx listen to encrypted HTTP/2 requests at port 8443, and a
 backend Web server is configured to listen to HTTP request at port
-8080 in the same host, run nghttpx command-line like this::
+8080 in the same host, run nghttpx command-line like this:
+
+.. code-block:: text
 
     $ nghttpx -f0.0.0.0,8443 -b127.0.0.1,8080 /path/to/server.key /path/to/server.crt
 
 Then HTTP/2 enabled client can access to the nghttpx in HTTP/2.  For
-example, you can send GET request to the server using nghttp::
+example, you can send GET request to the server using nghttp:
+
+.. code-block:: text
 
     $ nghttp -nv https://localhost:8443/
 
@@ -89,7 +93,9 @@ connection, use :option:`--backend` option, and specify ``h2`` in
 For example, to make nghttpx listen to encrypted HTTP/2 requests at
 port 8443, and a backend HTTP proxy server is configured to listen to
 HTTP/1 request at port 8080 in the same host, run nghttpx command-line
-like this::
+like this:
+
+.. code-block:: text
 
     $ nghttpx -s -f'*,8443' -b127.0.0.1,8080 /path/to/server.key /path/to/server.crt
 
@@ -118,13 +124,17 @@ to proxy.pac file, something like this:
 
     file:///path/to/proxy.pac
 
-For Chromium, use following command-line::
+For Chromium, use following command-line:
+
+.. code-block:: text
 
     $ google-chrome --proxy-pac-url=file:///path/to/proxy.pac --use-npn
 
 As HTTP/1 proxy server, Squid may work as out-of-box.  Traffic server
 requires to be configured as forward proxy.  Here is the minimum
-configuration items to edit::
+configuration items to edit:
+
+.. code-block:: text
 
     CONFIG proxy.config.reverse_proxy.enabled INT 0
     CONFIG proxy.config.url_remap.remap_required INT 0
@@ -152,9 +162,9 @@ Enable SSL/TLS on memcached connection
 --------------------------------------
 
 By default, memcached connection is not encrypted.  To enable
-encryption, use :option:`--tls-ticket-key-memcached-tls` for TLS
-ticket key, and use :option:`--tls-session-cache-memcached-tls` for
-TLS session cache.
+encryption, use ``tls`` keyword in
+:option:`--tls-ticket-key-memcached` for TLS ticket key, and
+:option:`--tls-session-cache-memcached` for TLS session cache.
 
 Specifying additional server certificates
 -----------------------------------------

doc/sources/python-apiref.rst

@@ -140,7 +140,9 @@ HTTP/2 servers
    Python 3.4 or later is required to use these objects.  To
    explicitly configure nghttp2 build to use Python 3.4, specify the
    ``PYTHON`` variable to the path to Python 3.4 executable when
-   invoking configure script like this::
+   invoking configure script like this:
+
+   .. code-block:: text
 
        $ ./configure PYTHON=/usr/bin/python3.4
 

doc/sources/tutorial-client.rst

@@ -7,7 +7,9 @@ the end of this page.  It also resides in the examples directory in
 the archive or repository.
 
 This simple client takes a single HTTPS URI and retrieves the resource
-at the URI. The synopsis is::
+at the URI. The synopsis is:
+
+.. code-block:: text
 
     $ libevent-client HTTPS_URI
 
@@ -31,6 +33,17 @@ protocol the library supports::
       return SSL_TLSEXT_ERR_OK;
     }
 
+If you are following TLS related RFC, you know that NPN is not the
+standardized way to negotiate HTTP/2.  NPN itself is not event
+published as RFC.  The standard way to negotiate HTTP/2 is ALPN,
+Application-Layer Protocol Negotiation Extension, defined in `RFC 7301
+<https://tools.ietf.org/html/rfc7301>`_.  The one caveat of ALPN is
+that OpenSSL >= 1.0.2 is required.  We use macro to enable/disable
+ALPN support depending on OpenSSL version.  OpenSSL's ALPN
+implementation does not require callback function like the above.  But
+we have to instruct OpenSSL SSL_CTX to use ALPN, which we'll talk
+about soon.
+
 The callback is added to the SSL_CTX object using
 ``SSL_CTX_set_next_proto_select_cb()``::
 
@@ -46,9 +59,18 @@ The callback is added to the SSL_CTX object using
                               SSL_OP_NO_COMPRESSION |
                               SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION);
       SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_next_proto_cb, NULL);
+
+    #if OPENSSL_VERSION_NUMBER >= 0x10002000L
+      SSL_CTX_set_alpn_protos(ssl_ctx, (const unsigned char *)"\x02h2", 3);
+    #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+
       return ssl_ctx;
     }
 
+Here we see ``SSL_CTX_get_alpn_protos()`` function call.  We instructs
+OpenSSL to notify the server that we support h2, ALPN identifier for
+HTTP/2.
+
 The example client defines a couple of structs:
 
 We define and use a ``http2_session_data`` structure to store data
@@ -124,7 +146,27 @@ underlying network socket::
       if (events & BEV_EVENT_CONNECTED) {
         int fd = bufferevent_getfd(bev);
         int val = 1;
+        const unsigned char *alpn = NULL;
+        unsigned int alpnlen = 0;
+        SSL *ssl;
+
         fprintf(stderr, "Connected\n");
+
+        ssl = bufferevent_openssl_get_ssl(session_data->bev);
+
+        SSL_get0_next_proto_negotiated(ssl, &alpn, &alpnlen);
+    #if OPENSSL_VERSION_NUMBER >= 0x10002000L
+        if (alpn == NULL) {
+          SSL_get0_alpn_selected(ssl, &alpn, &alpnlen);
+        }
+    #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+
+        if (alpn == NULL || alpnlen != 2 || memcmp("h2", alpn, 2) != 0) {
+          fprintf(stderr, "h2 is not negotiated\n");
+          delete_http2_session_data(session_data);
+          return;
+        }
+
         setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *)&val, sizeof(val));
         initialize_nghttp2_session(session_data);
         send_client_connection_header(session_data);
@@ -144,6 +186,9 @@ underlying network socket::
       delete_http2_session_data(session_data);
     }
 
+Here we validate that HTTP/2 is negotiated, and if not, drop
+connection.
+
 For ``BEV_EVENT_EOF``, ``BEV_EVENT_ERROR``, and ``BEV_EVENT_TIMEOUT``
 events, we just simply tear down the connection.
 

doc/sources/tutorial-server.rst

@@ -10,7 +10,9 @@ archive or repository.
 
 This simple server takes 3 arguments: The port number to listen on,
 the path to your SSL/TLS private key file, and the path to your
-certificate file.  The synopsis is::
+certificate file.  The synopsis is:
+
+.. code-block:: text
 
     $ libevent-server PORT /path/to/server.key /path/to/server.crt
 
@@ -25,7 +27,17 @@ application protocols the server supports to a client.  In this
 example program, when creating the ``SSL_CTX`` object, we store the
 application protocol name in the wire format of NPN in a statically
 allocated buffer. This is safe because we only create one ``SSL_CTX``
-object in the program's entire lifetime::
+object in the program's entire lifetime.
+
+If you are following TLS related RFC, you know that NPN is not the
+standardized way to negotiate HTTP/2.  NPN itself is not even
+published as RFC.  The standard way to negotiate HTTP/2 is ALPN,
+Application-Layer Protocol Negotiation Extension, defined in `RFC 7301
+<https://tools.ietf.org/html/rfc7301>`_.  The one caveat of ALPN is
+that OpenSSL >= 1.0.2 is required.  We use macro to enable/disable
+ALPN support depending on OpenSSL version.  In ALPN, client sends the
+list of supported application protocols, and server selects one of
+them.  We provide the callback for it::
 
     static unsigned char next_proto_list[256];
     static size_t next_proto_list_len;
@@ -37,6 +49,22 @@ object in the program's entire lifetime::
       return SSL_TLSEXT_ERR_OK;
     }
 
+    #if OPENSSL_VERSION_NUMBER >= 0x10002000L
+    static int alpn_select_proto_cb(SSL *ssl _U_, const unsigned char **out,
+                                    unsigned char *outlen, const unsigned char *in,
+                                    unsigned int inlen, void *arg _U_) {
+      int rv;
+
+      rv = nghttp2_select_next_protocol((unsigned char **)out, outlen, in, inlen);
+
+      if (rv != 1) {
+        return SSL_TLSEXT_ERR_NOACK;
+      }
+
+      return SSL_TLSEXT_ERR_OK;
+    }
+    #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+
     static SSL_CTX *create_ssl_ctx(const char *key_file, const char *cert_file) {
       SSL_CTX *ssl_ctx;
       EC_KEY *ecdh;
@@ -51,6 +79,11 @@ object in the program's entire lifetime::
       next_proto_list_len = 1 + NGHTTP2_PROTO_VERSION_ID_LEN;
 
       SSL_CTX_set_next_protos_advertised_cb(ssl_ctx, next_proto_cb, NULL);
+
+    #if OPENSSL_VERSION_NUMBER >= 0x10002000L
+      SSL_CTX_set_alpn_select_cb(ssl_ctx, alpn_select_proto_cb, NULL);
+    #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+
       return ssl_ctx;
     }
 
@@ -64,6 +97,11 @@ OpenSSL implementation, we just assign the pointer to the NPN buffers
 we filled in earlier. The NPN callback function is set to the
 ``SSL_CTX`` object using ``SSL_CTX_set_next_protos_advertised_cb()``.
 
+In ``alpn_select_proto_cb()``, we use `nghttp2_select_next_protocol()`
+to select application protocol.  The `nghttp2_select_next_protocol()`
+returns 1 only if it selected h2 (ALPN identifier for HTTP/2), and out
+parameters were assigned accordingly.
+
 Next, let's take a look at the main structures used by the example
 application:
 
@@ -167,11 +205,31 @@ underlying network socket::
     static void eventcb(struct bufferevent *bev _U_, short events, void *ptr) {
       http2_session_data *session_data = (http2_session_data *)ptr;
       if (events & BEV_EVENT_CONNECTED) {
+        const unsigned char *alpn = NULL;
+        unsigned int alpnlen = 0;
+        SSL *ssl;
+
         fprintf(stderr, "%s connected\n", session_data->client_addr);
 
+        ssl = bufferevent_openssl_get_ssl(session_data->bev);
+
+        SSL_get0_next_proto_negotiated(ssl, &alpn, &alpnlen);
+    #if OPENSSL_VERSION_NUMBER >= 0x10002000L
+        if (alpn == NULL) {
+          SSL_get0_alpn_selected(ssl, &alpn, &alpnlen);
+        }
+    #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+
+        if (alpn == NULL || alpnlen != 2 || memcmp("h2", alpn, 2) != 0) {
+          fprintf(stderr, "%s h2 is not negotiated\n", session_data->client_addr);
+          delete_http2_session_data(session_data);
+          return;
+        }
+
         initialize_nghttp2_session(session_data);
 
-        if (send_server_connection_header(session_data) != 0) {
+        if (send_server_connection_header(session_data) != 0 ||
+            session_send(session_data) != 0) {
           delete_http2_session_data(session_data);
           return;
         }
@@ -188,6 +246,9 @@ underlying network socket::
       delete_http2_session_data(session_data);
     }
 
+Here we validate that HTTP/2 is negotiated, and if not, drop
+connection.
+
 For the ``BEV_EVENT_EOF``, ``BEV_EVENT_ERROR``, and
 ``BEV_EVENT_TIMEOUT`` events, we just simply tear down the connection.
 The ``delete_http2_session_data()`` function destroys the

examples/client.c

@@ -219,9 +219,9 @@ static int on_frame_send_callback(nghttp2_session *session,
       const nghttp2_nv *nva = frame->headers.nva;
       printf("[INFO] C ----------------------------> S (HEADERS)\n");
       for (i = 0; i < frame->headers.nvlen; ++i) {
-        fwrite(nva[i].name, nva[i].namelen, 1, stdout);
+        fwrite(nva[i].name, 1, nva[i].namelen, stdout);
         printf(": ");
-        fwrite(nva[i].value, nva[i].valuelen, 1, stdout);
+        fwrite(nva[i].value, 1, nva[i].valuelen, stdout);
         printf("\n");
       }
     }
@@ -249,9 +249,9 @@ static int on_frame_recv_callback(nghttp2_session *session,
       if (req) {
         printf("[INFO] C <---------------------------- S (HEADERS)\n");
         for (i = 0; i < frame->headers.nvlen; ++i) {
-          fwrite(nva[i].name, nva[i].namelen, 1, stdout);
+          fwrite(nva[i].name, 1, nva[i].namelen, stdout);
           printf(": ");
-          fwrite(nva[i].value, nva[i].valuelen, 1, stdout);
+          fwrite(nva[i].value, 1, nva[i].valuelen, stdout);
           printf("\n");
         }
       }
@@ -562,7 +562,11 @@ static void fetch_uri(const struct URI *uri) {
     diec("nghttp2_session_client_new", rv);
   }
 
-  nghttp2_submit_settings(connection.session, NGHTTP2_FLAG_NONE, NULL, 0);
+  rv = nghttp2_submit_settings(connection.session, NGHTTP2_FLAG_NONE, NULL, 0);
+
+  if (rv != 0) {
+    diec("nghttp2_submit_settings", rv);
+  }
 
   /* Submit the HTTP request to the outbound queue. */
   submit_request(&connection, &req);
@@ -691,9 +695,6 @@ int main(int argc, char **argv) {
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, 0);
 
-#ifndef OPENSSL_IS_BORINGSSL
-  OPENSSL_config(NULL);
-#endif /* OPENSSL_IS_BORINGSSL */
   SSL_load_error_strings();
   SSL_library_init();
 

examples/deflate.c

@@ -109,9 +109,9 @@ static void deflate(nghttp2_hd_deflater *deflater,
   printf("Input (%zu byte(s)):\n\n", sum);
 
   for (i = 0; i < nvlen; ++i) {
-    fwrite(nva[i].name, nva[i].namelen, 1, stdout);
+    fwrite(nva[i].name, 1, nva[i].namelen, stdout);
     printf(": ");
-    fwrite(nva[i].value, nva[i].valuelen, 1, stdout);
+    fwrite(nva[i].value, 1, nva[i].valuelen, stdout);
     printf("\n");
   }
 
@@ -186,9 +186,9 @@ int inflate_header_block(nghttp2_hd_inflater *inflater, uint8_t *in,
     inlen -= proclen;
 
     if (inflate_flags & NGHTTP2_HD_INFLATE_EMIT) {
-      fwrite(nv.name, nv.namelen, 1, stderr);
+      fwrite(nv.name, 1, nv.namelen, stderr);
       fprintf(stderr, ": ");
-      fwrite(nv.value, nv.valuelen, 1, stderr);
+      fwrite(nv.value, 1, nv.valuelen, stderr);
       fprintf(stderr, "\n");
     }
 

examples/libevent-client.c

@@ -179,9 +179,9 @@ static void delete_http2_session_data(http2_session_data *session_data) {
 
 static void print_header(FILE *f, const uint8_t *name, size_t namelen,
                          const uint8_t *value, size_t valuelen) {
-  fwrite(name, namelen, 1, f);
+  fwrite(name, 1, namelen, f);
   fprintf(f, ": ");
-  fwrite(value, valuelen, 1, f);
+  fwrite(value, 1, valuelen, f);
   fprintf(f, "\n");
 }
 
@@ -272,7 +272,7 @@ static int on_data_chunk_recv_callback(nghttp2_session *session _U_,
                                        void *user_data) {
   http2_session_data *session_data = (http2_session_data *)user_data;
   if (session_data->stream_data->stream_id == stream_id) {
-    fwrite(data, len, 1, stdout);
+    fwrite(data, 1, len, stdout);
   }
   return 0;
 }
@@ -322,6 +322,11 @@ static SSL_CTX *create_ssl_ctx(void) {
                           SSL_OP_NO_COMPRESSION |
                           SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION);
   SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_next_proto_cb, NULL);
+
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+  SSL_CTX_set_alpn_protos(ssl_ctx, (const unsigned char *)"\x02h2", 3);
+#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+
   return ssl_ctx;
 }
 
@@ -475,7 +480,27 @@ static void eventcb(struct bufferevent *bev, short events, void *ptr) {
   if (events & BEV_EVENT_CONNECTED) {
     int fd = bufferevent_getfd(bev);
     int val = 1;
+    const unsigned char *alpn = NULL;
+    unsigned int alpnlen = 0;
+    SSL *ssl;
+
     fprintf(stderr, "Connected\n");
+
+    ssl = bufferevent_openssl_get_ssl(session_data->bev);
+
+    SSL_get0_next_proto_negotiated(ssl, &alpn, &alpnlen);
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+    if (alpn == NULL) {
+      SSL_get0_alpn_selected(ssl, &alpn, &alpnlen);
+    }
+#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+
+    if (alpn == NULL || alpnlen != 2 || memcmp("h2", alpn, 2) != 0) {
+      fprintf(stderr, "h2 is not negotiated\n");
+      delete_http2_session_data(session_data);
+      return;
+    }
+
     setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *)&val, sizeof(val));
     initialize_nghttp2_session(session_data);
     send_client_connection_header(session_data);
@@ -569,9 +594,6 @@ int main(int argc, char **argv) {
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, NULL);
 
-#ifndef OPENSSL_IS_BORINGSSL
-  OPENSSL_config(NULL);
-#endif /* OPENSSL_IS_BORINGSSL */
   SSL_load_error_strings();
   SSL_library_init();
 

examples/libevent-server.c

@@ -116,6 +116,22 @@ static int next_proto_cb(SSL *s _U_, const unsigned char **data,
   return SSL_TLSEXT_ERR_OK;
 }
 
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+static int alpn_select_proto_cb(SSL *ssl _U_, const unsigned char **out,
+                                unsigned char *outlen, const unsigned char *in,
+                                unsigned int inlen, void *arg _U_) {
+  int rv;
+
+  rv = nghttp2_select_next_protocol((unsigned char **)out, outlen, in, inlen);
+
+  if (rv != 1) {
+    return SSL_TLSEXT_ERR_NOACK;
+  }
+
+  return SSL_TLSEXT_ERR_OK;
+}
+#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+
 /* Create SSL_CTX. */
 static SSL_CTX *create_ssl_ctx(const char *key_file, const char *cert_file) {
   SSL_CTX *ssl_ctx;
@@ -152,6 +168,11 @@ static SSL_CTX *create_ssl_ctx(const char *key_file, const char *cert_file) {
   next_proto_list_len = 1 + NGHTTP2_PROTO_VERSION_ID_LEN;
 
   SSL_CTX_set_next_protos_advertised_cb(ssl_ctx, next_proto_cb, NULL);
+
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+  SSL_CTX_set_alpn_select_cb(ssl_ctx, alpn_select_proto_cb, NULL);
+#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+
   return ssl_ctx;
 }
 
@@ -640,11 +661,31 @@ static void writecb(struct bufferevent *bev, void *ptr) {
 static void eventcb(struct bufferevent *bev _U_, short events, void *ptr) {
   http2_session_data *session_data = (http2_session_data *)ptr;
   if (events & BEV_EVENT_CONNECTED) {
+    const unsigned char *alpn = NULL;
+    unsigned int alpnlen = 0;
+    SSL *ssl;
+
     fprintf(stderr, "%s connected\n", session_data->client_addr);
 
+    ssl = bufferevent_openssl_get_ssl(session_data->bev);
+
+    SSL_get0_next_proto_negotiated(ssl, &alpn, &alpnlen);
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+    if (alpn == NULL) {
+      SSL_get0_alpn_selected(ssl, &alpn, &alpnlen);
+    }
+#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+
+    if (alpn == NULL || alpnlen != 2 || memcmp("h2", alpn, 2) != 0) {
+      fprintf(stderr, "%s h2 is not negotiated\n", session_data->client_addr);
+      delete_http2_session_data(session_data);
+      return;
+    }
+
     initialize_nghttp2_session(session_data);
 
-    if (send_server_connection_header(session_data) != 0) {
+    if (send_server_connection_header(session_data) != 0 ||
+        session_send(session_data) != 0) {
       delete_http2_session_data(session_data);
       return;
     }
@@ -740,9 +781,6 @@ int main(int argc, char **argv) {
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, NULL);
 
-#ifndef OPENSSL_IS_BORINGSSL
-  OPENSSL_config(NULL);
-#endif /* OPENSSL_IS_BORINGSSL */
   SSL_load_error_strings();
   SSL_library_init();
 

gennghttpxfun.py

@@ -132,6 +132,8 @@ OPTIONS = [
     "no-kqueue",
     "frontend-http2-settings-timeout",
     "backend-http2-settings-timeout",
+    "api-max-request-body",
+    "backend-max-backoff",
 ]
 
 LOGVARS = [

help2rst.py

@@ -166,7 +166,8 @@ def format_text(text):
     else:
         text = re.sub(r'\*', r'\*', text)
     # markup option reference
-    text = re.sub(r'(^|\s)(-[a-zA-Z0-9-]+)', r'\1:option:`\2`', text)
+    text = re.sub(r'(^|\s)(-[a-zA-Z0-9]|--[a-zA-Z0-9-]+)',
+                  r'\1:option:`\2`', text)
     # sphinx does not like markup like ':option:`-f`='.  We need
     # backslash between ` and =.
     text = re.sub(r'(:option:`.*?`)(\S)', r'\1\\\2', text)

integration-tests/nghttpx_http1_test.go

@@ -3,6 +3,7 @@ package nghttp2
 import (
 	"bufio"
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"golang.org/x/net/http2/hpack"
 	"golang.org/x/net/websocket"
@@ -793,3 +794,197 @@ func TestH1H2RespPhaseReturn(t *testing.T) {
 		t.Errorf("body = %v; want %v", got, want)
 	}
 }
+
+// TestH1APIBackendconfig exercise backendconfig API endpoint routine
+// for successful case.
+func TestH1APIBackendconfig(t *testing.T) {
+	st := newServerTesterConnectPort([]string{"-f127.0.0.1,3010;api;no-tls"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.http1(requestParam{
+		name:   "TestH1APIBackendconfig",
+		path:   "/api/v1beta1/backendconfig",
+		method: "PUT",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.http1() = %v", err)
+	}
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Success"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 200; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestH1APIBackendconfigQuery exercise backendconfig API endpoint
+// routine with query.
+func TestH1APIBackendconfigQuery(t *testing.T) {
+	st := newServerTesterConnectPort([]string{"-f127.0.0.1,3010;api;no-tls"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.http1(requestParam{
+		name:   "TestH1APIBackendconfigQuery",
+		path:   "/api/v1beta1/backendconfig?foo=bar",
+		method: "PUT",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.http1() = %v", err)
+	}
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Success"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 200; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestH1APIBackendconfigBadMethod exercise backendconfig API endpoint
+// routine with bad method.
+func TestH1APIBackendconfigBadMethod(t *testing.T) {
+	st := newServerTesterConnectPort([]string{"-f127.0.0.1,3010;api;no-tls"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.http1(requestParam{
+		name:   "TestH1APIBackendconfigBadMethod",
+		path:   "/api/v1beta1/backendconfig",
+		method: "GET",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.http1() = %v", err)
+	}
+	if got, want := res.status, 405; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Failure"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 405; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestH1APINotFound exercise backendconfig API endpoint routine when
+// API endpoint is not found.
+func TestH1APINotFound(t *testing.T) {
+	st := newServerTesterConnectPort([]string{"-f127.0.0.1,3010;api;no-tls"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.http1(requestParam{
+		name:   "TestH1APINotFound",
+		path:   "/api/notfound",
+		method: "GET",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.http1() = %v", err)
+	}
+	if got, want := res.status, 404; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Failure"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 404; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestH1Healthmon tests health monitor endpoint.
+func TestH1Healthmon(t *testing.T) {
+	st := newServerTesterConnectPort([]string{"-f127.0.0.1,3011;healthmon;no-tls"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3011)
+	defer st.Close()
+
+	res, err := st.http1(requestParam{
+		name: "TestH1Healthmon",
+		path: "/alpha/bravo",
+	})
+	if err != nil {
+		t.Fatalf("Error st.http1() = %v", err)
+	}
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+}
+
+// TestH1ResponseBeforeRequestEnd tests the situation where response
+// ends before request body finishes.
+func TestH1ResponseBeforeRequestEnd(t *testing.T) {
+	st := newServerTester([]string{"--mruby-file=" + testDir + "/req-return.rb"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatal("request should not be forwarded")
+	})
+	defer st.Close()
+
+	if _, err := io.WriteString(st.conn, fmt.Sprintf(`POST / HTTP/1.1
+Host: %v
+Test-Case: TestH1ResponseBeforeRequestEnd
+Content-Length: 1000000
+
+`, st.authority)); err != nil {
+		t.Fatalf("Error io.WriteString() = %v", err)
+	}
+
+	resp, err := http.ReadResponse(bufio.NewReader(st.conn), nil)
+	if err != nil {
+		t.Fatalf("Error http.ReadResponse() = %v", err)
+	}
+
+	if got, want := resp.StatusCode, 404; got != want {
+		t.Errorf("status: %v; want %v", got, want)
+	}
+}

integration-tests/nghttpx_http2_test.go

@@ -2,6 +2,7 @@ package nghttp2
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
@@ -1843,3 +1844,190 @@ func TestH2H2RespPhaseReturn(t *testing.T) {
 		t.Errorf("body = %v; want %v", got, want)
 	}
 }
+
+// TestH2APIBackendconfig exercise backendconfig API endpoint routine
+// for successful case.
+func TestH2APIBackendconfig(t *testing.T) {
+	st := newServerTesterConnectPort([]string{"-f127.0.0.1,3010;api;no-tls"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.http2(requestParam{
+		name:   "TestH2APIBackendconfig",
+		path:   "/api/v1beta1/backendconfig",
+		method: "PUT",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Success"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 200; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestH2APIBackendconfigQuery exercise backendconfig API endpoint
+// routine with query.
+func TestH2APIBackendconfigQuery(t *testing.T) {
+	st := newServerTesterConnectPort([]string{"-f127.0.0.1,3010;api;no-tls"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.http2(requestParam{
+		name:   "TestH2APIBackendconfigQuery",
+		path:   "/api/v1beta1/backendconfig?foo=bar",
+		method: "PUT",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Success"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 200; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestH2APIBackendconfigBadMethod exercise backendconfig API endpoint
+// routine with bad method.
+func TestH2APIBackendconfigBadMethod(t *testing.T) {
+	st := newServerTesterConnectPort([]string{"-f127.0.0.1,3010;api;no-tls"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.http2(requestParam{
+		name:   "TestH2APIBackendconfigBadMethod",
+		path:   "/api/v1beta1/backendconfig",
+		method: "GET",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+	if got, want := res.status, 405; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Failure"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 405; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestH2APINotFound exercise backendconfig API endpoint routine when
+// API endpoint is not found.
+func TestH2APINotFound(t *testing.T) {
+	st := newServerTesterConnectPort([]string{"-f127.0.0.1,3010;api;no-tls"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.http2(requestParam{
+		name:   "TestH2APINotFound",
+		path:   "/api/notfound",
+		method: "GET",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+	if got, want := res.status, 404; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Failure"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 404; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestH2Healthmon tests health monitor endpoint.
+func TestH2Healthmon(t *testing.T) {
+	st := newServerTesterConnectPort([]string{"-f127.0.0.1,3011;healthmon;no-tls"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3011)
+	defer st.Close()
+
+	res, err := st.http2(requestParam{
+		name: "TestH2Healthmon",
+		path: "/alpha/bravo",
+	})
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+}
+
+// TestH2ResponseBeforeRequestEnd tests the situation where response
+// ends before request body finishes.
+func TestH2ResponseBeforeRequestEnd(t *testing.T) {
+	st := newServerTester([]string{"--mruby-file=" + testDir + "/req-return.rb"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatal("request should not be forwarded")
+	})
+	defer st.Close()
+
+	res, err := st.http2(requestParam{
+		name:        "TestH2ResponseBeforeRequestEnd",
+		noEndStream: true,
+	})
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+	if got, want := res.status, 404; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+}

integration-tests/nghttpx_spdy_test.go

@@ -1,6 +1,7 @@
 package nghttp2
 
 import (
+	"encoding/json"
 	"github.com/tatsuhiro-t/spdy"
 	"golang.org/x/net/http2/hpack"
 	"net/http"
@@ -474,3 +475,190 @@ func TestS3H2RespPhaseReturn(t *testing.T) {
 		t.Errorf("body = %v; want %v", got, want)
 	}
 }
+
+// TestS3APIBackendconfig exercise backendconfig API endpoint routine
+// for successful case.
+func TestS3APIBackendconfig(t *testing.T) {
+	st := newServerTesterTLSConnectPort([]string{"--npn-list=spdy/3.1", "-f127.0.0.1,3010;api"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.spdy(requestParam{
+		name:   "TestS3APIBackendconfig",
+		path:   "/api/v1beta1/backendconfig",
+		method: "PUT",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.spdy() = %v", err)
+	}
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Success"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 200; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestS3APIBackendconfigQuery exercise backendconfig API endpoint
+// routine with query.
+func TestS3APIBackendconfigQuery(t *testing.T) {
+	st := newServerTesterTLSConnectPort([]string{"--npn-list=spdy/3.1", "-f127.0.0.1,3010;api"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.spdy(requestParam{
+		name:   "TestS3APIBackendconfigQuery",
+		path:   "/api/v1beta1/backendconfig?foo=bar",
+		method: "PUT",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.spdy() = %v", err)
+	}
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Success"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 200; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestS3APIBackendconfigBadMethod exercise backendconfig API endpoint
+// routine with bad method.
+func TestS3APIBackendconfigBadMethod(t *testing.T) {
+	st := newServerTesterTLSConnectPort([]string{"--npn-list=spdy/3.1", "-f127.0.0.1,3010;api"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.spdy(requestParam{
+		name:   "TestS3APIBackendconfigBadMethod",
+		path:   "/api/v1beta1/backendconfig",
+		method: "GET",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.spdy() = %v", err)
+	}
+	if got, want := res.status, 405; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Failure"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 405; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestS3APINotFound exercise backendconfig API endpoint routine when
+// API endpoint is not found.
+func TestS3APINotFound(t *testing.T) {
+	st := newServerTesterTLSConnectPort([]string{"--npn-list=spdy/3.1", "-f127.0.0.1,3010;api"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3010)
+	defer st.Close()
+
+	res, err := st.spdy(requestParam{
+		name:   "TestS3APINotFound",
+		path:   "/api/notfound",
+		method: "GET",
+		body: []byte(`# comment
+backend=127.0.0.1,3011
+
+`),
+	})
+	if err != nil {
+		t.Fatalf("Error st.spdy() = %v", err)
+	}
+	if got, want := res.status, 404; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+
+	var apiResp APIResponse
+	err = json.Unmarshal(res.body, &apiResp)
+	if err != nil {
+		t.Fatalf("Error unmarshaling API response: %v", err)
+	}
+	if got, want := apiResp.Status, "Failure"; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+	if got, want := apiResp.Code, 404; got != want {
+		t.Errorf("apiResp.Status: %v; want %v", got, want)
+	}
+}
+
+// TestS3Healthmon tests health monitor endpoint.
+func TestS3Healthmon(t *testing.T) {
+	st := newServerTesterTLSConnectPort([]string{"--npn-list=spdy/3.1", "-f127.0.0.1,3011;healthmon"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatalf("request should not be forwarded")
+	}, 3011)
+	defer st.Close()
+
+	res, err := st.spdy(requestParam{
+		name: "TestS3Healthmon",
+		path: "/alpha/bravo",
+	})
+	if err != nil {
+		t.Fatalf("Error st.spdy() = %v", err)
+	}
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+}
+
+// TestS3ResponseBeforeRequestEnd tests the situation where response
+// ends before request body finishes.
+func TestS3ResponseBeforeRequestEnd(t *testing.T) {
+	st := newServerTesterTLS([]string{"--npn-list=spdy/3.1", "--mruby-file=" + testDir + "/req-return.rb"}, t, func(w http.ResponseWriter, r *http.Request) {
+		t.Fatal("request should not be forwarded")
+	})
+	defer st.Close()
+
+	res, err := st.spdy(requestParam{
+		name:        "TestS3ResponseBeforeRequestEnd",
+		noEndStream: true,
+	})
+	if err != nil {
+		t.Fatalf("Error st.spdy() = %v", err)
+	}
+	if got, want := res.status, 404; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+}

integration-tests/server_tester.go

@@ -66,27 +66,36 @@ type serverTester struct {
 // newServerTester creates test context for plain TCP frontend
 // connection.
 func newServerTester(args []string, t *testing.T, handler http.HandlerFunc) *serverTester {
-	return newServerTesterInternal(args, t, handler, false, nil)
+	return newServerTesterInternal(args, t, handler, false, serverPort, nil)
+}
+
+func newServerTesterConnectPort(args []string, t *testing.T, handler http.HandlerFunc, port int) *serverTester {
+	return newServerTesterInternal(args, t, handler, false, port, nil)
 }
 
 func newServerTesterHandler(args []string, t *testing.T, handler http.Handler) *serverTester {
-	return newServerTesterInternal(args, t, handler, false, nil)
+	return newServerTesterInternal(args, t, handler, false, serverPort, nil)
 }
 
 // newServerTester creates test context for TLS frontend connection.
 func newServerTesterTLS(args []string, t *testing.T, handler http.HandlerFunc) *serverTester {
-	return newServerTesterInternal(args, t, handler, true, nil)
+	return newServerTesterInternal(args, t, handler, true, serverPort, nil)
+}
+
+func newServerTesterTLSConnectPort(args []string, t *testing.T, handler http.HandlerFunc, port int) *serverTester {
+	return newServerTesterInternal(args, t, handler, true, port, nil)
 }
 
 // newServerTester creates test context for TLS frontend connection
 // with given clientConfig
 func newServerTesterTLSConfig(args []string, t *testing.T, handler http.HandlerFunc, clientConfig *tls.Config) *serverTester {
-	return newServerTesterInternal(args, t, handler, true, clientConfig)
+	return newServerTesterInternal(args, t, handler, true, serverPort, clientConfig)
 }
 
 // newServerTesterInternal creates test context.  If frontendTLS is
-// true, set up TLS frontend connection.
-func newServerTesterInternal(src_args []string, t *testing.T, handler http.Handler, frontendTLS bool, clientConfig *tls.Config) *serverTester {
+// true, set up TLS frontend connection.  connectPort is the server
+// side port where client connection is made.
+func newServerTesterInternal(src_args []string, t *testing.T, handler http.Handler, frontendTLS bool, connectPort int, clientConfig *tls.Config) *serverTester {
 	ts := httptest.NewUnstartedServer(handler)
 
 	args := []string{}
@@ -138,7 +147,7 @@ func newServerTesterInternal(src_args []string, t *testing.T, handler http.Handl
 	args = append(args, fmt.Sprintf("-f127.0.0.1,%v;%v", serverPort, noTLS), b,
 		"--errorlog-file="+logDir+"/log.txt", "-LINFO")
 
-	authority := fmt.Sprintf("127.0.0.1:%v", serverPort)
+	authority := fmt.Sprintf("127.0.0.1:%v", connectPort)
 
 	st := &serverTester{
 		cmd:          exec.Command(serverBin, args...),
@@ -160,6 +169,8 @@ func newServerTesterInternal(src_args []string, t *testing.T, handler http.Handl
 
 	retry := 0
 	for {
+		time.Sleep(50 * time.Millisecond)
+
 		var conn net.Conn
 		var err error
 		if frontendTLS {
@@ -170,7 +181,7 @@ func newServerTesterInternal(src_args []string, t *testing.T, handler http.Handl
 				tlsConfig = clientConfig
 			}
 			tlsConfig.InsecureSkipVerify = true
-			tlsConfig.NextProtos = []string{"h2-14", "spdy/3.1"}
+			tlsConfig.NextProtos = []string{"h2", "spdy/3.1"}
 			conn, err = tls.Dial("tcp", authority, tlsConfig)
 		} else {
 			conn, err = net.Dial("tcp", authority)
@@ -181,7 +192,6 @@ func newServerTesterInternal(src_args []string, t *testing.T, handler http.Handl
 				st.Close()
 				st.t.Fatalf("Error server is not responding too long; server command-line arguments may be invalid")
 			}
-			time.Sleep(150 * time.Millisecond)
 			continue
 		}
 		if frontendTLS {
@@ -287,6 +297,7 @@ type requestParam struct {
 	body        []byte              // request body
 	trailer     []hpack.HeaderField // trailer part
 	httpUpgrade bool                // true if upgraded to HTTP/2 through HTTP Upgrade
+	noEndStream bool                // true if END_STREAM should not be sent
 }
 
 // wrapper for request body to set trailer part
@@ -370,8 +381,9 @@ func (st *serverTester) http1(rp requestParam) (*serverResponse, error) {
 		if err != nil {
 			st.t.Fatalf("Error parsing URL from st.url %v: %v", st.url, err)
 		}
-		u.Path = rp.path
-		reqURL = u.String()
+		u.Path = ""
+		u.RawQuery = ""
+		reqURL = u.String() + rp.path
 	}
 
 	req, err := http.NewRequest(method, reqURL, body)
@@ -460,7 +472,7 @@ func (st *serverTester) spdy(rp requestParam) (*serverResponse, error) {
 	}
 
 	var synStreamFlags spdy.ControlFlags
-	if len(rp.body) == 0 {
+	if len(rp.body) == 0 && !rp.noEndStream {
 		synStreamFlags = spdy.ControlFlagFin
 	}
 	if err := st.spdyFr.WriteFrame(&spdy.SynStreamFrame{
@@ -474,9 +486,13 @@ func (st *serverTester) spdy(rp requestParam) (*serverResponse, error) {
 	}
 
 	if len(rp.body) != 0 {
+		var dataFlags spdy.DataFlags
+		if !rp.noEndStream {
+			dataFlags = spdy.DataFlagFin
+		}
 		if err := st.spdyFr.WriteFrame(&spdy.DataFrame{
 			StreamId: id,
-			Flags:    spdy.DataFlagFin,
+			Flags:    dataFlags,
 			Data:     rp.body,
 		}); err != nil {
 			return nil, err
@@ -589,7 +605,7 @@ func (st *serverTester) http2(rp requestParam) (*serverResponse, error) {
 
 		err := st.fr.WriteHeaders(http2.HeadersFrameParam{
 			StreamID:      id,
-			EndStream:     len(rp.body) == 0 && len(rp.trailer) == 0,
+			EndStream:     len(rp.body) == 0 && len(rp.trailer) == 0 && !rp.noEndStream,
 			EndHeaders:    true,
 			BlockFragment: st.headerBlkBuf.Bytes(),
 		})
@@ -599,7 +615,7 @@ func (st *serverTester) http2(rp requestParam) (*serverResponse, error) {
 
 		if len(rp.body) != 0 {
 			// TODO we assume rp.body fits in 1 frame
-			if err := st.fr.WriteData(id, len(rp.trailer) == 0, rp.body); err != nil {
+			if err := st.fr.WriteData(id, len(rp.trailer) == 0 && !rp.noEndStream, rp.body); err != nil {
 				return nil, err
 			}
 		}
@@ -746,3 +762,8 @@ func cloneHeader(h http.Header) http.Header {
 }
 
 func noopHandler(w http.ResponseWriter, r *http.Request) {}
+
+type APIResponse struct {
+	Status string `json:"status,omitempty"`
+	Code   int    `json:"code,omitempty"`
+}

lib/includes/nghttp2/nghttp2.h

@@ -81,7 +81,7 @@ extern "C" {
 /**
  * @macro
  *
- * The seriazlied form of ALPN protocol identifier this library
+ * The serialized form of ALPN protocol identifier this library
  * supports.  Notice that first byte is the length of following
  * protocol identifier.  This is the same wire format of `TLS ALPN
  * extension <https://tools.ietf.org/html/rfc7301>`_.  This is useful
@@ -1454,9 +1454,19 @@ typedef int (*nghttp2_on_data_chunk_recv_callback)(nghttp2_session *session,
  * `nghttp2_session_server_new()`.
  *
  * The implementation of this function must return 0 if it succeeds.
- * If nonzero is returned, it is treated as fatal error and
- * `nghttp2_session_send()` and `nghttp2_session_mem_send()` functions
- * immediately return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`.
+ * It can also return :enum:`NGHTTP2_ERR_CANCEL` to cancel the
+ * transmission of the given frame.
+ *
+ * If there is a fatal error while executing this callback, the
+ * implementation should return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`,
+ * which makes `nghttp2_session_send()` and
+ * `nghttp2_session_mem_send()` functions immediately return
+ * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`.
+ *
+ * If the other value is returned, it is treated as if
+ * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE` is returned.  But the
+ * implementation should not rely on this since the library may define
+ * new return value to extend its capability.
  *
  * To set this callback to :type:`nghttp2_session_callbacks`, use
  * `nghttp2_session_callbacks_set_before_frame_send_callback()`.
@@ -2412,6 +2422,21 @@ nghttp2_option_set_builtin_recv_extension_type(nghttp2_option *option,
 NGHTTP2_EXTERN void nghttp2_option_set_no_auto_ping_ack(nghttp2_option *option,
                                                         int val);
 
+/**
+ * @function
+ *
+ * This option sets the maximum length of header block (a set of
+ * header fields per one HEADERS frame) to send.  The length of a
+ * given set of header fields is calculated using
+ * `nghttp2_hd_deflate_bound()`.  The default value is 64KiB.  If
+ * application attempts to send header fields larger than this limit,
+ * the transmission of the frame fails with error code
+ * :enum:`NGHTTP2_ERR_FRAME_SIZE_ERROR`.
+ */
+NGHTTP2_EXTERN void
+nghttp2_option_set_max_send_header_block_length(nghttp2_option *option,
+                                                size_t val);
+
 /**
  * @function
  *
@@ -2604,13 +2629,19 @@ NGHTTP2_EXTERN void nghttp2_session_del(nghttp2_session *session);
  *
  * 6. :type:`nghttp2_before_frame_send_callback` is invoked.
  *
- * 7. :type:`nghttp2_send_callback` is invoked one or more times to
+ * 7. If :enum:`NGHTTP2_ERR_CANCEL` is returned from
+ *    :type:`nghttp2_before_frame_send_callback`, the current frame
+ *    transmission is canceled, and
+ *    :type:`nghttp2_on_frame_not_send_callback` is invoked.  Abort
+ *    the following steps.
+ *
+ * 8. :type:`nghttp2_send_callback` is invoked one or more times to
  *    send the frame.
  *
- * 8. :type:`nghttp2_on_frame_send_callback` is invoked.
+ * 9. :type:`nghttp2_on_frame_send_callback` is invoked.
  *
- * 9. If the transmission of the frame triggers closure of the stream,
- *    the stream is closed and
+ * 10. If the transmission of the frame triggers closure of the
+ *     stream, the stream is closed and
  *     :type:`nghttp2_on_stream_close_callback` is invoked.
  *
  * This function returns 0 if it succeeds, or one of the following
@@ -3583,8 +3614,8 @@ nghttp2_submit_response(nghttp2_session *session, int32_t stream_id,
  *
  * The |nva| is an array of name/value pair :type:`nghttp2_nv` with
  * |nvlen| elements.  The application is responsible not to include
- * required pseudo-header fields (header field whose name starts with
- * ":") in |nva|.
+ * pseudo-header fields (header field whose name starts with ":") in
+ * |nva|.
  *
  * This function creates copies of all name/value pairs in |nva|.  It
  * also lower-cases all names in |nva|.  The order of elements in
@@ -3599,20 +3630,20 @@ nghttp2_submit_response(nghttp2_session *session, int32_t stream_id,
  * :type:`nghttp2_on_frame_not_send_callback` is called.
  *
  * For server, trailer fields must follow response HEADERS or response
- * DATA with END_STREAM flag set.  The library does not enforce this
- * requirement, and applications should do this for themselves.  If
- * `nghttp2_submit_trailer()` is called before any response HEADERS
+ * DATA without END_STREAM flat set.  The library does not enforce
+ * this requirement, and applications should do this for themselves.
+ * If `nghttp2_submit_trailer()` is called before any response HEADERS
  * submission (usually by `nghttp2_submit_response()`), the content of
  * |nva| will be sent as response headers, which will result in error.
  *
  * This function has the same effect with `nghttp2_submit_headers()`,
- * with flags = :enum:`NGHTTP2_FLAG_END_HEADERS` and both pri_spec and
+ * with flags = :enum:`NGHTTP2_FLAG_END_STREAM` and both pri_spec and
  * stream_user_data to NULL.
  *
  * To submit trailer fields after `nghttp2_submit_response()` is
  * called, the application has to specify
- * :type:`nghttp2_data_provider` to `nghttp2_submit_response()`.  In
- * side :type:`nghttp2_data_source_read_callback`, when setting
+ * :type:`nghttp2_data_provider` to `nghttp2_submit_response()`.
+ * Inside of :type:`nghttp2_data_source_read_callback`, when setting
  * :enum:`NGHTTP2_DATA_FLAG_EOF`, also set
  * :enum:`NGHTTP2_DATA_FLAG_NO_END_STREAM`.  After that, the
  * application can send trailer fields using
@@ -4071,14 +4102,17 @@ nghttp2_session_check_server_session(nghttp2_session *session);
  * that value as window_size_increment is queued.  If the
  * |window_size_increment| is larger than the received bytes from the
  * remote endpoint, the local window size is increased by that
- * difference.
+ * difference.  If the sole purpose is to increase the local window
+ * size, consider to use `nghttp2_session_set_local_window_size()`.
  *
  * If the |window_size_increment| is negative, the local window size
  * is decreased by -|window_size_increment|.  If automatic
  * WINDOW_UPDATE is enabled
  * (`nghttp2_option_set_no_auto_window_update()`), and the library
  * decided that the WINDOW_UPDATE should be submitted, then
- * WINDOW_UPDATE is queued with the current received bytes count.
+ * WINDOW_UPDATE is queued with the current received bytes count.  If
+ * the sole purpose is to decrease the local window size, consider to
+ * use `nghttp2_session_set_local_window_size()`.
  *
  * If the |window_size_increment| is 0, the function does nothing and
  * returns 0.
@@ -4096,6 +4130,44 @@ NGHTTP2_EXTERN int nghttp2_submit_window_update(nghttp2_session *session,
                                                 int32_t stream_id,
                                                 int32_t window_size_increment);
 
+/**
+ * @function
+ *
+ * Set local window size (local endpoints's window size) to the given
+ * |window_size| for the given stream denoted by |stream_id|.  To
+ * change connection level window size, specify 0 to |stream_id|.  To
+ * increase window size, this function may submit WINDOW_UPDATE frame
+ * to transmission queue.
+ *
+ * The |flags| is currently ignored and should be
+ * :enum:`NGHTTP2_FLAG_NONE`.
+ *
+ * This sounds similar to `nghttp2_submit_window_update()`, but there
+ * are 2 differences.  The first difference is that this function
+ * takes the absolute value of window size to set, rather than the
+ * delta.  To change the window size, this may be easier to use since
+ * the application just declares the intended window size, rather than
+ * calculating delta.  The second difference is that
+ * `nghttp2_submit_window_update()` affects the received bytes count
+ * which has not acked yet.  By the specification of
+ * `nghttp2_submit_window_update()`, to strictly increase the local
+ * window size, we have to submit delta including all received bytes
+ * count, which might not be desirable in some cases.  On the other
+ * hand, this function does not affect the received bytes count.  It
+ * just sets the local window size to the given value.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * :enum:`NGHTTP2_ERR_INVALID_ARGUMENT`
+ *     The |stream_id| is negative.
+ * :enum:`NGHTTP2_ERR_NOMEM`
+ *     Out of memory.
+ */
+NGHTTP2_EXTERN int
+nghttp2_session_set_local_window_size(nghttp2_session *session, uint8_t flags,
+                                      int32_t stream_id, int32_t window_size);
+
 /**
  * @function
  *

lib/nghttp2_frame.h

@@ -52,14 +52,12 @@
 #define NGHTTP2_FRAMEBUF_CHUNKLEN                                              \
   (NGHTTP2_FRAME_HDLEN + 1 + NGHTTP2_MAX_PAYLOADLEN)
 
-/* Number of inbound buffer */
-#define NGHTTP2_FRAMEBUF_MAX_NUM 5
-
 /* The default length of DATA frame payload. */
 #define NGHTTP2_DATA_PAYLOADLEN NGHTTP2_MAX_FRAME_SIZE_MIN
 
-/* Maximum headers payload length, calculated in compressed form.
-   This applies to transmission only. */
+/* Maximum headers block size to send, calculated using
+   nghttp2_hd_deflate_bound().  This is the default value, and can be
+   overridden by nghttp2_option_set_max_send_header_block_size(). */
 #define NGHTTP2_MAX_HEADERSLEN 65536
 
 /* The number of bytes for each SETTINGS entry */

lib/nghttp2_hd.c

@@ -994,7 +994,7 @@ static int emit_string(nghttp2_bufs *bufs, const uint8_t *str, size_t len) {
   blocklen = count_encoded_length(enclen, 7);
 
   DEBUGF(fprintf(stderr, "deflatehd: emit string str="));
-  DEBUGF(fwrite(str, len, 1, stderr));
+  DEBUGF(fwrite(str, 1, len, stderr));
   DEBUGF(fprintf(stderr, ", length=%zu, huffman=%d, encoded_length=%zu\n", len,
                  huffman, enclen));
 

lib/nghttp2_helper.c

@@ -213,6 +213,38 @@ int nghttp2_adjust_local_window_size(int32_t *local_window_size_ptr,
   return 0;
 }
 
+int nghttp2_increase_local_window_size(int32_t *local_window_size_ptr,
+                                       int32_t *recv_window_size_ptr,
+                                       int32_t *recv_reduction_ptr,
+                                       int32_t *delta_ptr) {
+  int32_t recv_reduction_delta;
+  int32_t delta;
+
+  delta = *delta_ptr;
+
+  assert(delta >= 0);
+
+  /* The delta size is strictly more than received bytes. Increase
+     local_window_size by that difference |delta|. */
+  if (*local_window_size_ptr > NGHTTP2_MAX_WINDOW_SIZE - delta) {
+    return NGHTTP2_ERR_FLOW_CONTROL;
+  }
+
+  *local_window_size_ptr += delta;
+  /* If there is recv_reduction due to earlier window_size
+     reduction, we have to adjust it too. */
+  recv_reduction_delta = nghttp2_min(*recv_reduction_ptr, delta);
+  *recv_reduction_ptr -= recv_reduction_delta;
+
+  *recv_window_size_ptr += recv_reduction_delta;
+
+  /* recv_reduction_delta must be paied from *delta_ptr, since it was
+     added in window size reduction (see below). */
+  *delta_ptr -= recv_reduction_delta;
+
+  return 0;
+}
+
 int nghttp2_should_send_window_update(int32_t local_window_size,
                                       int32_t recv_window_size) {
   return recv_window_size > 0 && recv_window_size >= local_window_size / 2;

lib/nghttp2_helper.h

@@ -89,6 +89,22 @@ int nghttp2_adjust_local_window_size(int32_t *local_window_size_ptr,
                                      int32_t *recv_reduction_ptr,
                                      int32_t *delta_ptr);
 
+/*
+ * This function works like nghttp2_adjust_local_window_size().  The
+ * difference is that this function assumes *delta_ptr >= 0, and
+ * *recv_window_size_ptr is not decreased by *delta_ptr.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * NGHTTP2_ERR_FLOW_CONTROL
+ *     local_window_size overflow or gets negative.
+ */
+int nghttp2_increase_local_window_size(int32_t *local_window_size_ptr,
+                                       int32_t *recv_window_size_ptr,
+                                       int32_t *recv_reduction_ptr,
+                                       int32_t *delta_ptr);
+
 /*
  * Returns non-zero if the function decided that WINDOW_UPDATE should
  * be sent.

lib/nghttp2_mem.c

@@ -52,8 +52,8 @@ void nghttp2_mem_free(nghttp2_mem *mem, void *ptr) {
   mem->free(ptr, mem->mem_user_data);
 }
 
-void nghttp2_mem_free2(nghttp2_free free, void *ptr, void *mem_user_data) {
-  free(ptr, mem_user_data);
+void nghttp2_mem_free2(nghttp2_free free_func, void *ptr, void *mem_user_data) {
+  free_func(ptr, mem_user_data);
 }
 
 void *nghttp2_mem_calloc(nghttp2_mem *mem, size_t nmemb, size_t size) {

lib/nghttp2_mem.h

@@ -38,7 +38,7 @@ nghttp2_mem *nghttp2_mem_default(void);
    |mem|. */
 void *nghttp2_mem_malloc(nghttp2_mem *mem, size_t size);
 void nghttp2_mem_free(nghttp2_mem *mem, void *ptr);
-void nghttp2_mem_free2(nghttp2_free free, void *ptr, void *mem_user_data);
+void nghttp2_mem_free2(nghttp2_free free_func, void *ptr, void *mem_user_data);
 void *nghttp2_mem_calloc(nghttp2_mem *mem, size_t nmemb, size_t size);
 void *nghttp2_mem_realloc(nghttp2_mem *mem, void *ptr, size_t size);
 

lib/nghttp2_option.c

@@ -95,3 +95,9 @@ void nghttp2_option_set_no_auto_ping_ack(nghttp2_option *option, int val) {
   option->opt_set_mask |= NGHTTP2_OPT_NO_AUTO_PING_ACK;
   option->no_auto_ping_ack = val;
 }
+
+void nghttp2_option_set_max_send_header_block_length(nghttp2_option *option,
+                                                     size_t val) {
+  option->opt_set_mask |= NGHTTP2_OPT_MAX_SEND_HEADER_BLOCK_LENGTH;
+  option->max_send_header_block_length = val;
+}

lib/nghttp2_option.h

@@ -62,13 +62,18 @@ typedef enum {
   NGHTTP2_OPT_MAX_RESERVED_REMOTE_STREAMS = 1 << 4,
   NGHTTP2_OPT_USER_RECV_EXT_TYPES = 1 << 5,
   NGHTTP2_OPT_NO_AUTO_PING_ACK = 1 << 6,
-  NGHTTP2_OPT_BUILTIN_RECV_EXT_TYPES = 1 << 7
+  NGHTTP2_OPT_BUILTIN_RECV_EXT_TYPES = 1 << 7,
+  NGHTTP2_OPT_MAX_SEND_HEADER_BLOCK_LENGTH = 1 << 8,
 } nghttp2_option_flag;
 
 /**
  * Struct to store option values for nghttp2_session.
  */
 struct nghttp2_option {
+  /**
+   * NGHTTP2_OPT_MAX_SEND_HEADER_BLOCK_LENGTH
+   */
+  size_t max_send_header_block_length;
   /**
    * Bitwise OR of nghttp2_option_flag to determine that which fields
    * are specified.

lib/nghttp2_session.c

@@ -389,6 +389,7 @@ static int session_new(nghttp2_session **session_ptr,
                        void *user_data, int server,
                        const nghttp2_option *option, nghttp2_mem *mem) {
   int rv;
+  size_t nbuffer;
 
   if (mem == NULL) {
     mem = nghttp2_mem_default();
@@ -441,16 +442,6 @@ static int session_new(nghttp2_session **session_ptr,
     (*session_ptr)->server = 1;
   }
 
-  /* 1 for Pad Field. */
-  rv = nghttp2_bufs_init3(&(*session_ptr)->aob.framebufs,
-                          NGHTTP2_FRAMEBUF_CHUNKLEN, NGHTTP2_FRAMEBUF_MAX_NUM,
-                          1, NGHTTP2_FRAME_HDLEN + 1, mem);
-  if (rv != 0) {
-    goto fail_aob_framebuf;
-  }
-
-  active_outbound_item_reset(&(*session_ptr)->aob, mem);
-
   init_settings(&(*session_ptr)->remote_settings);
   init_settings(&(*session_ptr)->local_settings);
 
@@ -460,6 +451,8 @@ static int session_new(nghttp2_session **session_ptr,
   /* Limit max outgoing concurrent streams to sensible value */
   (*session_ptr)->remote_settings.max_concurrent_streams = 100;
 
+  (*session_ptr)->max_send_header_block_length = NGHTTP2_MAX_HEADERSLEN;
+
   if (option) {
     if ((option->opt_set_mask & NGHTTP2_OPT_NO_AUTO_WINDOW_UPDATE) &&
         option->no_auto_window_update) {
@@ -504,7 +497,30 @@ static int session_new(nghttp2_session **session_ptr,
         option->no_auto_ping_ack) {
       (*session_ptr)->opt_flags |= NGHTTP2_OPTMASK_NO_AUTO_PING_ACK;
     }
+
+    if (option->opt_set_mask & NGHTTP2_OPT_MAX_SEND_HEADER_BLOCK_LENGTH) {
+      (*session_ptr)->max_send_header_block_length =
+          option->max_send_header_block_length;
     }
+  }
+
+  nbuffer = ((*session_ptr)->max_send_header_block_length +
+             NGHTTP2_FRAMEBUF_CHUNKLEN - 1) /
+            NGHTTP2_FRAMEBUF_CHUNKLEN;
+
+  if (nbuffer == 0) {
+    nbuffer = 1;
+  }
+
+  /* 1 for Pad Field. */
+  rv = nghttp2_bufs_init3(&(*session_ptr)->aob.framebufs,
+                          NGHTTP2_FRAMEBUF_CHUNKLEN, nbuffer, 1,
+                          NGHTTP2_FRAME_HDLEN + 1, mem);
+  if (rv != 0) {
+    goto fail_aob_framebuf;
+  }
+
+  active_outbound_item_reset(&(*session_ptr)->aob, mem);
 
   (*session_ptr)->callbacks = *callbacks;
   (*session_ptr)->user_data = user_data;
@@ -1951,7 +1967,7 @@ static int session_prep_frame(nghttp2_session *session,
             session, frame->headers.nva, frame->headers.nvlen,
             NGHTTP2_PRIORITY_SPECLEN);
 
-        if (estimated_payloadlen > NGHTTP2_MAX_HEADERSLEN) {
+        if (estimated_payloadlen > session->max_send_header_block_length) {
           return NGHTTP2_ERR_FRAME_SIZE_ERROR;
         }
 
@@ -1970,7 +1986,7 @@ static int session_prep_frame(nghttp2_session *session,
             session, frame->headers.nva, frame->headers.nvlen,
             NGHTTP2_PRIORITY_SPECLEN);
 
-        if (estimated_payloadlen > NGHTTP2_MAX_HEADERSLEN) {
+        if (estimated_payloadlen > session->max_send_header_block_length) {
           return NGHTTP2_ERR_FRAME_SIZE_ERROR;
         }
 
@@ -2089,7 +2105,7 @@ static int session_prep_frame(nghttp2_session *session,
       estimated_payloadlen = session_estimate_headers_payload(
           session, frame->push_promise.nva, frame->push_promise.nvlen, 0);
 
-      if (estimated_payloadlen > NGHTTP2_MAX_HEADERSLEN) {
+      if (estimated_payloadlen > session->max_send_header_block_length) {
         return NGHTTP2_ERR_FRAME_SIZE_ERROR;
       }
 
@@ -2348,6 +2364,10 @@ static int session_call_before_frame_send(nghttp2_session *session,
   if (session->callbacks.before_frame_send_callback) {
     rv = session->callbacks.before_frame_send_callback(session, frame,
                                                        session->user_data);
+    if (rv == NGHTTP2_ERR_CANCEL) {
+      return rv;
+    }
+
     if (rv != 0) {
       return NGHTTP2_ERR_CALLBACK_FAILURE;
     }
@@ -2988,6 +3008,51 @@ static ssize_t nghttp2_session_mem_send_internal(nghttp2_session *session,
         if (nghttp2_is_fatal(rv)) {
           return rv;
         }
+
+        if (rv == NGHTTP2_ERR_CANCEL) {
+          int32_t opened_stream_id = 0;
+          uint32_t error_code = NGHTTP2_INTERNAL_ERROR;
+
+          if (session->callbacks.on_frame_not_send_callback) {
+            if (session->callbacks.on_frame_not_send_callback(
+                    session, frame, rv, session->user_data) != 0) {
+              return NGHTTP2_ERR_CALLBACK_FAILURE;
+            }
+          }
+
+          /* We have to close stream opened by canceled request
+             HEADERS or PUSH_PROMISE. */
+          switch (item->frame.hd.type) {
+          case NGHTTP2_HEADERS:
+            if (item->frame.headers.cat == NGHTTP2_HCAT_REQUEST) {
+              opened_stream_id = item->frame.hd.stream_id;
+              /* We don't have to check
+                 item->aux_data.headers.canceled since it has already
+                 been checked. */
+              /* Set error_code to REFUSED_STREAM so that application
+                 can send request again. */
+              error_code = NGHTTP2_REFUSED_STREAM;
+            }
+            break;
+          case NGHTTP2_PUSH_PROMISE:
+            opened_stream_id = item->frame.push_promise.promised_stream_id;
+            break;
+          }
+          if (opened_stream_id) {
+            /* careful not to override rv */
+            int rv2;
+            rv2 = nghttp2_session_close_stream(session, opened_stream_id,
+                                               error_code);
+
+            if (nghttp2_is_fatal(rv2)) {
+              return rv2;
+            }
+          }
+
+          active_outbound_item_reset(aob, mem);
+
+          break;
+        }
       } else {
         DEBUGF(fprintf(stderr, "send: next frame: DATA\n"));
 
@@ -5588,7 +5653,6 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in,
                                iframe->frame.hd.type)) {
           if (!session->callbacks.unpack_extension_callback) {
             /* Silently ignore unknown frame type. */
-
             busy = 1;
 
             iframe->state = NGHTTP2_IB_IGN_PAYLOAD;
@@ -6666,17 +6730,12 @@ int nghttp2_session_add_window_update(nghttp2_session *session, uint8_t flags,
 static void
 session_append_inflight_settings(nghttp2_session *session,
                                  nghttp2_inflight_settings *settings) {
-  nghttp2_inflight_settings *i;
+  nghttp2_inflight_settings **i;
 
-  if (!session->inflight_settings_head) {
-    session->inflight_settings_head = settings;
-    return;
-  }
-
-  for (i = session->inflight_settings_head; i->next; i = i->next)
+  for (i = &session->inflight_settings_head; *i; i = &(*i)->next)
     ;
 
-  i->next = settings;
+  *i = settings;
 }
 
 int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags,
@@ -6750,9 +6809,9 @@ int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags,
 
   if (flags & NGHTTP2_FLAG_ACK) {
     ++session->obq_flood_counter_;
-  }
-
+  } else {
     session_append_inflight_settings(session, inflight_settings);
+  }
 
   /* Extract NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS and ENABLE_PUSH
      here.  We use it to refuse the incoming stream and PUSH_PROMISE

lib/nghttp2_session.h

@@ -256,6 +256,9 @@ struct nghttp2_session {
   size_t nvbuflen;
   /* Counter for detecting flooding in outbound queue */
   size_t obq_flood_counter_;
+  /* The maximum length of header block to send.  Calculated by the
+     same way as nghttp2_hd_deflate_bound() does. */
+  size_t max_send_header_block_length;
   /* Next Stream ID. Made unsigned int to detect >= (1 << 31). */
   uint32_t next_stream_id;
   /* The last stream ID this session initiated.  For client session,

lib/nghttp2_submit.c

@@ -410,6 +410,75 @@ int nghttp2_submit_window_update(nghttp2_session *session, uint8_t flags,
   return 0;
 }
 
+int nghttp2_session_set_local_window_size(nghttp2_session *session,
+                                          uint8_t flags, int32_t stream_id,
+                                          int32_t window_size) {
+  int32_t window_size_increment;
+  nghttp2_stream *stream;
+  int rv;
+
+  if (window_size < 0) {
+    return NGHTTP2_ERR_INVALID_ARGUMENT;
+  }
+
+  flags = 0;
+
+  if (stream_id == 0) {
+    window_size_increment = window_size - session->local_window_size;
+
+    if (window_size_increment == 0) {
+      return 0;
+    }
+
+    if (window_size_increment < 0) {
+      return nghttp2_adjust_local_window_size(
+          &session->local_window_size, &session->recv_window_size,
+          &session->recv_reduction, &window_size_increment);
+    }
+
+    rv = nghttp2_increase_local_window_size(
+        &session->local_window_size, &session->recv_window_size,
+        &session->recv_reduction, &window_size_increment);
+
+    if (rv != 0) {
+      return rv;
+    }
+  } else {
+    stream = nghttp2_session_get_stream(session, stream_id);
+
+    if (stream == NULL) {
+      return 0;
+    }
+
+    window_size_increment = window_size - stream->local_window_size;
+
+    if (window_size_increment == 0) {
+      return 0;
+    }
+
+    if (window_size_increment < 0) {
+      return nghttp2_adjust_local_window_size(
+          &stream->local_window_size, &stream->recv_window_size,
+          &stream->recv_reduction, &window_size_increment);
+    }
+
+    rv = nghttp2_increase_local_window_size(
+        &stream->local_window_size, &stream->recv_window_size,
+        &stream->recv_reduction, &window_size_increment);
+
+    if (rv != 0) {
+      return rv;
+    }
+  }
+
+  if (window_size_increment > 0) {
+    return nghttp2_session_add_window_update(session, flags, stream_id,
+                                             window_size_increment);
+  }
+
+  return 0;
+}
+
 int nghttp2_submit_altsvc(nghttp2_session *session, uint8_t flags _U_,
                           int32_t stream_id, const uint8_t *origin,
                           size_t origin_len, const uint8_t *field_value,

src/CMakeLists.txt

@@ -39,7 +39,7 @@ link_libraries(
 if(ENABLE_APP)
   set(HELPER_OBJECTS
     util.cc
-    http2.cc timegm.c app_helper.cc nghttp2_gzip.c
+    http2.cc timegm.c app_helper.cc nghttp2_gzip.c cache_digest.cc
   )
 
   # nghttp client
@@ -109,6 +109,9 @@ if(ENABLE_APP)
     shrpx_worker_process.cc
     shrpx_signal.cc
     shrpx_router.cc
+    shrpx_api_downstream_connection.cc
+    shrpx_health_monitor_downstream_connection.cc
+    cache_digest.cc
   )
   if(HAVE_SPDYLAY)
     list(APPEND NGHTTPX_SRCS
@@ -148,6 +151,7 @@ if(ENABLE_APP)
       shrpx_config_test.cc
       shrpx_worker_test.cc
       shrpx_http_test.cc
+      shrpx_router_test.cc
       http2_test.cc
       util_test.cc
       nghttp2_gzip_test.c
@@ -156,6 +160,7 @@ if(ENABLE_APP)
       memchunk_test.cc
       template_test.cc
       base64_test.cc
+      cache_digest_test.cc
     )
     add_executable(nghttpx-unittest EXCLUDE_FROM_ALL
       ${NGHTTPX_UNITTEST_SOURCES}
@@ -163,7 +168,7 @@ if(ENABLE_APP)
     )
     target_include_directories(nghttpx-unittest PRIVATE ${CUNIT_INCLUDE_DIRS})
     target_compile_definitions(nghttpx-unittest
-      PRIVATE "-DNGHTTP2_TESTS_DIR=\"${CMAKE_SOURCE_DIR}/tests\""
+      PRIVATE "-DNGHTTP2_SRC_DIR=\"${CMAKE_SOURCE_DIR}/src\""
     )
     target_link_libraries(nghttpx-unittest nghttpx_static ${CUNIT_LIBRARIES})
     if(HAVE_MRUBY)

src/HttpServer.cc

@@ -62,6 +62,7 @@
 #include "util.h"
 #include "ssl.h"
 #include "template.h"
+#include "cache_digest.h"
 
 #ifndef O_BINARY
 #define O_BINARY (0)
@@ -825,10 +826,22 @@ int Http2Handler::on_write() { return write_(*this); }
 int Http2Handler::connection_made() {
   int r;
 
-  r = nghttp2_session_server_new(&session_, sessions_->get_callbacks(), this);
+  nghttp2_option *opt;
+  r = nghttp2_option_new(&opt);
 
   if (r != 0) {
-    return r;
+    return -1;
+  }
+
+  nghttp2_option_set_user_recv_extension_type(opt, NGHTTP2_DRAFT_CACHE_DIGEST);
+
+  r = nghttp2_session_server_new2(&session_, sessions_->get_callbacks(), this,
+                                  opt);
+
+  nghttp2_option_del(opt);
+
+  if (r != 0) {
+    return -1;
   }
 
   auto config = sessions_->get_config();
@@ -852,16 +865,15 @@ int Http2Handler::connection_made() {
 
   r = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, entry.data(), niv);
   if (r != 0) {
-    return r;
+    return -1;
   }
 
   if (config->connection_window_bits != -1) {
-    r = nghttp2_submit_window_update(
+    r = nghttp2_session_set_local_window_size(
         session_, NGHTTP2_FLAG_NONE, 0,
-        (1 << config->connection_window_bits) - 1 -
-            NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE);
+        (1 << config->connection_window_bits) - 1);
     if (r != 0) {
-      return r;
+      return -1;
     }
   }
 
@@ -1065,6 +1077,39 @@ void Http2Handler::terminate_session(uint32_t error_code) {
   nghttp2_session_terminate_session(session_, error_code);
 }
 
+std::vector<uint8_t> &Http2Handler::get_extbuf() { return extbuf_; }
+
+void Http2Handler::set_cache_digest(const StringRef &authority,
+                                    std::unique_ptr<CacheDigest> cache_digest) {
+  origin_cache_digest_[authority.str()] = std::move(cache_digest);
+}
+
+bool Http2Handler::cache_digest_includes(const StringRef &authority,
+                                         const StringRef &uri) const {
+  uint64_t key;
+  int rv;
+
+  auto it = origin_cache_digest_.find(authority.str());
+
+  if (it == std::end(origin_cache_digest_)) {
+    return false;
+  }
+
+  auto &cache_digest = (*it).second;
+
+  auto key_nbits = cache_digest->logn + cache_digest->logp;
+
+  rv = cache_digest_hash(key, key_nbits, uri);
+
+  if (rv != 0) {
+    return false;
+  }
+
+  const auto &keys = cache_digest->keys;
+
+  return std::binary_search(std::begin(keys), std::end(keys), key);
+}
+
 ssize_t file_read_callback(nghttp2_session *session, int32_t stream_id,
                            uint8_t *buf, size_t length, uint32_t *data_flags,
                            nghttp2_data_source *source, void *user_data) {
@@ -1289,7 +1334,7 @@ void prepare_response(Stream *stream, Http2Handler *hd,
     p = std::copy(std::begin(htdocs), std::end(htdocs), p);
     p = std::copy(std::begin(path), std::end(path), p);
     if (trailing_slash) {
-      p = std::copy(std::begin(DEFAULT_HTML), std::end(DEFAULT_HTML), p);
+      std::copy(std::begin(DEFAULT_HTML), std::end(DEFAULT_HTML), p);
     }
   }
 
@@ -1543,6 +1588,20 @@ int hd_on_frame_recv_callback(nghttp2_session *session,
       hd->remove_settings_timer();
     }
     break;
+  case NGHTTP2_DRAFT_CACHE_DIGEST: {
+    auto stream = hd->get_stream(frame->hd.stream_id);
+    if (!stream) {
+      return 0;
+    }
+
+    auto &header = stream->header;
+
+    hd->set_cache_digest(header.authority,
+                         std::unique_ptr<CacheDigest>(
+                             static_cast<CacheDigest *>(frame->ext.payload)));
+
+    break;
+  }
   default:
     break;
   }
@@ -1550,6 +1609,36 @@ int hd_on_frame_recv_callback(nghttp2_session *session,
 }
 } // namespace
 
+namespace {
+int before_frame_send_callback(nghttp2_session *session,
+                               const nghttp2_frame *frame, void *user_data) {
+  auto hd = static_cast<Http2Handler *>(user_data);
+
+  if (frame->hd.type != NGHTTP2_PUSH_PROMISE) {
+    return 0;
+  }
+
+  auto promised_stream = hd->get_stream(frame->push_promise.promised_stream_id);
+
+  if (promised_stream == nullptr) {
+    return 0;
+  }
+
+  auto &header = promised_stream->header;
+
+  auto uri = header.scheme.str();
+  uri += "://";
+  uri += header.authority;
+  uri += header.path;
+
+  if (hd->cache_digest_includes(header.authority, StringRef{uri})) {
+    return NGHTTP2_ERR_CANCEL;
+  }
+
+  return 0;
+}
+} // namespace
+
 namespace {
 int hd_on_frame_send_callback(nghttp2_session *session,
                               const nghttp2_frame *frame, void *user_data) {
@@ -1722,6 +1811,46 @@ int on_stream_close_callback(nghttp2_session *session, int32_t stream_id,
 }
 } // namespace
 
+namespace {
+int on_extension_chunk_recv_callback(nghttp2_session *session,
+                                     const nghttp2_frame_hd *frame_hd,
+                                     const uint8_t *data, size_t len,
+                                     void *user_data) {
+  auto hd = static_cast<Http2Handler *>(user_data);
+
+  auto &buf = hd->get_extbuf();
+  buf.insert(std::end(buf), data, data + len);
+
+  return 0;
+}
+} // namespace
+
+namespace {
+int unpack_extension_callback(nghttp2_session *session, void **payload,
+                              const nghttp2_frame_hd *frame_hd,
+                              void *user_data) {
+  int rv;
+  auto hd = static_cast<Http2Handler *>(user_data);
+
+  auto cache_digest = make_unique<CacheDigest>();
+
+  auto &buf = hd->get_extbuf();
+
+  rv = cache_digest_decode(cache_digest->keys, cache_digest->logn,
+                           cache_digest->logp, buf.data(), buf.size());
+
+  buf.clear();
+
+  if (rv != 0) {
+    return NGHTTP2_ERR_CANCEL;
+  }
+
+  *payload = cache_digest.release();
+
+  return 0;
+}
+} // namespace
+
 namespace {
 void fill_callback(nghttp2_session_callbacks *callbacks, const Config *config) {
   nghttp2_session_callbacks_set_on_stream_close_callback(
@@ -1757,6 +1886,15 @@ void fill_callback(nghttp2_session_callbacks *callbacks, const Config *config) {
     nghttp2_session_callbacks_set_select_padding_callback(
         callbacks, select_padding_callback);
   }
+
+  nghttp2_session_callbacks_set_unpack_extension_callback(
+      callbacks, unpack_extension_callback);
+
+  nghttp2_session_callbacks_set_on_extension_chunk_recv_callback(
+      callbacks, on_extension_chunk_recv_callback);
+
+  nghttp2_session_callbacks_set_before_frame_send_callback(
+      callbacks, before_frame_send_callback);
 }
 } // namespace
 

src/HttpServer.h

@@ -153,6 +153,12 @@ struct Stream {
 
 class Sessions;
 
+struct CacheDigest {
+  std::vector<uint64_t> keys;
+  uint32_t logn;
+  uint32_t logp;
+};
+
 class Http2Handler {
 public:
   Http2Handler(Sessions *sessions, int fd, SSL *ssl, int64_t session_id);
@@ -206,6 +212,15 @@ public:
 
   WriteBuf *get_wb();
 
+  std::vector<uint8_t> &get_extbuf();
+
+  // Sets given cache digest.  Overwrites existing one if any.
+  void set_cache_digest(const StringRef &origin,
+                        std::unique_ptr<CacheDigest> cache_digest);
+  // Returns true if |uri| is included in cache digest.
+  bool cache_digest_includes(const StringRef &origin,
+                             const StringRef &uri) const;
+
 private:
   ev_io wev_;
   ev_io rev_;
@@ -213,6 +228,10 @@ private:
   std::map<int32_t, std::unique_ptr<Stream>> id2stream_;
   WriteBuf wb_;
   std::function<int(Http2Handler &)> read_, write_;
+  // Received cache digest hash keys per origin
+  std::map<std::string, std::unique_ptr<CacheDigest>> origin_cache_digest_;
+  // Buffer for extension frame payload
+  std::vector<uint8_t> extbuf_;
   int64_t session_id_;
   nghttp2_session *session_;
   Sessions *sessions_;

src/Makefile.am

@@ -22,7 +22,10 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 SUBDIRS = includes
 
-EXTRA_DIST = CMakeLists.txt
+EXTRA_DIST = \
+	CMakeLists.txt \
+	test.example.com.pem \
+	test.nghttp2.org.pem
 
 bin_PROGRAMS =
 check_PROGRAMS =
@@ -61,10 +64,10 @@ if ENABLE_APP
 bin_PROGRAMS += nghttp nghttpd nghttpx
 
 HELPER_OBJECTS = util.cc \
-	http2.cc timegm.c app_helper.cc nghttp2_gzip.c
+	http2.cc timegm.c app_helper.cc nghttp2_gzip.c cache_digest.cc
 HELPER_HFILES = util.h \
 	http2.h timegm.h app_helper.h nghttp2_config.h \
-	nghttp2_gzip.h network.h
+	nghttp2_gzip.h network.h cache_digest.h
 
 HTML_PARSER_OBJECTS =
 HTML_PARSER_HFILES = HtmlParser.h
@@ -132,7 +135,11 @@ NGHTTPX_SRCS = \
 	shrpx_process.h \
 	shrpx_signal.cc shrpx_signal.h \
 	shrpx_router.cc shrpx_router.h \
-	buffer.h memchunk.h template.h allocator.h
+	shrpx_api_downstream_connection.cc shrpx_api_downstream_connection.h \
+	shrpx_health_monitor_downstream_connection.cc \
+	shrpx_health_monitor_downstream_connection.h \
+	buffer.h memchunk.h template.h allocator.h \
+	cache_digest.cc cache_digest.h
 
 if HAVE_SPDYLAY
 NGHTTPX_SRCS += shrpx_spdy_upstream.cc shrpx_spdy_upstream.h
@@ -174,6 +181,7 @@ nghttpx_unittest_SOURCES = shrpx-unittest.cc \
 	shrpx_config_test.cc shrpx_config_test.h \
 	shrpx_worker_test.cc shrpx_worker_test.h \
 	shrpx_http_test.cc shrpx_http_test.h \
+	shrpx_router_test.cc shrpx_router_test.h \
 	http2_test.cc http2_test.h \
 	util_test.cc util_test.h \
 	nghttp2_gzip_test.c nghttp2_gzip_test.h \
@@ -181,9 +189,10 @@ nghttpx_unittest_SOURCES = shrpx-unittest.cc \
 	buffer_test.cc buffer_test.h \
 	memchunk_test.cc memchunk_test.h \
 	template_test.cc template_test.h \
-	base64_test.cc base64_test.h
+	base64_test.cc base64_test.h \
+	cache_digest_test.cc cache_digest_test.h
 nghttpx_unittest_CPPFLAGS = ${AM_CPPFLAGS} \
-	-DNGHTTP2_TESTS_DIR=\"$(top_srcdir)/tests\"
+	-DNGHTTP2_SRC_DIR=\"$(top_srcdir)/src\"
 nghttpx_unittest_LDADD = libnghttpx.a ${LDADD} @CUNIT_LIBS@ @TESTLDADD@
 
 if HAVE_MRUBY

src/allocator.h

@@ -54,20 +54,45 @@ struct BlockAllocator {
         block_size(block_size),
         isolation_threshold(std::min(block_size, isolation_threshold)) {}
 
-  ~BlockAllocator() {
+  ~BlockAllocator() { reset(); }
+
+  BlockAllocator(BlockAllocator &&other) noexcept
+      : retain(other.retain),
+        head(other.head),
+        block_size(other.block_size),
+        isolation_threshold(other.isolation_threshold) {
+    other.retain = nullptr;
+    other.head = nullptr;
+  }
+
+  BlockAllocator &operator=(BlockAllocator &&other) noexcept {
+    reset();
+
+    retain = other.retain;
+    head = other.head;
+    block_size = other.block_size;
+    isolation_threshold = other.isolation_threshold;
+
+    other.retain = nullptr;
+    other.head = nullptr;
+
+    return *this;
+  }
+
+  BlockAllocator(const BlockAllocator &) = delete;
+  BlockAllocator &operator=(const BlockAllocator &) = delete;
+
+  void reset() {
     for (auto mb = retain; mb;) {
       auto next = mb->next;
       delete[] reinterpret_cast<uint8_t *>(mb);
       mb = next;
     }
+
+    retain = nullptr;
+    head = nullptr;
   }
 
-  BlockAllocator(BlockAllocator &&) = default;
-  BlockAllocator &operator=(BlockAllocator &&) = default;
-
-  BlockAllocator(const BlockAllocator &) = delete;
-  BlockAllocator &operator=(const BlockAllocator &) = delete;
-
   MemBlock *alloc_mem_block(size_t size) {
     auto block = new uint8_t[sizeof(MemBlock) + size];
     auto mb = reinterpret_cast<MemBlock *>(block);

src/app_helper.cc

@@ -106,6 +106,8 @@ std::string strframetype(uint8_t type) {
     return "WINDOW_UPDATE";
   case NGHTTP2_ALTSVC:
     return "ALTSVC";
+  case NGHTTP2_DRAFT_CACHE_DIGEST:
+    return "CACHE_DIGSET";
   }
 
   std::string s = "extension(0x";

src/app_helper.h

@@ -41,6 +41,11 @@
 
 namespace nghttp2 {
 
+enum nghttp2_draft_frame_type {
+  // draft-kazuho-h2-cache-digest-01
+  NGHTTP2_DRAFT_CACHE_DIGEST = 0xf1
+};
+
 int verbose_on_header_callback(nghttp2_session *session,
                                const nghttp2_frame *frame, const uint8_t *name,
                                size_t namelen, const uint8_t *value,

src/asio_client_session_impl.cc

@@ -373,9 +373,8 @@ bool session_impl::setup_session() {
        {NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, window_size}}};
   nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, iv.data(), iv.size());
   // increase connection window size up to window_size
-  nghttp2_submit_window_update(session_, NGHTTP2_FLAG_NONE, 0,
-                               window_size -
-                                   NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE);
+  nghttp2_session_set_local_window_size(session_, NGHTTP2_FLAG_NONE, 0,
+                                        window_size);
   return true;
 }
 

src/ca-config.json

@@ -0,0 +1,17 @@
+{
+    "signing": {
+	"default": {
+	    "expiry": "87600h"
+	},
+	"profiles": {
+	    "server": {
+		"expiry": "87600h",
+		"usages": [
+		    "signing",
+		    "key encipherment",
+		    "server auth"
+		]
+	    }
+	}
+    }
+}

src/ca.nghttp2.org-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEA1kVkF8QSUwW/HV9EFRPSMoiOVmYwB8vqKDtT0d6MFiKAM8/Y
+JFUq2uKlUydgT4IPE7PATvVcIj3GtL9XzPhscqYO/S0Y7scyTE2VAPmtz+StPWf2
+wZ1IQR09HrnDTc44KvYGZpefBZkD9UjbmJ9a1ZmJjJiMr3hTnKE/sxZ2+dMsnMZX
+N822cfaHyTN+T0+Tyw5vBBboCDsZzxmf+9FFIDJNs3NL34cR8EZRhpfaegapH8bt
+OJ+D+RZ2kg7E/YYkGcS6NodvTjSUFCFHpWjHCfTFhn/owBIAooCdWorh6dc8Q72l
+AodwNLXS8uuPgPqM5s4Cz57m7Zgs4OilNmIdawIDAQABAoIBAQCwqLtygLye6KD+
+RXorapEmCsJX5553/x6Klwdvg+25ni5XCWjp47IWj0DBQzi7tL5bfxrxvod8z7QR
+d6SbIMLA77px8Ima7G7CzEAqcrBkM+TFOP8P+G4HCWVH/N5SOtDCUt9KHH4Grna9
+95jdx5yreRAX8/oh/bHp9GRBcicbpwYMVWOnjTE2seEUYQOpdpYdP4bOPUvAju0l
+mwmy2/dDGmbibktN3sdHEhDodKu+Znv7nFZo0jzhlyoXse653WcvaQeZZYuojvSe
+Sr92DvPp7UaYrb4KvT7ujXiPavSV2m/4EmGtyqevUf2dZ6sfMXZjmXsjWz9txhWp
+4BgbHyHRAoGBAPqyuNj2CDD3FE7N3Hxyba8d+ZtsVUNawjq2gwOvT9NLsMstOGyH
+OCc1v4W6Sq4w1wo4nIJyY8kNZwtReaTHOPZlDgBhVvk/x8eLBu+QTMRyocRt1LoD
+8HyKxWSAnYTtCh/GUEQ37amIqvOJ5GNL+25WDzevLa5kMYWG743uxEupAoGBANrN
+c/fVxepvP0GISlLpL3aZCFGAjMrq3xUYcf/w4wPoMq6AdpIPeRVBmJ1/Uqw1FkV8
+NRKJNPE2YcMuv8iMeQlacoPd34KT9ob80EYVlMwAkeC0NK+FfiM/UteR0wB49gmi
+ugX9YlJytOP9aUgPvEGT6l+XtgGC44W1TQWe62zzAoGBAKZenNU+0UjNb6isbToZ
+Jjkkh1Vhm2PLg0I7hM6ZNTxf6r+rDtrXEajTvnocmxrmRo796r+W8immv09/jl6P
+53l8rsIJ1xIqBYai+MNa29cyy6/zw0x++MVtwnlj8SUZubJEhVgAVbRAglKEnBBZ
+iE48xnSJyKMG0uZuGePzJEmhAoGBAIOHJcNBumum3DuklikpC+MbMyjrQbdpYRjp
+TP4x7AWZO34ysxQyQPNKL1feBfCHKRA0DiNKX4zwx+vw2lDQQKIiwNwMMCPqljOn
+HfxDVOMdJJQTP+iTMrQ1iLMVceXC0QQR0glvu/8b/SlgWD19WAmDxUwZgst9xw/F
+YLuUQKmJAoGAREeTugd4hc0U/YV/BQQjSCLhl11EtVry/oQMHj8KZpIJhP7tj8lw
+hSE0+z04oMhiTeq55PYKQkTo5l6V4PW0zfpEwlKEEm0erab1G9Ddh7us47XFcKLl
+Rmk192EVZ0lQuzftsYv7dzRLiAR7yDFXwD1ELIK/uPkwBtu7wtHlq+M=
+-----END RSA PRIVATE KEY-----

src/ca.nghttp2.org.csr

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICwjCCAaoCAQAwXjELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUx
+ITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEXMBUGA1UEAxMOY2Eu
+bmdodHRwMi5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDWRWQX
+xBJTBb8dX0QVE9IyiI5WZjAHy+ooO1PR3owWIoAzz9gkVSra4qVTJ2BPgg8Ts8BO
+9VwiPca0v1fM+Gxypg79LRjuxzJMTZUA+a3P5K09Z/bBnUhBHT0eucNNzjgq9gZm
+l58FmQP1SNuYn1rVmYmMmIyveFOcoT+zFnb50yycxlc3zbZx9ofJM35PT5PLDm8E
+FugIOxnPGZ/70UUgMk2zc0vfhxHwRlGGl9p6Bqkfxu04n4P5FnaSDsT9hiQZxLo2
+h29ONJQUIUelaMcJ9MWGf+jAEgCigJ1aiuHp1zxDvaUCh3A0tdLy64+A+ozmzgLP
+nubtmCzg6KU2Yh1rAgMBAAGgHzAdBgkqhkiG9w0BCQ4xEDAOMAwGA1UdEwQFMAMB
+Af8wDQYJKoZIhvcNAQELBQADggEBACI5v8GbOXKv38h9/tuGEwJ9uxpYEljgGt8h
+QL5lwfEifh/7A8b39b9JEzWk5hnMRCOb8J6Jc3/6nmVgtKkQ+Mceupqpwsp1gT/v
+uUoAkJE03Iuja9zLhHmy74oZ7LWOQrZ1T7Z0eGQ+5u+LBZiPKnKxmkLCQoUPTbc4
+NQ9BbKhr8OaoJ4DDvJnszcL7to6kih7SkdoNZsq4zB0/ai/cPhvoVgkYfbLH2++D
+Tcs7TqU2L7gKzqXUtHeAKM2y81ewL7QTrcYzgiW86s3NmquxZG5pq0mjD+P4BYLc
+MOdnCxKbBuE/1R29pa6+JKgc46jOa2yRgv5+8rXkkpu53Ke3FGc=
+-----END CERTIFICATE REQUEST-----

src/ca.nghttp2.org.csr.json

@@ -0,0 +1,17 @@
+{
+    "CN": "ca.nghttp2.org",
+    "key": {
+	"algo": "rsa",
+	"size": 2048
+    },
+    "ca": {
+	"expiry": "87600h"
+    },
+    "names": [
+	{
+	    "C": "AU",
+	    "ST": "Some-State",
+	    "O": "Internet Widgits Pty Ltd"
+	}
+    ]
+}

src/ca.nghttp2.org.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDrTCCApWgAwIBAgIUe4dvx8haIjsT3ZpNCMrl62Xk6E0wDQYJKoZIhvcNAQEL
+BQAwXjELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoT
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEXMBUGA1UEAxMOY2EubmdodHRwMi5v
+cmcwHhcNMTYwNjI1MDkzMzAwWhcNMjYwNjIzMDkzMzAwWjBeMQswCQYDVQQGEwJB
+VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0
+cyBQdHkgTHRkMRcwFQYDVQQDEw5jYS5uZ2h0dHAyLm9yZzCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBANZFZBfEElMFvx1fRBUT0jKIjlZmMAfL6ig7U9He
+jBYigDPP2CRVKtripVMnYE+CDxOzwE71XCI9xrS/V8z4bHKmDv0tGO7HMkxNlQD5
+rc/krT1n9sGdSEEdPR65w03OOCr2BmaXnwWZA/VI25ifWtWZiYyYjK94U5yhP7MW
+dvnTLJzGVzfNtnH2h8kzfk9Pk8sObwQW6Ag7Gc8Zn/vRRSAyTbNzS9+HEfBGUYaX
+2noGqR/G7Tifg/kWdpIOxP2GJBnEujaHb040lBQhR6Voxwn0xYZ/6MASAKKAnVqK
+4enXPEO9pQKHcDS10vLrj4D6jObOAs+e5u2YLODopTZiHWsCAwEAAaNjMGEwDgYD
+VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNA5xVR1Zcax
+RJL9VC6pzuLmvduGMB8GA1UdIwQYMBaAFNA5xVR1ZcaxRJL9VC6pzuLmvduGMA0G
+CSqGSIb3DQEBCwUAA4IBAQCmdVfn/hUyEdvkKG7svg5d8o6BENOj8695KtWmzJjK
+zxH8J5Vy3mn89XrHQ+BOYXCDPyhs0aDS8aq3Z+HY0n9z1oAicyGzlVwZQQNX3YId
+Y2vcf7qu/2ATm/1S+mebE1/EXMUlWISKKUYXjggCwFgjDhH87Ai+A8MKScVdmqgL
+Hf+fRSzH3ToW7BCXlRl5bPAq2g+v1ALYc8wU9cT1MYm4dqAXh870LGFyUpaSWmFr
+TtX1DXBTgLp62syNlDthAvGigYFDtCa4cDM2vdTD9wpec2V9EKpfVqiRDDuYjUVX
+UXl27MvkNWnEBKCIoNv5abWXpZVG2zQdEMmUOkVuAXUC
+-----END CERTIFICATE-----

src/cache_digest.cc

@@ -0,0 +1,395 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2016 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "cache_digest.h"
+
+#include <cassert>
+#include <array>
+#include <limits>
+
+#include <openssl/evp.h>
+
+namespace nghttp2 {
+
+namespace {
+// Truncates |md| to |nbits| bits counting from MSB.  |nbits| is
+// guaranteed to be less than or equal to 62.
+uint64_t truncate_hash(const uint8_t *md, uint32_t nbits) {
+  uint64_t v;
+
+  v = (static_cast<uint64_t>(md[0]) << 56) +
+      (static_cast<uint64_t>(md[1]) << 48) +
+      (static_cast<uint64_t>(md[2]) << 40) +
+      (static_cast<uint64_t>(md[3]) << 32) +
+      (static_cast<uint64_t>(md[4]) << 24) +
+      (static_cast<uint64_t>(md[5]) << 16) +
+      (static_cast<uint64_t>(md[6]) << 8) + static_cast<uint64_t>(md[31]);
+
+  v >>= 64 - nbits;
+
+  return v;
+}
+} // namespace
+
+namespace {
+int compute_hash_values(std::vector<uint64_t> &hash_values,
+                        const std::vector<std::string> &uris, uint32_t nbits) {
+  int rv;
+
+  if (nbits > 62) {
+    return -1;
+  }
+
+  auto ctx = EVP_MD_CTX_create();
+
+  hash_values.resize(uris.size());
+
+  std::array<uint8_t, 32> md;
+
+  auto p = std::begin(hash_values);
+  for (auto &u : uris) {
+    rv = EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr);
+    if (rv != 1) {
+      return -1;
+    }
+
+    rv = EVP_DigestUpdate(ctx, u.c_str(), u.size());
+    if (rv != 1) {
+      return -1;
+    }
+
+    unsigned int len = md.size();
+
+    rv = EVP_DigestFinal_ex(ctx, md.data(), &len);
+    if (rv != 1) {
+      return -1;
+    }
+
+    assert(len == 32);
+
+    *p++ = truncate_hash(md.data(), nbits);
+  }
+
+  EVP_MD_CTX_destroy(ctx);
+
+  return 0;
+}
+} // namespace
+
+namespace {
+std::pair<uint8_t *, size_t> append_uint32(uint8_t *p, size_t b, uint32_t v,
+                                           size_t nbits) {
+  v &= (1 << nbits) - 1;
+
+  if (8 > b + nbits) {
+    *p |= (v << (8 - b - nbits));
+    return {p, b + nbits};
+  }
+
+  if (8 == b + nbits) {
+    *p++ |= v;
+    return {p, 0};
+  }
+
+  auto h = 8 - b;
+  auto left = nbits - h;
+
+  *p++ |= (v >> left);
+  b = 0;
+
+  for (; left >= 8; left -= 8) {
+    *p++ = (v >> (left - 8)) & 0xff;
+  }
+
+  if (left > 0) {
+    *p = (v & ((1 << left) - 1)) << (8 - left);
+  }
+
+  return {p, left};
+}
+} // namespace
+
+namespace {
+std::pair<uint8_t *, size_t> append_0bit(uint8_t *p, size_t b, size_t nbits) {
+  if (8 > b + nbits) {
+    return {p, b + nbits};
+  }
+
+  if (8 == b + nbits) {
+    return {++p, 0};
+  }
+
+  nbits -= 8 - b;
+  ++p;
+
+  p += nbits / 8;
+
+  return {p, nbits % 8};
+}
+
+std::pair<uint8_t *, size_t> append_single_1bit(uint8_t *p, size_t b) {
+  if (8 > b + 1) {
+    *p |= (1 << (7 - b));
+    return {p, b + 1};
+  }
+
+  *p++ |= 1;
+
+  return {p, 0};
+}
+} // namespace
+
+ssize_t cache_digest_encode(uint8_t *data, size_t datalen,
+                            const std::vector<std::string> &uris,
+                            uint32_t logp) {
+  uint32_t n = 1;
+  uint32_t logn = 0;
+
+  if (logp > 31) {
+    return -1;
+  }
+
+  for (; n < uris.size(); n *= 2, ++logn)
+    ;
+
+  if (n - uris.size() > uris.size() - n / 2) {
+    --logn;
+  }
+
+  auto maxlen = 2 * n + n * logp;
+  if (maxlen > datalen) {
+    return -1;
+  }
+
+  std::vector<uint64_t> hash_values;
+
+  if (compute_hash_values(hash_values, uris, logn + logp) != 0) {
+    return -1;
+  }
+
+  std::sort(std::begin(hash_values), std::end(hash_values));
+
+  auto last = data;
+
+  size_t b = 0;
+
+  std::fill_n(data, maxlen, 0);
+
+  std::tie(last, b) = append_uint32(last, b, logn, 5);
+  std::tie(last, b) = append_uint32(last, b, logp, 5);
+
+  auto c = std::numeric_limits<uint64_t>::max();
+
+  for (auto v : hash_values) {
+    if (v == c) {
+      continue;
+    }
+    auto d = v - c - 1;
+    auto q = d >> logp;
+    auto r = d & ((1u << logp) - 1);
+
+    std::tie(last, b) = append_0bit(last, b, q);
+    std::tie(last, b) = append_single_1bit(last, b);
+    std::tie(last, b) = append_uint32(last, b, r, logp);
+
+    c = v;
+  }
+
+  if (b != 0) {
+    // we already zero-filled.
+    ++last;
+  }
+
+  return last - data;
+}
+
+int cache_digest_hash(uint64_t &key, size_t nbits, const StringRef &s) {
+  int rv;
+
+  std::array<uint8_t, 32> md;
+
+  auto ctx = EVP_MD_CTX_create();
+
+  rv = EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr);
+  if (rv != 1) {
+    return -1;
+  }
+
+  rv = EVP_DigestUpdate(ctx, s.c_str(), s.size());
+  if (rv != 1) {
+    return -1;
+  }
+
+  unsigned int len = md.size();
+
+  rv = EVP_DigestFinal_ex(ctx, md.data(), &len);
+  if (rv != 1) {
+    return -1;
+  }
+
+  assert(len == 32);
+
+  EVP_MD_CTX_destroy(ctx);
+
+  key = truncate_hash(md.data(), nbits);
+
+  return 0;
+}
+
+namespace {
+std::pair<const uint8_t *, size_t> read_uint32(uint32_t &res, size_t nbits,
+                                               const uint8_t *p, size_t b) {
+  if (b + nbits < 8) {
+    res = (*p >> (8 - b - nbits)) & ((1 << nbits) - 1);
+    return {p, b + nbits};
+  }
+
+  if (b + nbits == 8) {
+    res = *p & ((1 << nbits) - 1);
+    return {++p, 0};
+  }
+
+  res = *p & ((1 << (8 - b)) - 1);
+
+  ++p;
+  nbits -= 8 - b;
+
+  for (; nbits >= 8; nbits -= 8) {
+    res <<= 8;
+    res += *p++;
+  }
+
+  if (nbits) {
+    res <<= nbits;
+    res += *p >> (8 - nbits);
+  }
+
+  return {p, nbits};
+}
+} // namespace
+
+namespace {
+size_t leading_zero(uint8_t c) {
+  for (size_t i = 0; i < 8; ++i) {
+    if (c & (1 << (7 - i))) {
+      return i;
+    }
+  }
+
+  return 8;
+}
+} // namespace
+
+namespace {
+std::pair<const uint8_t *, size_t>
+read_until_1bit(uint32_t &res, const uint8_t *p, size_t b, const uint8_t *end) {
+  uint8_t mask = (1 << (8 - b)) - 1;
+
+  if (*p & mask) {
+    res = leading_zero(*p & mask) - b;
+    b += res + 1;
+    if (b == 8) {
+      return {++p, 0};
+    }
+    return {p, b};
+  }
+
+  res = 8 - b;
+
+  ++p;
+
+  for (; p != end; ++p, res += 8) {
+    if (!*p) {
+      continue;
+    }
+
+    auto nlz = leading_zero(*p);
+
+    res += nlz;
+    b = nlz + 1;
+
+    if (b == 8) {
+      return {++p, 0};
+    }
+    return {p, b};
+  }
+
+  return {end, 0};
+}
+} // namespace
+
+int cache_digest_decode(std::vector<uint64_t> &keys, uint32_t &logn,
+                        uint32_t &logp, const uint8_t *data, size_t datalen) {
+  auto last = data;
+  size_t b = 0;
+
+  auto end = data + datalen;
+
+  if ((end - data) * 8 < 10) {
+    return -1;
+  }
+
+  keys.resize(0);
+
+  logn = 0;
+  logp = 0;
+
+  std::tie(last, b) = read_uint32(logn, 5, last, b);
+  std::tie(last, b) = read_uint32(logp, 5, last, b);
+
+  uint64_t c = std::numeric_limits<uint64_t>::max();
+
+  for (;;) {
+    uint32_t q, r;
+
+    auto may_end = end - last == 1 && b > 0;
+    std::tie(last, b) = read_until_1bit(q, last, b, end);
+
+    if (last == end) {
+      if (may_end) {
+        return 0;
+      }
+
+      return -1;
+    }
+
+    if ((end - last) * 8 < static_cast<intptr_t>(b + logp)) {
+      return -1;
+    }
+
+    std::tie(last, b) = read_uint32(r, logp, last, b);
+
+    auto d = (static_cast<uint64_t>(q) << logp) + r;
+
+    c += d + 1;
+
+    keys.push_back(c);
+
+    if (last == end) {
+      return 0;
+    }
+  }
+}
+
+} // namespace nghttp2

src/cache_digest.h

@@ -0,0 +1,48 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2016 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CACHE_DIGEST_H
+#define CACHE_DIGEST_H
+
+#include "nghttp2_config.h"
+
+#include <vector>
+#include <string>
+
+#include "template.h"
+
+namespace nghttp2 {
+
+ssize_t cache_digest_encode(uint8_t *data, size_t datalen,
+                            const std::vector<std::string> &uris,
+                            uint32_t logp);
+
+int cache_digest_decode(std::vector<uint64_t> &keys, uint32_t &logn,
+                        uint32_t &logp, const uint8_t *data, size_t datalen);
+
+int cache_digest_hash(uint64_t &key, size_t nbits, const StringRef &s);
+
+} // namespace nghttp2
+
+#endif // CACHE_DIGEST_H

src/cache_digest_test.cc

@@ -0,0 +1,76 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2016 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "cache_digest_test.h"
+
+#include <vector>
+#include <algorithm>
+
+#include <CUnit/CUnit.h>
+
+#include "cache_digest.h"
+#include "template.h"
+
+namespace nghttp2 {
+
+void test_cache_digest_encode_decode(void) {
+  int rv;
+
+  auto uris = std::vector<std::string>{"https://nghttp2.org/foo",
+                                       "https://nghttp2.org/bar",
+                                       "https://nghttp2.org/buzz"};
+
+  auto pbits = 31;
+  std::array<uint8_t, 16_k> cdbuf;
+  auto cdlen = cache_digest_encode(cdbuf.data(), cdbuf.size(), uris, pbits);
+
+  std::vector<uint64_t> keys;
+  uint32_t logn, logp;
+
+  rv = cache_digest_decode(keys, logn, logp, cdbuf.data(), cdlen);
+
+  CU_ASSERT(0 == rv);
+
+  auto query_keys = std::vector<uint64_t>(uris.size());
+  for (size_t i = 0; i < uris.size(); ++i) {
+    auto &uri = uris[i];
+
+    uint64_t key;
+
+    rv = cache_digest_hash(key, logn + logp, StringRef{uri});
+
+    CU_ASSERT(0 == rv);
+
+    query_keys[i] = key;
+  }
+
+  CU_ASSERT(
+      std::binary_search(std::begin(keys), std::end(keys), query_keys[0]));
+  CU_ASSERT(
+      std::binary_search(std::begin(keys), std::end(keys), query_keys[1]));
+  CU_ASSERT(
+      std::binary_search(std::begin(keys), std::end(keys), query_keys[2]));
+}
+
+} // namespace nghttp2

src/cache_digest_test.h

@@ -0,0 +1,38 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2016 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CACHE_DIGEST_TEST_H
+#define CACHE_DIGEST_TEST_H
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif // HAVE_CONFIG_H
+
+namespace nghttp2 {
+
+void test_cache_digest_encode_decode(void);
+
+} // namespace nghttp2
+
+#endif // CACHE_DIGEST_TEST_H

src/h2load.cc

@@ -269,7 +269,7 @@ bool check_stop_client_request_timeout(Client *client, ev_timer *w) {
   auto nreq = client->req_todo - client->req_started;
 
   if (nreq == 0 ||
-      client->streams.size() >= (size_t)config.max_concurrent_streams) {
+      client->streams.size() >= client->session->max_concurrent_streams()) {
     // no more requests to make, stop timer
     ev_timer_stop(client->worker->loop, w);
     return true;
@@ -318,7 +318,8 @@ void client_request_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 } // namespace
 
 Client::Client(uint32_t id, Worker *worker, size_t req_todo)
-    : cstat{},
+    : wb(&worker->mcpool),
+      cstat{},
       worker(worker),
       ssl(nullptr),
       next_addr(config.addrs),
@@ -330,7 +331,8 @@ Client::Client(uint32_t id, Worker *worker, size_t req_todo)
       req_done(0),
       id(id),
       fd(-1),
-      new_connection_requested(false) {
+      new_connection_requested(false),
+      final(false) {
   ev_io_init(&wev, writecb, 0, EV_WRITE);
   ev_io_init(&rev, readcb, 0, EV_READ);
 
@@ -518,6 +520,8 @@ void Client::disconnect() {
     close(fd);
     fd = -1;
   }
+
+  final = false;
 }
 
 int Client::submit_request() {
@@ -860,7 +864,7 @@ int Client::connection_made() {
 
   if (!config.timing_script) {
     auto nreq =
-        std::min(req_todo - req_started, (size_t)config.max_concurrent_streams);
+        std::min(req_todo - req_started, session->max_concurrent_streams());
     for (; nreq > 0; --nreq) {
       if (submit_request() != 0) {
         process_request_failure();
@@ -907,6 +911,10 @@ int Client::on_read(const uint8_t *data, size_t len) {
 }
 
 int Client::on_write() {
+  if (wb.rleft() >= BACKOFF_WRITE_BUFFER_THRES) {
+    return 0;
+  }
+
   if (session->on_write() != 0) {
     return -1;
   }
@@ -940,11 +948,23 @@ int Client::read_clear() {
 }
 
 int Client::write_clear() {
+  std::array<struct iovec, 2> iov;
+
   for (;;) {
-    if (wb.rleft() > 0) {
+    if (on_write() != 0) {
+      return -1;
+    }
+
+    auto iovcnt = wb.riovec(iov.data(), iov.size());
+
+    if (iovcnt == 0) {
+      break;
+    }
+
     ssize_t nwrite;
-      while ((nwrite = write(fd, wb.pos, wb.rleft())) == -1 && errno == EINTR)
+    while ((nwrite = writev(fd, iov.data(), iovcnt)) == -1 && errno == EINTR)
       ;
+
     if (nwrite == -1) {
       if (errno == EAGAIN || errno == EWOULDBLOCK) {
         ev_io_start(worker->loop, &wev);
@@ -952,16 +972,8 @@ int Client::write_clear() {
       }
       return -1;
     }
+
     wb.drain(nwrite);
-      continue;
-    }
-    wb.reset();
-    if (on_write() != 0) {
-      return -1;
-    }
-    if (wb.rleft() == 0) {
-      break;
-    }
   }
 
   ev_io_stop(worker->loop, &wev);
@@ -1054,9 +1066,20 @@ int Client::read_tls() {
 int Client::write_tls() {
   ERR_clear_error();
 
+  struct iovec iov;
+
   for (;;) {
-    if (wb.rleft() > 0) {
-      auto rv = SSL_write(ssl, wb.pos, wb.rleft());
+    if (on_write() != 0) {
+      return -1;
+    }
+
+    auto iovcnt = wb.riovec(&iov, 1);
+
+    if (iovcnt == 0) {
+      break;
+    }
+
+    auto rv = SSL_write(ssl, iov.iov_base, iov.iov_len);
 
     if (rv <= 0) {
       auto err = SSL_get_error(ssl, rv);
@@ -1073,16 +1096,6 @@ int Client::write_tls() {
     }
 
     wb.drain(rv);
-
-      continue;
-    }
-    wb.reset();
-    if (on_write() != 0) {
-      return -1;
-    }
-    if (wb.rleft() == 0) {
-      break;
-    }
   }
 
   ev_io_stop(worker->loop, &wev);
@@ -1667,7 +1680,9 @@ Options:
               Default: )" << NGHTTP2_CLEARTEXT_PROTO_VERSION_ID << R"(
   -d, --data=<PATH>
               Post FILE to  server.  The request method  is changed to
-              POST.
+              POST.   For  http/1.1 connection,  if  -d  is used,  the
+              maximum number of in-flight pipelined requests is set to
+              1.
   -r, --rate=<N>
               Specifies  the  fixed  rate  at  which  connections  are
               created.   The   rate  must   be  a   positive  integer,
@@ -2214,6 +2229,11 @@ int main(int argc, char **argv) {
     }
   }
 
+  std::string content_length_str;
+  if (config.data_fd != -1) {
+    content_length_str = util::utos(config.data_length);
+  }
+
   auto method_it =
       std::find_if(std::begin(shared_nva), std::end(shared_nva),
                    [](const Header &nv) { return nv.name == ":method"; });
@@ -2244,14 +2264,20 @@ int main(int argc, char **argv) {
       h1req += nv.value;
       h1req += "\r\n";
     }
+
+    if (!content_length_str.empty()) {
+      h1req += "Content-Length: ";
+      h1req += content_length_str;
+      h1req += "\r\n";
+    }
     h1req += "\r\n";
 
     config.h1reqs.push_back(std::move(h1req));
 
     // For nghttp2
     std::vector<nghttp2_nv> nva;
-    // 1 for :path
-    nva.reserve(1 + shared_nva.size());
+    // 2 for :path, and possible content-length
+    nva.reserve(2 + shared_nva.size());
 
     nva.push_back(http2::make_nv_ls(":path", req));
 
@@ -2259,12 +2285,18 @@ int main(int argc, char **argv) {
       nva.push_back(http2::make_nv(nv.name, nv.value, false));
     }
 
+    if (!content_length_str.empty()) {
+      nva.push_back(http2::make_nv(StringRef::from_lit("content-length"),
+                                   StringRef{content_length_str}));
+    }
+
     config.nva.push_back(std::move(nva));
 
     // For spdylay
     std::vector<const char *> cva;
-    // 2 for :path and :version, 1 for terminal nullptr
-    cva.reserve(2 * (2 + shared_nva.size()) + 1);
+    // 3 for :path, :version, and possible content-length, 1 for
+    // terminal nullptr
+    cva.reserve(2 * (3 + shared_nva.size()) + 1);
 
     cva.push_back(":path");
     cva.push_back(req.c_str());
@@ -2279,6 +2311,12 @@ int main(int argc, char **argv) {
     }
     cva.push_back(":version");
     cva.push_back("HTTP/1.1");
+
+    if (!content_length_str.empty()) {
+      cva.push_back("content-length");
+      cva.push_back(content_length_str.c_str());
+    }
+
     cva.push_back(nullptr);
 
     config.nv.push_back(std::move(cva));

src/h2load.h

@@ -50,13 +50,15 @@
 #include <openssl/ssl.h>
 
 #include "http2.h"
-#include "buffer.h"
+#include "memchunk.h"
 #include "template.h"
 
 using namespace nghttp2;
 
 namespace h2load {
 
+constexpr auto BACKOFF_WRITE_BUFFER_THRES = 16_k;
+
 class Session;
 struct Worker;
 
@@ -225,6 +227,7 @@ struct Sampling {
 };
 
 struct Worker {
+  MemchunkPool mcpool;
   Stats stats;
   Sampling request_times_smp;
   Sampling client_smp;
@@ -267,6 +270,7 @@ struct Stream {
 };
 
 struct Client {
+  DefaultMemchunks wb;
   std::unordered_map<int32_t, Stream> streams;
   ClientStat cstat;
   std::unique_ptr<Session> session;
@@ -293,11 +297,13 @@ struct Client {
   // The client id per worker
   uint32_t id;
   int fd;
-  Buffer<64_k> wb;
   ev_timer conn_active_watcher;
   ev_timer conn_inactivity_watcher;
   std::string selected_proto;
   bool new_connection_requested;
+  // true if the current connection will be closed, and no more new
+  // request cannot be processed.
+  bool final;
 
   enum { ERR_CONNECT_FAIL = -100 };
 

src/h2load_http1_session.cc

@@ -57,7 +57,7 @@ namespace {
 int htp_msg_begincb(http_parser *htp) {
   auto session = static_cast<Http1Session *>(htp->data);
 
-  if (session->stream_resp_counter_ >= session->stream_req_counter_) {
+  if (session->stream_resp_counter_ > session->stream_req_counter_) {
     return -1;
   }
 
@@ -82,16 +82,21 @@ int htp_msg_completecb(http_parser *htp) {
   auto session = static_cast<Http1Session *>(htp->data);
   auto client = session->get_client();
 
-  auto final = http_should_keep_alive(htp) == 0;
+  client->final = http_should_keep_alive(htp) == 0;
   auto req_stat = client->get_req_stat(session->stream_resp_counter_);
 
   assert(req_stat);
 
-  client->on_stream_close(session->stream_resp_counter_, true, final);
+  auto config = client->worker->config;
+  if (req_stat->data_offset >= config->data_length) {
+    client->on_stream_close(session->stream_resp_counter_, true, client->final);
+  }
 
   session->stream_resp_counter_ += 2;
 
-  if (final) {
+  if (client->final) {
+    session->stream_req_counter_ = session->stream_resp_counter_;
+
     http_parser_pause(htp, 1);
     // Connection is going down.  If we have still request to do,
     // create new connection and keep on doing the job.
@@ -169,14 +174,18 @@ int Http1Session::submit_request() {
   auto req_stat = client_->get_req_stat(stream_req_counter_);
 
   client_->record_request_time(req_stat);
-  client_->wb.write(req.c_str(), req.size());
+  client_->wb.append(req);
 
+  if (config->data_fd == -1 || config->data_length == 0) {
     // increment for next request
     stream_req_counter_ += 2;
 
     return 0;
   }
 
+  return on_write();
+}
+
 int Http1Session::on_read(const uint8_t *data, size_t len) {
   auto nread = http_parser_execute(&htp_, &htp_hooks,
                                    reinterpret_cast<const char *>(data), len);
@@ -206,6 +215,51 @@ int Http1Session::on_write() {
   if (complete_) {
     return -1;
   }
+
+  auto config = client_->worker->config;
+  auto req_stat = client_->get_req_stat(stream_req_counter_);
+  if (!req_stat) {
+    return 0;
+  }
+
+  if (req_stat->data_offset < config->data_length) {
+    auto req_stat = client_->get_req_stat(stream_req_counter_);
+    auto &wb = client_->wb;
+
+    // TODO unfortunately, wb has no interface to use with read(2)
+    // family functions.
+    std::array<uint8_t, 16_k> buf;
+
+    ssize_t nread;
+    while ((nread = pread(config->data_fd, buf.data(), buf.size(),
+                          req_stat->data_offset)) == -1 &&
+           errno == EINTR)
+      ;
+
+    if (nread == -1) {
+      return -1;
+    }
+
+    req_stat->data_offset += nread;
+
+    wb.append(buf.data(), nread);
+
+    if (client_->worker->config->verbose) {
+      std::cout << "[send " << nread << " byte(s)]" << std::endl;
+    }
+
+    if (req_stat->data_offset == config->data_length) {
+      // increment for next request
+      stream_req_counter_ += 2;
+
+      if (stream_resp_counter_ == stream_req_counter_) {
+        // Response has already been received
+        client_->on_stream_close(stream_resp_counter_ - 2, true,
+                                 client_->final);
+      }
+    }
+  }
+
   return 0;
 }
 
@@ -213,4 +267,10 @@ void Http1Session::terminate() { complete_ = true; }
 
 Client *Http1Session::get_client() { return client_; }
 
+size_t Http1Session::max_concurrent_streams() {
+  auto config = client_->worker->config;
+
+  return config->data_fd == -1 ? config->max_concurrent_streams : 1;
+}
+
 } // namespace h2load

src/h2load_http1_session.h

@@ -42,6 +42,7 @@ public:
   virtual int on_read(const uint8_t *data, size_t len);
   virtual int on_write();
   virtual void terminate();
+  virtual size_t max_concurrent_streams();
   Client *get_client();
   int32_t stream_req_counter_;
   int32_t stream_resp_counter_;

src/h2load_http2_session.cc

@@ -169,11 +169,11 @@ ssize_t send_callback(nghttp2_session *session, const uint8_t *data,
   auto client = static_cast<Client *>(user_data);
   auto &wb = client->wb;
 
-  if (wb.wleft() == 0) {
+  if (wb.rleft() >= BACKOFF_WRITE_BUFFER_THRES) {
     return NGHTTP2_ERR_WOULDBLOCK;
   }
 
-  return wb.write(data, length);
+  return wb.append(data, length);
 }
 } // namespace
 
@@ -219,13 +219,10 @@ void Http2Session::on_connect() {
 
   assert(rv == 0);
 
-  auto extra_connection_window =
-      (1 << client_->worker->config->connection_window_bits) - 1 -
-      NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE;
-  if (extra_connection_window != 0) {
-    nghttp2_submit_window_update(session_, NGHTTP2_FLAG_NONE, 0,
-                                 extra_connection_window);
-  }
+  auto connection_window =
+      (1 << client_->worker->config->connection_window_bits) - 1;
+  nghttp2_session_set_local_window_size(session_, NGHTTP2_FLAG_NONE, 0,
+                                        connection_window);
 
   client_->signal_write();
 }
@@ -292,4 +289,8 @@ void Http2Session::terminate() {
   nghttp2_session_terminate_session(session_, NGHTTP2_NO_ERROR);
 }
 
+size_t Http2Session::max_concurrent_streams() {
+  return (size_t)client_->worker->config->max_concurrent_streams;
+}
+
 } // namespace h2load

src/h2load_http2_session.h

@@ -42,6 +42,7 @@ public:
   virtual int on_read(const uint8_t *data, size_t len);
   virtual int on_write();
   virtual void terminate();
+  virtual size_t max_concurrent_streams();
 
 private:
   Client *client_;

src/h2load_session.h

@@ -50,6 +50,8 @@ public:
   virtual int on_write() = 0;
   // Called when the underlying session must be terminated.
   virtual void terminate() = 0;
+  // Return the maximum concurrency per connection
+  virtual size_t max_concurrent_streams() = 0;
 };
 
 } // namespace h2load

src/h2load_spdy_session.cc

@@ -112,11 +112,11 @@ ssize_t send_callback(spdylay_session *session, const uint8_t *data,
   auto client = static_cast<Client *>(user_data);
   auto &wb = client->wb;
 
-  if (wb.wleft() == 0) {
+  if (wb.rleft() >= BACKOFF_WRITE_BUFFER_THRES) {
     return SPDYLAY_ERR_DEFERRED;
   }
 
-  return wb.write(data, length);
+  return wb.append(data, length);
 }
 } // namespace
 
@@ -282,4 +282,8 @@ void SpdySession::handle_window_update(int32_t stream_id, size_t recvlen) {
   }
 }
 
+size_t SpdySession::max_concurrent_streams() {
+  return (size_t)client_->worker->config->max_concurrent_streams;
+}
+
 } // namespace h2load

src/h2load_spdy_session.h

@@ -44,6 +44,7 @@ public:
   virtual int on_read(const uint8_t *data, size_t len);
   virtual int on_write();
   virtual void terminate();
+  virtual size_t max_concurrent_streams();
   void handle_window_update(int32_t stream_id, size_t recvlen);
 
 private:

src/nghttp.cc

@@ -59,6 +59,7 @@
 #include "base64.h"
 #include "ssl.h"
 #include "template.h"
+#include "cache_digest.h"
 
 #ifndef O_BINARY
 #define O_BINARY (0)
@@ -98,6 +99,7 @@ Config::Config()
       padding(0),
       max_concurrent_streams(100),
       peer_max_concurrent_streams(100),
+      cache_digest_bits(7),
       weight(NGHTTP2_DEFAULT_WEIGHT),
       multiply(1),
       timeout(0.),
@@ -544,7 +546,8 @@ HttpClient::HttpClient(const nghttp2_session_callbacks *callbacks,
       state(ClientState::IDLE),
       upgrade_response_status_code(0),
       fd(-1),
-      upgrade_response_complete(false) {
+      upgrade_response_complete(false),
+      cache_digest_sent(false) {
   ev_io_init(&wev, writecb, 0, EV_WRITE);
   ev_io_init(&rev, readcb, 0, EV_READ);
 
@@ -1151,9 +1154,9 @@ int HttpClient::connection_made() {
   ev_timer_again(loop, &settings_timer);
 
   if (config.connection_window_bits != -1) {
-    int32_t wininc = (1 << config.connection_window_bits) - 1 -
-                     NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE;
-    rv = nghttp2_submit_window_update(session, NGHTTP2_FLAG_NONE, 0, wininc);
+    int32_t window_size = (1 << config.connection_window_bits) - 1;
+    rv = nghttp2_session_set_local_window_size(session, NGHTTP2_FLAG_NONE, 0,
+                                               window_size);
     if (rv != 0) {
       return -1;
     }
@@ -1997,6 +2000,8 @@ int before_frame_send_callback(nghttp2_session *session,
 namespace {
 int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame,
                            void *user_data) {
+  int rv;
+
   if (config.verbose) {
     verbose_on_frame_send_callback(session, frame, user_data);
   }
@@ -2017,6 +2022,19 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame,
     req->continue_timer->start();
   }
 
+  auto client = get_client(user_data);
+
+  if (!client->cache_digest_sent && !config.cache_digest_uris.empty()) {
+    client->cache_digest_sent = true;
+
+    rv = nghttp2_submit_extension(session, NGHTTP2_DRAFT_CACHE_DIGEST,
+                                  NGHTTP2_FLAG_NONE, frame->hd.stream_id, NULL);
+
+    if (nghttp2_is_fatal(rv)) {
+      return NGHTTP2_ERR_CALLBACK_FAILURE;
+    }
+  }
+
   return 0;
 }
 } // namespace
@@ -2363,6 +2381,35 @@ ssize_t file_read_callback(nghttp2_session *session, int32_t stream_id,
 }
 } // namespace
 
+namespace {
+ssize_t pack_cache_digest_frame(nghttp2_session *session, uint8_t *buf,
+                                size_t len, const nghttp2_frame *frame,
+                                void *user_data) {
+  ssize_t encodedlen;
+
+  encodedlen = cache_digest_encode(buf, len, config.cache_digest_uris,
+                                   config.cache_digest_bits);
+
+  if (encodedlen == -1) {
+    return NGHTTP2_ERR_CANCEL;
+  }
+
+  return encodedlen;
+}
+} // namespace
+
+namespace {
+ssize_t pack_extension_callback(nghttp2_session *session, uint8_t *buf,
+                                size_t len, const nghttp2_frame *frame,
+                                void *user_data) {
+  if (frame->hd.type != NGHTTP2_DRAFT_CACHE_DIGEST) {
+    return NGHTTP2_ERR_CANCEL;
+  }
+
+  return pack_cache_digest_frame(session, buf, len, frame, user_data);
+}
+} // namespace
+
 namespace {
 int run(char **uris, int n) {
   nghttp2_session_callbacks *callbacks;
@@ -2407,6 +2454,9 @@ int run(char **uris, int n) {
         callbacks, select_padding_callback);
   }
 
+  nghttp2_session_callbacks_set_pack_extension_callback(
+      callbacks, pack_extension_callback);
+
   std::string prev_scheme;
   std::string prev_host;
   uint16_t prev_port = 0;
@@ -2632,6 +2682,13 @@ Options:
               (up to  a short  timeout)  until the server sends  a 100
               Continue interim response. This option is ignored unless
               combined with the -d option.
+  -C, --cache-digest-uri=<URI>
+              Add  <URI> to  cache digest.   Use this  option multiple
+              times to add more than 1 URI to cache digest.
+  --cache-digest-bits=<N>
+              Set the number  of bits to specify the  probability of a
+              false positive that is acceptable, expressed as "1/<N>".
+              Default: 7
   --version   Display version information and exit.
   -h, --help  Display this help and exit.
 
@@ -2672,6 +2729,7 @@ int main(int argc, char **argv) {
         {"header-table-size", required_argument, nullptr, 'c'},
         {"padding", required_argument, nullptr, 'b'},
         {"har", required_argument, nullptr, 'r'},
+        {"cache-digest-uri", required_argument, nullptr, 'C'},
         {"cert", required_argument, &flag, 1},
         {"key", required_argument, &flag, 2},
         {"color", no_argument, &flag, 3},
@@ -2684,14 +2742,18 @@ int main(int argc, char **argv) {
         {"no-push", no_argument, &flag, 11},
         {"max-concurrent-streams", required_argument, &flag, 12},
         {"expect-continue", no_argument, &flag, 13},
+        {"cache-digest-bits", required_argument, &flag, 14},
         {nullptr, 0, nullptr, 0}};
     int option_index = 0;
-    int c = getopt_long(argc, argv, "M:Oab:c:d:gm:np:r:hH:vst:uw:W:",
+    int c = getopt_long(argc, argv, "C:M:Oab:c:d:gm:np:r:hH:vst:uw:W:",
                         long_options, &option_index);
     if (c == -1) {
       break;
     }
     switch (c) {
+    case 'C':
+      config.cache_digest_uris.push_back(optarg);
+      break;
     case 'M':
       // peer-max-concurrent-streams option
       config.peer_max_concurrent_streams = strtoul(optarg, nullptr, 10);
@@ -2885,6 +2947,18 @@ int main(int argc, char **argv) {
         // expect-continue option
         config.expect_continue = true;
         break;
+      case 14: {
+        // cache-digest-bits
+        auto n = strtoul(optarg, nullptr, 10);
+        if (n <= 0 || n >= 32) {
+          std::cerr
+              << "--cache-digest-bits: specify in the range [1, 31], inclusive"
+              << std::endl;
+          exit(EXIT_FAILURE);
+        }
+        config.cache_digest_bits = n;
+        break;
+      }
       }
       break;
     default:

src/nghttp.h

@@ -64,6 +64,7 @@ struct Config {
 
   Headers headers;
   Headers trailer;
+  std::vector<std::string> cache_digest_uris;
   std::string certfile;
   std::string keyfile;
   std::string datafile;
@@ -74,6 +75,7 @@ struct Config {
   size_t padding;
   size_t max_concurrent_streams;
   ssize_t peer_max_concurrent_streams;
+  uint32_t cache_digest_bits;
   int32_t weight;
   int multiply;
   // milliseconds
@@ -283,6 +285,8 @@ struct HttpClient {
   // true if the response message of HTTP Upgrade request is fully
   // received. It is not relevant the upgrade succeeds, or not.
   bool upgrade_response_complete;
+  // true if cache digest was sent or there is no need to send it.
+  bool cache_digest_sent;
   // SETTINGS payload sent as token68 in HTTP Upgrade
   std::array<uint8_t, 128> settings_payload;
 

src/shrpx-unittest.cc

@@ -44,6 +44,8 @@
 #include "base64_test.h"
 #include "shrpx_config.h"
 #include "ssl.h"
+#include "shrpx_router_test.h"
+#include "cache_digest_test.h"
 
 static int init_suite1(void) { return 0; }
 
@@ -71,8 +73,8 @@ int main(int argc, char *argv[]) {
   // add the tests to the suite
   if (!CU_add_test(pSuite, "ssl_create_lookup_tree",
                    shrpx::test_shrpx_ssl_create_lookup_tree) ||
-      !CU_add_test(pSuite, "ssl_cert_lookup_tree_add_cert_from_file",
-                   shrpx::test_shrpx_ssl_cert_lookup_tree_add_cert_from_file) ||
+      !CU_add_test(pSuite, "ssl_cert_lookup_tree_add_cert_from_x509",
+                   shrpx::test_shrpx_ssl_cert_lookup_tree_add_cert_from_x509) ||
       !CU_add_test(pSuite, "ssl_tls_hostname_match",
                    shrpx::test_shrpx_ssl_tls_hostname_match) ||
       !CU_add_test(pSuite, "http2_add_header", shrpx::test_http2_add_header) ||
@@ -125,6 +127,9 @@ int main(int argc, char *argv[]) {
                    shrpx::test_shrpx_http_create_forwarded) ||
       !CU_add_test(pSuite, "http_create_via_header_value",
                    shrpx::test_shrpx_http_create_via_header_value) ||
+      !CU_add_test(pSuite, "router_match", shrpx::test_shrpx_router_match) ||
+      !CU_add_test(pSuite, "router_match_prefix",
+                   shrpx::test_shrpx_router_match_prefix) ||
       !CU_add_test(pSuite, "util_streq", shrpx::test_util_streq) ||
       !CU_add_test(pSuite, "util_strieq", shrpx::test_util_strieq) ||
       !CU_add_test(pSuite, "util_inp_strlower",
@@ -194,7 +199,9 @@ int main(int argc, char *argv[]) {
       !CU_add_test(pSuite, "template_string_ref",
                    nghttp2::test_template_string_ref) ||
       !CU_add_test(pSuite, "base64_encode", nghttp2::test_base64_encode) ||
-      !CU_add_test(pSuite, "base64_decode", nghttp2::test_base64_decode)) {
+      !CU_add_test(pSuite, "base64_decode", nghttp2::test_base64_decode) ||
+      !CU_add_test(pSuite, "cache_digest_encode_decode",
+                   nghttp2::test_cache_digest_encode_decode)) {
     CU_cleanup_registry();
     return CU_get_error();
   }

src/shrpx.cc

@@ -146,52 +146,6 @@ struct SignalServer {
   pid_t worker_process_pid;
 };
 
-namespace {
-int resolve_hostname(Address *addr, const char *hostname, uint16_t port,
-                     int family) {
-  int rv;
-
-  auto service = util::utos(port);
-
-  addrinfo hints{};
-  hints.ai_family = family;
-  hints.ai_socktype = SOCK_STREAM;
-#ifdef AI_ADDRCONFIG
-  hints.ai_flags |= AI_ADDRCONFIG;
-#endif // AI_ADDRCONFIG
-  addrinfo *res;
-
-  rv = getaddrinfo(hostname, service.c_str(), &hints, &res);
-  if (rv != 0) {
-    LOG(FATAL) << "Unable to resolve address for " << hostname << ": "
-               << gai_strerror(rv);
-    return -1;
-  }
-
-  auto res_d = defer(freeaddrinfo, res);
-
-  char host[NI_MAXHOST];
-  rv = getnameinfo(res->ai_addr, res->ai_addrlen, host, sizeof(host), nullptr,
-                   0, NI_NUMERICHOST);
-  if (rv != 0) {
-    LOG(FATAL) << "Address resolution for " << hostname
-               << " failed: " << gai_strerror(rv);
-
-    return -1;
-  }
-
-  if (LOG_ENABLED(INFO)) {
-    LOG(INFO) << "Address resolution for " << hostname
-              << " succeeded: " << host;
-  }
-
-  memcpy(&addr->su, res->ai_addr, res->ai_addrlen);
-  addr->len = res->ai_addrlen;
-
-  return 0;
-}
-} // namespace
-
 namespace {
 int chown_to_running_user(const char *path) {
   return chown(path, get_config()->uid, get_config()->gid);
@@ -1075,11 +1029,6 @@ constexpr auto DEFAULT_ACCESSLOG_FORMAT = StringRef::from_lit(
     R"("$http_referer" "$http_user_agent")");
 } // namespace
 
-namespace {
-constexpr char DEFAULT_DOWNSTREAM_HOST[] = "127.0.0.1";
-constexpr int16_t DEFAULT_DOWNSTREAM_PORT = 80;
-} // namespace;
-
 namespace {
 void fill_default_config() {
   *mod_config() = {};
@@ -1157,6 +1106,11 @@ void fill_default_config() {
     nghttp2_option_new(&upstreamconf.option);
     nghttp2_option_set_no_auto_window_update(upstreamconf.option, 1);
     nghttp2_option_set_no_recv_client_magic(upstreamconf.option, 1);
+
+    // For API endpoint, we enable automatic window update.  This is
+    // because we are a sink.
+    nghttp2_option_new(&upstreamconf.alt_mode_option);
+    nghttp2_option_set_no_recv_client_magic(upstreamconf.alt_mode_option, 1);
   }
 
   {
@@ -1213,7 +1167,8 @@ void fill_default_config() {
   }
 
   {
-    auto &downstreamconf = connconf.downstream;
+    connconf.downstream = std::make_shared<DownstreamConfig>();
+    auto &downstreamconf = *connconf.downstream;
     {
       auto &timeoutconf = downstreamconf.timeout;
       // Read/Write timeouts for downstream connection
@@ -1221,6 +1176,7 @@ void fill_default_config() {
       timeoutconf.write = 30_s;
       // Timeout for pooled (idle) connections
       timeoutconf.idle_read = 2_s;
+      timeoutconf.max_backoff = 120_s;
     }
 
     downstreamconf.connections_per_host = 8;
@@ -1228,6 +1184,9 @@ void fill_default_config() {
     downstreamconf.response_buffer_size = 128_k;
     downstreamconf.family = AF_UNSPEC;
   }
+
+  auto &apiconf = mod_config()->api;
+  apiconf.max_request_body = 16_k;
 }
 
 } // namespace
@@ -1326,12 +1285,13 @@ Connections:
               Several parameters <PARAM> are accepted after <PATTERN>.
               The  parameters are  delimited  by  ";".  The  available
               parameters       are:      "proto=<PROTO>",       "tls",
-              "sni=<SNI_HOST>",   "fall=<N>",  and   "rise=<N>".   The
-              parameter consists  of keyword, and  optionally followed
-              by "=" and value.  For example, the parameter "proto=h2"
-              consists  of the  keyword "proto"  and value  "h2".  The
-              parameter "tls"  consists of  the keyword  "tls" without
-              value.  Each parameter is described as follows.
+              "sni=<SNI_HOST>",     "fall=<N>",    "rise=<N>",     and
+              "affinity=<METHOD>".  The parameter consists of keyword,
+              and optionally followed by  "=" and value.  For example,
+              the parameter "proto=h2" consists of the keyword "proto"
+              and  value "h2".   The parameter  "tls" consists  of the
+              keyword   "tls"  without   value.   Each   parameter  is
+              described as follows.
 
               The backend application protocol  can be specified using
               optional  "proto"   parameter,  and   in  the   form  of
@@ -1366,13 +1326,27 @@ Connections:
               backend  is permanently  offline, once  it goes  in that
               state, and this is the default behaviour.
 
+              The     session     affinity    is     enabled     using
+              "affinity=<METHOD>"  parameter.   If  "ip" is  given  in
+              <METHOD>, client  IP based session affinity  is enabled.
+              If  "none" is  given  in <METHOD>,  session affinity  is
+              disabled, and this is the default.  The session affinity
+              is enabled per  <PATTERN>.  If at least  one backend has
+              "affinity" parameter,  and its  <METHOD> is  not "none",
+              session  affinity is  enabled  for  all backend  servers
+              sharing  the  same  <PATTERN>.   It is  advised  to  set
+              "affinity"  parameter  to   all  backend  explicitly  if
+              session affinity  is desired.  The session  affinity may
+              break if one of the backend gets unreachable, or backend
+              settings are reloaded or replaced by API.
+
               Since ";" and ":" are  used as delimiter, <PATTERN> must
               not  contain these  characters.  Since  ";" has  special
               meaning in shell, the option value must be quoted.
 
               Default: )" << DEFAULT_DOWNSTREAM_HOST << ","
       << DEFAULT_DOWNSTREAM_PORT << R"(
-  -f, --frontend=(<HOST>,<PORT>|unix:<PATH>)[;no-tls]
+  -f, --frontend=(<HOST>,<PORT>|unix:<PATH>)[[;PARAM]...]
               Set  frontend  host and  port.   If  <HOST> is  '*',  it
               assumes  all addresses  including  both  IPv4 and  IPv6.
               UNIX domain  socket can  be specified by  prefixing path
@@ -1380,9 +1354,25 @@ Connections:
               This  option can  be used  multiple times  to listen  to
               multiple addresses.
 
+              This option  can take  0 or  more parameters,  which are
+              described  below.   Note   that  "api"  and  "healthmon"
+              parameters are mutually exclusive.
+
               Optionally, TLS  can be disabled by  specifying "no-tls"
               parameter.  TLS is enabled by default.
 
+              To  make this  frontend as  API endpoint,  specify "api"
+              parameter.   This   is  disabled  by  default.    It  is
+              important  to  limit the  access  to  the API  frontend.
+              Otherwise, someone  may change  the backend  server, and
+              break your services,  or expose confidential information
+              to the outside the world.
+
+              To  make  this  frontend  as  health  monitor  endpoint,
+              specify  "healthmon"  parameter.   This is  disabled  by
+              default.  Any  requests which come through  this address
+              are replied with 200 HTTP status, without no body.
+
               Default: *,3000
   --backlog=<N>
               Set listen backlog size.
@@ -1469,7 +1459,7 @@ Performance:
               HTTP/2).   To  limit  the   number  of  connections  per
               frontend        for       default        mode,       use
               --backend-connections-per-frontend.
-              Default: )" << get_config()->conn.downstream.connections_per_host
+              Default: )" << get_config()->conn.downstream->connections_per_host
       << R"(
   --backend-connections-per-frontend=<N>
               Set  maximum number  of  backend concurrent  connections
@@ -1479,7 +1469,7 @@ Performance:
               with          --http2-proxy         option,          use
               --backend-connections-per-host.
               Default: )"
-      << get_config()->conn.downstream.connections_per_frontend << R"(
+      << get_config()->conn.downstream->connections_per_frontend << R"(
   --rlimit-nofile=<N>
               Set maximum number of open files (RLIMIT_NOFILE) to <N>.
               If 0 is given, nghttpx does not set the limit.
@@ -1487,12 +1477,12 @@ Performance:
   --backend-request-buffer=<SIZE>
               Set buffer size used to store backend request.
               Default: )"
-      << util::utos_unit(get_config()->conn.downstream.request_buffer_size)
+      << util::utos_unit(get_config()->conn.downstream->request_buffer_size)
       << R"(
   --backend-response-buffer=<SIZE>
               Set buffer size used to store backend response.
               Default: )"
-      << util::utos_unit(get_config()->conn.downstream.response_buffer_size)
+      << util::utos_unit(get_config()->conn.downstream->response_buffer_size)
       << R"(
   --fastopen=<N>
               Enables  "TCP Fast  Open" for  the listening  socket and
@@ -1532,15 +1522,15 @@ Timeout:
   --backend-read-timeout=<DURATION>
               Specify read timeout for backend connection.
               Default: )"
-      << util::duration_str(get_config()->conn.downstream.timeout.read) << R"(
+      << util::duration_str(get_config()->conn.downstream->timeout.read) << R"(
   --backend-write-timeout=<DURATION>
               Specify write timeout for backend connection.
               Default: )"
-      << util::duration_str(get_config()->conn.downstream.timeout.write) << R"(
+      << util::duration_str(get_config()->conn.downstream->timeout.write) << R"(
   --backend-keep-alive-timeout=<DURATION>
               Specify keep-alive timeout for backend connection.
               Default: )"
-      << util::duration_str(get_config()->conn.downstream.timeout.idle_read)
+      << util::duration_str(get_config()->conn.downstream->timeout.idle_read)
       << R"(
   --listener-disable-timeout=<DURATION>
               After accepting  connection failed,  connection listener
@@ -1560,6 +1550,18 @@ Timeout:
               Default: )"
       << util::duration_str(get_config()->http2.downstream.timeout.settings)
       << R"(
+  --backend-max-backoff=<DURATION>
+              Specify  maximum backoff  interval.  This  is used  when
+              doing health  check against offline backend  (see "fail"
+              parameter  in --backend  option).   It is  also used  to
+              limit  the  maximum   interval  to  temporarily  disable
+              backend  when nghttpx  failed to  connect to  it.  These
+              intervals are calculated  using exponential backoff, and
+              consecutive failed attempts increase the interval.  This
+              option caps its maximum value.
+              Default: )"
+      << util::duration_str(get_config()->conn.downstream->timeout.max_backoff)
+      << R"(
 
 SSL/TLS:
   --ciphers=<SUITE>
@@ -1959,6 +1961,12 @@ HTTP:
               HTTP  status  code.  If  error  status  code comes  from
               backend server, the custom error pages are not used.
 
+API:
+  --api-max-request-body=<SIZE>
+              Set the maximum size of request body for API request.
+              Default: )" << util::utos_unit(get_config()->api.max_request_body)
+      << R"(
+
 Debug:
   --frontend-http2-dump-request-header=<PATH>
               Dumps request headers received by HTTP/2 frontend to the
@@ -2041,7 +2049,7 @@ void process_options(int argc, char **argv,
     std::set<StringRef> include_set;
 
     for (auto &p : cmdcfgs) {
-      if (parse_config(p.first, p.second, include_set) == -1) {
+      if (parse_config(mod_config(), p.first, p.second, include_set) == -1) {
         LOG(FATAL) << "Failed to parse command-line argument.";
         exit(EXIT_FAILURE);
       }
@@ -2146,7 +2154,6 @@ void process_options(int argc, char **argv,
 
   auto &listenerconf = mod_config()->conn.listener;
   auto &upstreamconf = mod_config()->conn.upstream;
-  auto &downstreamconf = mod_config()->conn.downstream;
 
   if (listenerconf.addrs.empty()) {
     UpstreamAddr addr{};
@@ -2180,140 +2187,11 @@ void process_options(int argc, char **argv,
     }
   }
 
-  auto &addr_groups = downstreamconf.addr_groups;
-
-  if (addr_groups.empty()) {
-    DownstreamAddrConfig addr{};
-    addr.host = ImmutableString::from_lit(DEFAULT_DOWNSTREAM_HOST);
-    addr.port = DEFAULT_DOWNSTREAM_PORT;
-    addr.proto = PROTO_HTTP1;
-
-    DownstreamAddrGroupConfig g(StringRef::from_lit("/"));
-    g.addrs.push_back(std::move(addr));
-    mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size());
-    addr_groups.push_back(std::move(g));
-  } else if (get_config()->http2_proxy) {
-    // We don't support host mapping in these cases.  Move all
-    // non-catch-all patterns to catch-all pattern.
-    DownstreamAddrGroupConfig catch_all(StringRef::from_lit("/"));
-    for (auto &g : addr_groups) {
-      std::move(std::begin(g.addrs), std::end(g.addrs),
-                std::back_inserter(catch_all.addrs));
-    }
-    std::vector<DownstreamAddrGroupConfig>().swap(addr_groups);
-    std::vector<WildcardPattern>().swap(mod_config()->wildcard_patterns);
-    // maybe not necessary?
-    mod_config()->router = Router();
-    mod_config()->router.add_route(StringRef{catch_all.pattern},
-                                   addr_groups.size());
-    addr_groups.push_back(std::move(catch_all));
-  } else {
-    auto &wildcard_patterns = mod_config()->wildcard_patterns;
-    std::sort(std::begin(wildcard_patterns), std::end(wildcard_patterns),
-              [](const WildcardPattern &lhs, const WildcardPattern &rhs) {
-                return std::lexicographical_compare(
-                    rhs.host.rbegin(), rhs.host.rend(), lhs.host.rbegin(),
-                    lhs.host.rend());
-              });
-    if (LOG_ENABLED(INFO)) {
-      LOG(INFO) << "Reverse sorted wildcard hosts (compared from tail to head, "
-                   "and sorted in reverse order):";
-      for (auto &wp : mod_config()->wildcard_patterns) {
-        LOG(INFO) << wp.host;
-      }
-    }
-  }
-
-  // backward compatibility: override all SNI fields with the option
-  // value --backend-tls-sni-field
-  if (!tlsconf.backend_sni_name.empty()) {
-    auto &sni = tlsconf.backend_sni_name;
-    for (auto &addr_group : addr_groups) {
-      for (auto &addr : addr_group.addrs) {
-        addr.sni = sni;
-      }
-    }
-  }
-
-  if (LOG_ENABLED(INFO)) {
-    LOG(INFO) << "Resolving backend address";
-  }
-
-  ssize_t catch_all_group = -1;
-  for (size_t i = 0; i < addr_groups.size(); ++i) {
-    auto &g = addr_groups[i];
-    if (g.pattern == StringRef::from_lit("/")) {
-      catch_all_group = i;
-    }
-    if (LOG_ENABLED(INFO)) {
-      LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern
-                << "'";
-      for (auto &addr : g.addrs) {
-        LOG(INFO) << "group " << i << " -> " << addr.host.c_str()
-                  << (addr.host_unix ? "" : ":" + util::utos(addr.port))
-                  << ", proto=" << strproto(addr.proto)
-                  << (addr.tls ? ", tls" : "");
-      }
-    }
-  }
-
-  if (catch_all_group == -1) {
-    LOG(FATAL) << "backend: No catch-all backend address is configured";
+  if (configure_downstream_group(mod_config(), get_config()->http2_proxy, false,
+                                 tlsconf) != 0) {
     exit(EXIT_FAILURE);
   }
 
-  downstreamconf.addr_group_catch_all = catch_all_group;
-
-  if (LOG_ENABLED(INFO)) {
-    LOG(INFO) << "Catch-all pattern is group " << catch_all_group;
-  }
-
-  for (auto &g : addr_groups) {
-    for (auto &addr : g.addrs) {
-
-      if (addr.host_unix) {
-        // for AF_UNIX socket, we use "localhost" as host for backend
-        // hostport.  This is used as Host header field to backend and
-        // not going to be passed to any syscalls.
-        addr.hostport = "localhost";
-
-        auto path = addr.host.c_str();
-        auto pathlen = addr.host.size();
-
-        if (pathlen + 1 > sizeof(addr.addr.su.un.sun_path)) {
-          LOG(FATAL) << "UNIX domain socket path " << path << " is too long > "
-                     << sizeof(addr.addr.su.un.sun_path);
-          exit(EXIT_FAILURE);
-        }
-
-        if (LOG_ENABLED(INFO)) {
-          LOG(INFO) << "Use UNIX domain socket path " << path
-                    << " for backend connection";
-        }
-
-        addr.addr.su.un.sun_family = AF_UNIX;
-        // copy path including terminal NULL
-        std::copy_n(path, pathlen + 1, addr.addr.su.un.sun_path);
-        addr.addr.len = sizeof(addr.addr.su.un);
-
-        continue;
-      }
-
-      addr.hostport = ImmutableString(
-          util::make_http_hostport(StringRef(addr.host), addr.port));
-
-      auto hostport = util::make_hostport(StringRef{addr.host}, addr.port);
-
-      if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port,
-                           downstreamconf.family) == -1) {
-        LOG(FATAL) << "Resolving backend address failed: " << hostport;
-        exit(EXIT_FAILURE);
-      }
-      LOG(NOTICE) << "Resolved backend address: " << hostport << " -> "
-                  << util::to_numeric_addr(&addr.addr);
-    }
-  }
-
   auto &proxy = mod_config()->downstream_http_proxy;
   if (!proxy.host.empty()) {
     auto hostport = util::make_hostport(StringRef{proxy.host}, proxy.port);
@@ -2622,6 +2500,8 @@ int main(int argc, char **argv) {
          &flag, 124},
         {SHRPX_OPT_BACKEND_HTTP2_SETTINGS_TIMEOUT.c_str(), required_argument,
          &flag, 125},
+        {SHRPX_OPT_API_MAX_REQUEST_BODY.c_str(), required_argument, &flag, 126},
+        {SHRPX_OPT_BACKEND_MAX_BACKOFF.c_str(), required_argument, &flag, 127},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -3211,6 +3091,14 @@ int main(int argc, char **argv) {
         cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_HTTP2_SETTINGS_TIMEOUT,
                              StringRef{optarg});
         break;
+      case 126:
+        // --api-max-request-body
+        cmdcfgs.emplace_back(SHRPX_OPT_API_MAX_REQUEST_BODY, StringRef{optarg});
+        break;
+      case 127:
+        // --backend-max-backoff
+        cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_MAX_BACKOFF, StringRef{optarg});
+        break;
       default:
         break;
       }

src/shrpx_accept_handler.cc

@@ -58,13 +58,12 @@ AcceptHandler::~AcceptHandler() {
 }
 
 void AcceptHandler::accept_connection() {
-  for (;;) {
   sockaddr_union sockaddr;
   socklen_t addrlen = sizeof(sockaddr);
 
 #ifdef HAVE_ACCEPT4
-    auto cfd = accept4(faddr_->fd, &sockaddr.sa, &addrlen,
-                       SOCK_NONBLOCK | SOCK_CLOEXEC);
+  auto cfd =
+      accept4(faddr_->fd, &sockaddr.sa, &addrlen, SOCK_NONBLOCK | SOCK_CLOEXEC);
 #else  // !HAVE_ACCEPT4
   auto cfd = accept(faddr_->fd, &sockaddr.sa, &addrlen);
 #endif // !HAVE_ACCEPT4
@@ -82,16 +81,16 @@ void AcceptHandler::accept_connection() {
     case EHOSTUNREACH:
     case EOPNOTSUPP:
     case ENETUNREACH:
-        continue;
+      return;
     case EMFILE:
     case ENFILE:
       LOG(WARN) << "acceptor: running out file descriptor; disable acceptor "
                    "temporarily";
       conn_hnr_->sleep_acceptor(get_config()->conn.listener.timeout.sleep);
-        break;
+      return;
+    default:
+      return;
     }
-
-      break;
   }
 
 #ifndef HAVE_ACCEPT4
@@ -103,7 +102,6 @@ void AcceptHandler::accept_connection() {
 
   conn_hnr_->handle_connection(cfd, &sockaddr.sa, addrlen, faddr_);
 }
-}
 
 void AcceptHandler::enable() { ev_io_start(conn_hnr_->get_loop(), &wev_); }
 

src/shrpx_api_downstream_connection.cc

@@ -0,0 +1,311 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2016 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "shrpx_api_downstream_connection.h"
+
+#include "shrpx_client_handler.h"
+#include "shrpx_upstream.h"
+#include "shrpx_downstream.h"
+#include "shrpx_worker.h"
+#include "shrpx_connection_handler.h"
+
+namespace shrpx {
+
+APIDownstreamConnection::APIDownstreamConnection(Worker *worker)
+    : worker_(worker), abandoned_(false) {}
+
+APIDownstreamConnection::~APIDownstreamConnection() {}
+
+int APIDownstreamConnection::attach_downstream(Downstream *downstream) {
+  if (LOG_ENABLED(INFO)) {
+    DCLOG(INFO, this) << "Attaching to DOWNSTREAM:" << downstream;
+  }
+
+  downstream_ = downstream;
+
+  return 0;
+}
+
+void APIDownstreamConnection::detach_downstream(Downstream *downstream) {
+  if (LOG_ENABLED(INFO)) {
+    DCLOG(INFO, this) << "Detaching from DOWNSTREAM:" << downstream;
+  }
+  downstream_ = nullptr;
+}
+
+// API status, which is independent from HTTP status code.  But
+// generally, 2xx code for API_SUCCESS, and otherwise API_FAILURE.
+enum {
+  API_SUCCESS,
+  API_FAILURE,
+};
+
+int APIDownstreamConnection::send_reply(unsigned int http_status,
+                                        int api_status) {
+  abandoned_ = true;
+
+  auto upstream = downstream_->get_upstream();
+
+  auto &resp = downstream_->response();
+
+  resp.http_status = http_status;
+
+  auto &balloc = downstream_->get_block_allocator();
+
+  StringRef api_status_str;
+
+  switch (api_status) {
+  case API_SUCCESS:
+    api_status_str = StringRef::from_lit("Success");
+    break;
+  case API_FAILURE:
+    api_status_str = StringRef::from_lit("Failure");
+    break;
+  default:
+    assert(0);
+  }
+
+  constexpr auto M1 = StringRef::from_lit("{\"status\":\"");
+  constexpr auto M2 = StringRef::from_lit("\",\"code\":");
+  constexpr auto M3 = StringRef::from_lit("}");
+
+  // 3 is the number of digits in http_status, assuming it is 3 digits
+  // number.
+  auto buflen = M1.size() + M2.size() + M3.size() + api_status_str.size() + 3;
+
+  auto buf = make_byte_ref(balloc, buflen);
+  auto p = buf.base;
+
+  p = std::copy(std::begin(M1), std::end(M1), p);
+  p = std::copy(std::begin(api_status_str), std::end(api_status_str), p);
+  p = std::copy(std::begin(M2), std::end(M2), p);
+  p = util::utos(p, http_status);
+  p = std::copy(std::begin(M3), std::end(M3), p);
+
+  buf.len = p - buf.base;
+
+  auto content_length = util::make_string_ref_uint(balloc, buf.len);
+
+  resp.fs.add_header_token(StringRef::from_lit("content-length"),
+                           content_length, false, http2::HD_CONTENT_LENGTH);
+
+  switch (http_status) {
+  case 400:
+  case 405:
+  case 413:
+    resp.fs.add_header_token(StringRef::from_lit("connection"),
+                             StringRef::from_lit("close"), false,
+                             http2::HD_CONNECTION);
+    break;
+  }
+
+  if (upstream->send_reply(downstream_, buf.base, buf.len) != 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+int APIDownstreamConnection::push_request_headers() {
+  auto &req = downstream_->request();
+  auto &resp = downstream_->response();
+
+  auto path =
+      StringRef{std::begin(req.path),
+                std::find(std::begin(req.path), std::end(req.path), '?')};
+
+  if (path != StringRef::from_lit("/api/v1beta1/backendconfig")) {
+    send_reply(404, API_FAILURE);
+
+    return 0;
+  }
+
+  if (req.method != HTTP_POST && req.method != HTTP_PUT) {
+    resp.fs.add_header_token(StringRef::from_lit("allow"),
+                             StringRef::from_lit("POST, PUT"), false, -1);
+    send_reply(405, API_FAILURE);
+
+    return 0;
+  }
+
+  // This works with req.fs.content_length == -1
+  if (req.fs.content_length >
+      static_cast<int64_t>(get_config()->api.max_request_body)) {
+    send_reply(413, API_FAILURE);
+
+    return 0;
+  }
+
+  return 0;
+}
+
+int APIDownstreamConnection::push_upload_data_chunk(const uint8_t *data,
+                                                    size_t datalen) {
+  if (abandoned_) {
+    return 0;
+  }
+
+  auto output = downstream_->get_request_buf();
+
+  auto &apiconf = get_config()->api;
+
+  if (output->rleft() + datalen > apiconf.max_request_body) {
+    send_reply(413, API_FAILURE);
+
+    return 0;
+  }
+
+  output->append(data, datalen);
+
+  // We don't have to call Upstream::resume_read() here, because
+  // request buffer is effectively unlimited.  Actually, we cannot
+  // call it here since it could recursively call this function again.
+
+  return 0;
+}
+
+int APIDownstreamConnection::end_upload_data() {
+  if (abandoned_) {
+    return 0;
+  }
+
+  auto output = downstream_->get_request_buf();
+
+  std::array<struct iovec, 2> iov;
+  auto iovcnt = output->riovec(iov.data(), 2);
+
+  if (iovcnt == 0) {
+    send_reply(200, API_SUCCESS);
+
+    return 0;
+  }
+
+  std::unique_ptr<uint8_t[]> large_buf;
+
+  // If data spans in multiple chunks, pull them up into one
+  // contiguous buffer.
+  if (iovcnt > 1) {
+    large_buf = make_unique<uint8_t[]>(output->rleft());
+    auto len = output->rleft();
+    output->remove(large_buf.get(), len);
+
+    iov[0].iov_base = large_buf.get();
+    iov[0].iov_len = len;
+  }
+
+  Config config{};
+  config.conn.downstream = std::make_shared<DownstreamConfig>();
+  const auto &downstreamconf = config.conn.downstream;
+
+  auto &src = get_config()->conn.downstream;
+
+  downstreamconf->timeout = src->timeout;
+  downstreamconf->connections_per_host = src->connections_per_host;
+  downstreamconf->connections_per_frontend = src->connections_per_frontend;
+  downstreamconf->request_buffer_size = src->request_buffer_size;
+  downstreamconf->response_buffer_size = src->response_buffer_size;
+  downstreamconf->family = src->family;
+
+  std::set<StringRef> include_set;
+
+  for (auto first = reinterpret_cast<const uint8_t *>(iov[0].iov_base),
+            last = first + iov[0].iov_len;
+       first != last;) {
+    auto eol = std::find(first, last, '\n');
+    if (eol == last) {
+      break;
+    }
+
+    if (first == eol || *first == '#') {
+      first = ++eol;
+      continue;
+    }
+
+    auto eq = std::find(first, eol, '=');
+    if (eq == eol) {
+      send_reply(400, API_FAILURE);
+      return 0;
+    }
+
+    auto opt = StringRef{first, eq};
+    auto optval = StringRef{eq + 1, eol};
+
+    auto optid = option_lookup_token(opt.c_str(), opt.size());
+
+    switch (optid) {
+    case SHRPX_OPTID_BACKEND:
+      break;
+    default:
+      first = ++eol;
+      continue;
+    }
+
+    if (parse_config(&config, optid, opt, optval, include_set) != 0) {
+      send_reply(400, API_FAILURE);
+      return 0;
+    }
+
+    first = ++eol;
+  }
+
+  auto &tlsconf = get_config()->tls;
+  if (configure_downstream_group(&config, get_config()->http2_proxy, true,
+                                 tlsconf) != 0) {
+    send_reply(400, API_FAILURE);
+    return 0;
+  }
+
+  auto conn_handler = worker_->get_connection_handler();
+
+  conn_handler->send_replace_downstream(downstreamconf);
+
+  send_reply(200, API_SUCCESS);
+
+  return 0;
+}
+
+void APIDownstreamConnection::pause_read(IOCtrlReason reason) {}
+
+int APIDownstreamConnection::resume_read(IOCtrlReason reason, size_t consumed) {
+  return 0;
+}
+
+void APIDownstreamConnection::force_resume_read() {}
+
+int APIDownstreamConnection::on_read() { return 0; }
+
+int APIDownstreamConnection::on_write() { return 0; }
+
+void APIDownstreamConnection::on_upstream_change(Upstream *uptream) {}
+
+bool APIDownstreamConnection::poolable() const { return false; }
+
+DownstreamAddrGroup *
+APIDownstreamConnection::get_downstream_addr_group() const {
+  return nullptr;
+}
+
+DownstreamAddr *APIDownstreamConnection::get_addr() const { return nullptr; }
+
+} // namespace shrpx

src/shrpx_api_downstream_connection.h

@@ -0,0 +1,69 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2016 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef SHRPX_API_DOWNSTREAM_CONNECTION_H
+#define SHRPX_API_DOWNSTREAM_CONNECTION_H
+
+#include "shrpx_downstream_connection.h"
+
+namespace shrpx {
+
+class Worker;
+
+class APIDownstreamConnection : public DownstreamConnection {
+public:
+  APIDownstreamConnection(Worker *worker);
+  virtual ~APIDownstreamConnection();
+  virtual int attach_downstream(Downstream *downstream);
+  virtual void detach_downstream(Downstream *downstream);
+
+  virtual int push_request_headers();
+  virtual int push_upload_data_chunk(const uint8_t *data, size_t datalen);
+  virtual int end_upload_data();
+
+  virtual void pause_read(IOCtrlReason reason);
+  virtual int resume_read(IOCtrlReason reason, size_t consumed);
+  virtual void force_resume_read();
+
+  virtual int on_read();
+  virtual int on_write();
+
+  virtual void on_upstream_change(Upstream *uptream);
+
+  // true if this object is poolable.
+  virtual bool poolable() const;
+
+  virtual DownstreamAddrGroup *get_downstream_addr_group() const;
+  virtual DownstreamAddr *get_addr() const;
+
+  int send_reply(unsigned int http_status, int api_status);
+
+private:
+  Worker *worker_;
+  bool abandoned_;
+};
+
+} // namespace shrpx
+
+#endif // SHRPX_API_DOWNSTREAM_CONNECTION_H

src/shrpx_client_handler.cc

@@ -48,6 +48,8 @@
 #include "shrpx_downstream.h"
 #include "shrpx_http2_session.h"
 #include "shrpx_connect_blocker.h"
+#include "shrpx_api_downstream_connection.h"
+#include "shrpx_health_monitor_downstream_connection.h"
 #ifdef HAVE_SPDYLAY
 #include "shrpx_spdy_upstream.h"
 #endif // HAVE_SPDYLAY
@@ -123,10 +125,6 @@ int ClientHandler::read_clear() {
       rb_.reset();
     } else if (rb_.wleft() == 0) {
       conn_.rlimit.stopw();
-      if (reset_conn_rtimer_required_) {
-        reset_conn_rtimer_required_ = false;
-        ev_timer_again(conn_.loop, &conn_.rt);
-      }
       return 0;
     }
 
@@ -137,10 +135,6 @@ int ClientHandler::read_clear() {
     auto nread = conn_.read_clear(rb_.last, rb_.wleft());
 
     if (nread == 0) {
-      if (reset_conn_rtimer_required_) {
-        reset_conn_rtimer_required_ = false;
-        ev_timer_again(conn_.loop, &conn_.rt);
-      }
       return 0;
     }
 
@@ -155,8 +149,6 @@ int ClientHandler::read_clear() {
 int ClientHandler::write_clear() {
   std::array<iovec, 2> iov;
 
-  ev_timer_again(conn_.loop, &conn_.rt);
-
   for (;;) {
     if (on_write() != 0) {
       return -1;
@@ -226,11 +218,6 @@ int ClientHandler::read_tls() {
       rb_.reset();
     } else if (rb_.wleft() == 0) {
       conn_.rlimit.stopw();
-      if (reset_conn_rtimer_required_) {
-        reset_conn_rtimer_required_ = false;
-        ev_timer_again(conn_.loop, &conn_.rt);
-      }
-
       return 0;
     }
 
@@ -241,11 +228,6 @@ int ClientHandler::read_tls() {
     auto nread = conn_.read_tls(rb_.last, rb_.wleft());
 
     if (nread == 0) {
-      if (reset_conn_rtimer_required_) {
-        reset_conn_rtimer_required_ = false;
-        ev_timer_again(conn_.loop, &conn_.rt);
-      }
-
       return 0;
     }
 
@@ -260,8 +242,6 @@ int ClientHandler::read_tls() {
 int ClientHandler::write_tls() {
   struct iovec iov;
 
-  ev_timer_again(conn_.loop, &conn_.rt);
-
   ERR_clear_error();
 
   for (;;) {
@@ -406,8 +386,9 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
       faddr_(faddr),
       worker_(worker),
       left_connhd_len_(NGHTTP2_CLIENT_MAGIC_LEN),
+      affinity_hash_(0),
       should_close_after_write_(false),
-      reset_conn_rtimer_required_(false) {
+      affinity_hash_computed_(false) {
 
   ++worker_->get_worker_stat()->num_connections;
 
@@ -513,10 +494,12 @@ void ClientHandler::reset_upstream_write_timeout(ev_tstamp t) {
   }
 }
 
-void ClientHandler::signal_reset_upstream_conn_rtimer() {
-  reset_conn_rtimer_required_ = true;
+void ClientHandler::repeat_read_timer() {
+  ev_timer_again(conn_.loop, &conn_.rt);
 }
 
+void ClientHandler::stop_read_timer() { ev_timer_stop(conn_.loop, &conn_.rt); }
+
 int ClientHandler::validate_next_proto() {
   const unsigned char *next_proto = nullptr;
   unsigned int next_proto_len = 0;
@@ -667,8 +650,18 @@ void ClientHandler::pool_downstream_connection(
                      << " in group " << group;
   }
 
+  auto &shared_addr = group->shared_addr;
+
+  if (shared_addr->affinity == AFFINITY_NONE) {
     auto &dconn_pool = group->shared_addr->dconn_pool;
     dconn_pool.add_downstream_connection(std::move(dconn));
+
+    return;
+  }
+
+  auto addr = dconn->get_addr();
+  auto &dconn_pool = addr->dconn_pool;
+  dconn_pool->add_downstream_connection(std::move(dconn));
 }
 
 void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) {
@@ -681,6 +674,65 @@ void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) {
   dconn_pool.remove_downstream_connection(dconn);
 }
 
+namespace {
+// Computes 32bits hash for session affinity for IP address |ip|.
+uint32_t compute_affinity_from_ip(const StringRef &ip) {
+  int rv;
+  std::array<uint8_t, 32> buf;
+
+  rv = util::sha256(buf.data(), ip);
+  if (rv != 0) {
+    // Not sure when sha256 failed.  Just fall back to another
+    // function.
+    return util::hash32(ip);
+  }
+
+  return (static_cast<uint32_t>(buf[0]) << 24) |
+         (static_cast<uint32_t>(buf[1]) << 16) |
+         (static_cast<uint32_t>(buf[2]) << 8) | static_cast<uint32_t>(buf[3]);
+}
+} // namespace
+
+Http2Session *ClientHandler::select_http2_session_with_affinity(
+    const std::shared_ptr<DownstreamAddrGroup> &group, DownstreamAddr *addr) {
+  auto &shared_addr = group->shared_addr;
+
+  if (LOG_ENABLED(INFO)) {
+    CLOG(INFO, this) << "Selected DownstreamAddr=" << addr
+                     << ", index=" << (addr - shared_addr->addrs.data());
+  }
+
+  if (addr->http2_extra_freelist.size()) {
+    auto session = addr->http2_extra_freelist.head;
+
+    if (LOG_ENABLED(INFO)) {
+      CLOG(INFO, this) << "Use Http2Session " << session
+                       << " from http2_extra_freelist";
+    }
+
+    if (session->max_concurrency_reached(1)) {
+      if (LOG_ENABLED(INFO)) {
+        CLOG(INFO, this) << "Maximum streams are reached for Http2Session("
+                         << session << ").";
+      }
+
+      session->remove_from_freelist();
+    }
+    return session;
+  }
+
+  auto session = new Http2Session(conn_.loop, worker_->get_cl_ssl_ctx(),
+                                  worker_, group, addr);
+
+  if (LOG_ENABLED(INFO)) {
+    CLOG(INFO, this) << "Create new Http2Session " << session;
+  }
+
+  session->add_to_extra_freelist();
+
+  return session;
+}
+
 namespace {
 // Returns true if load of |lhs| is lighter than that of |rhs|.
 // Currently, we assume that lesser streams means lesser load.
@@ -689,8 +741,9 @@ bool load_lighter(const DownstreamAddr *lhs, const DownstreamAddr *rhs) {
 }
 } // namespace
 
-Http2Session *ClientHandler::select_http2_session(DownstreamAddrGroup &group) {
-  auto &shared_addr = group.shared_addr;
+Http2Session *ClientHandler::select_http2_session(
+    const std::shared_ptr<DownstreamAddrGroup> &group) {
+  auto &shared_addr = group->shared_addr;
 
   // First count the working backend addresses.
   size_t min = 0;
@@ -778,7 +831,7 @@ Http2Session *ClientHandler::select_http2_session(DownstreamAddrGroup &group) {
   }
 
   auto session = new Http2Session(conn_.loop, worker_->get_cl_ssl_ctx(),
-                                  worker_, &group, selected_addr);
+                                  worker_, group, selected_addr);
 
   if (LOG_ENABLED(INFO)) {
     CLOG(INFO, this) << "Create new Http2Session " << session;
@@ -814,12 +867,21 @@ uint32_t next_cycle(const WeightedPri &pri) {
 std::unique_ptr<DownstreamConnection>
 ClientHandler::get_downstream_connection(Downstream *downstream) {
   size_t group_idx;
-  auto &downstreamconf = get_config()->conn.downstream;
+  auto &downstreamconf = *worker_->get_downstream_config();
+  auto &routerconf = downstreamconf.router;
+
   auto catch_all = downstreamconf.addr_group_catch_all;
   auto &groups = worker_->get_downstream_addr_groups();
 
   const auto &req = downstream->request();
 
+  switch (faddr_->alt_mode) {
+  case ALTMODE_API:
+    return make_unique<APIDownstreamConnection>(worker_);
+  case ALTMODE_HEALTHMON:
+    return make_unique<HealthMonitorDownstreamConnection>();
+  }
+
   // Fast path.  If we have one group, it must be catch-all group.
   // proxy mode falls in this case.
   if (groups.size() == 1) {
@@ -830,21 +892,19 @@ ClientHandler::get_downstream_connection(Downstream *downstream) {
     //  have dealt with proxy case already, just use catch-all group.
     group_idx = catch_all;
   } else {
-    auto &router = get_config()->router;
-    auto &wildcard_patterns = get_config()->wildcard_patterns;
+    auto &balloc = downstream->get_block_allocator();
+
     if (!req.authority.empty()) {
-      group_idx =
-          match_downstream_addr_group(router, wildcard_patterns, req.authority,
-                                      req.path, groups, catch_all);
+      group_idx = match_downstream_addr_group(
+          routerconf, req.authority, req.path, groups, catch_all, balloc);
     } else {
       auto h = req.fs.header(http2::HD_HOST);
       if (h) {
-        group_idx = match_downstream_addr_group(
-            router, wildcard_patterns, h->value, req.path, groups, catch_all);
+        group_idx = match_downstream_addr_group(routerconf, h->value, req.path,
+                                                groups, catch_all, balloc);
       } else {
-        group_idx =
-            match_downstream_addr_group(router, wildcard_patterns, StringRef{},
-                                        req.path, groups, catch_all);
+        group_idx = match_downstream_addr_group(
+            routerconf, StringRef{}, req.path, groups, catch_all, balloc);
       }
     }
   }
@@ -853,14 +913,56 @@ ClientHandler::get_downstream_connection(Downstream *downstream) {
     CLOG(INFO, this) << "Downstream address group_idx: " << group_idx;
   }
 
-  auto &group = worker_->get_downstream_addr_groups()[group_idx];
-  auto &shared_addr = group.shared_addr;
+  auto &group = groups[group_idx];
+  auto &shared_addr = group->shared_addr;
 
-  auto proto = PROTO_NONE;
+  if (shared_addr->affinity == AFFINITY_IP) {
+    if (!affinity_hash_computed_) {
+      affinity_hash_ = compute_affinity_from_ip(StringRef{ipaddr_});
+      affinity_hash_computed_ = true;
+    }
+
+    const auto &affinity_hash = shared_addr->affinity_hash;
+
+    auto it = std::lower_bound(
+        std::begin(affinity_hash), std::end(affinity_hash), affinity_hash_,
+        [](const AffinityHash &lhs, uint32_t rhs) { return lhs.hash < rhs; });
+
+    if (it == std::end(affinity_hash)) {
+      it = std::begin(affinity_hash);
+    }
+
+    auto idx = (*it).idx;
+
+    auto &addr = shared_addr->addrs[idx];
+    if (addr.proto == PROTO_HTTP2) {
+      auto http2session = select_http2_session_with_affinity(group, &addr);
+
+      auto dconn = make_unique<Http2DownstreamConnection>(http2session);
+
+      dconn->set_client_handler(this);
+
+      return std::move(dconn);
+    }
+
+    auto &dconn_pool = addr.dconn_pool;
+    auto dconn = dconn_pool->pop_downstream_connection();
+
+    if (!dconn) {
+      dconn = make_unique<HttpDownstreamConnection>(group, idx, conn_.loop,
+                                                    worker_);
+    }
+
+    dconn->set_client_handler(this);
+
+    return dconn;
+  }
 
   auto http1_weight = shared_addr->http1_pri.weight;
   auto http2_weight = shared_addr->http2_pri.weight;
 
+  auto proto = PROTO_NONE;
+
   if (http1_weight > 0 && http2_weight > 0) {
     // We only advance cycle if both weight has nonzero to keep its
     // distance under WEIGHT_MAX.
@@ -920,7 +1022,8 @@ ClientHandler::get_downstream_connection(Downstream *downstream) {
                        << " Create new one";
     }
 
-    dconn = make_unique<HttpDownstreamConnection>(&group, conn_.loop, worker_);
+    dconn =
+        make_unique<HttpDownstreamConnection>(group, -1, conn_.loop, worker_);
   }
 
   dconn->set_client_handler(this);

src/shrpx_client_handler.h

@@ -50,6 +50,7 @@ class DownstreamConnectionPool;
 class Worker;
 struct WorkerStat;
 struct DownstreamAddrGroup;
+struct DownstreamAddr;
 
 class ClientHandler {
 public:
@@ -87,7 +88,7 @@ public:
   struct ev_loop *get_loop() const;
   void reset_upstream_read_timeout(ev_tstamp t);
   void reset_upstream_write_timeout(ev_tstamp t);
-  void signal_reset_upstream_conn_rtimer();
+
   int validate_next_proto();
   const std::string &get_ipaddr() const;
   const std::string &get_port() const;
@@ -123,7 +124,7 @@ public:
                        int64_t body_bytes_sent);
   Worker *get_worker() const;
 
-  using ReadBuf = Buffer<8_k>;
+  using ReadBuf = Buffer<16_k>;
 
   ReadBuf *get_rb();
 
@@ -142,10 +143,17 @@ public:
   // header field.
   StringRef get_forwarded_for() const;
 
-  Http2Session *select_http2_session(DownstreamAddrGroup &group);
+  Http2Session *
+  select_http2_session(const std::shared_ptr<DownstreamAddrGroup> &group);
+
+  Http2Session *select_http2_session_with_affinity(
+      const std::shared_ptr<DownstreamAddrGroup> &group, DownstreamAddr *addr);
 
   const UpstreamAddr *get_upstream_addr() const;
 
+  void repeat_read_timer();
+  void stop_read_timer();
+
 private:
   Connection conn_;
   ev_timer reneg_shutdown_timer_;
@@ -166,8 +174,11 @@ private:
   Worker *worker_;
   // The number of bytes of HTTP/2 client connection header to read
   size_t left_connhd_len_;
+  // hash for session affinity using client IP
+  uint32_t affinity_hash_;
   bool should_close_after_write_;
-  bool reset_conn_rtimer_required_;
+  // true if affinity_hash_ is computed
+  bool affinity_hash_computed_;
   ReadBuf rb_;
 };
 

src/shrpx_config.h

@@ -280,11 +280,25 @@ constexpr auto SHRPX_OPT_FRONTEND_HTTP2_SETTINGS_TIMEOUT =
     StringRef::from_lit("frontend-http2-settings-timeout");
 constexpr auto SHRPX_OPT_BACKEND_HTTP2_SETTINGS_TIMEOUT =
     StringRef::from_lit("backend-http2-settings-timeout");
+constexpr auto SHRPX_OPT_API_MAX_REQUEST_BODY =
+    StringRef::from_lit("api-max-request-body");
+constexpr auto SHRPX_OPT_BACKEND_MAX_BACKOFF =
+    StringRef::from_lit("backend-max-backoff");
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
+constexpr char DEFAULT_DOWNSTREAM_HOST[] = "127.0.0.1";
+constexpr int16_t DEFAULT_DOWNSTREAM_PORT = 80;
+
 enum shrpx_proto { PROTO_NONE, PROTO_HTTP1, PROTO_HTTP2, PROTO_MEMCACHED };
 
+enum shrpx_session_affinity {
+  // No session affinity
+  AFFINITY_NONE,
+  // Client IP affinity
+  AFFINITY_IP,
+};
+
 enum shrpx_forwarded_param {
   FORWARDED_NONE = 0,
   FORWARDED_BY = 0x1,
@@ -304,6 +318,15 @@ struct AltSvc {
   uint16_t port;
 };
 
+enum UpstreamAltMode {
+  // No alternative mode
+  ALTMODE_NONE,
+  // API processing mode
+  ALTMODE_API,
+  // Health monitor mode
+  ALTMODE_HEALTHMON,
+};
+
 struct UpstreamAddr {
   // The frontend address (e.g., FQDN, hostname, IP address).  If
   // |host_unix| is true, this is UNIX domain socket path.
@@ -317,6 +340,8 @@ struct UpstreamAddr {
   // For TCP socket, this is either AF_INET or AF_INET6.  For UNIX
   // domain socket, this is 0.
   int family;
+  // Alternate mode
+  int alt_mode;
   // true if |host| contains UNIX domain socket path.
   bool host_unix;
   // true if TLS is enabled.
@@ -345,12 +370,26 @@ struct DownstreamAddrConfig {
   bool tls;
 };
 
+// Mapping hash to idx which is an index into
+// DownstreamAddrGroupConfig::addrs.
+struct AffinityHash {
+  AffinityHash(size_t idx, uint32_t hash) : idx(idx), hash(hash) {}
+
+  size_t idx;
+  uint32_t hash;
+};
+
 struct DownstreamAddrGroupConfig {
   DownstreamAddrGroupConfig(const StringRef &pattern)
-      : pattern(pattern.c_str(), pattern.size()) {}
+      : pattern(pattern.c_str(), pattern.size()), affinity(AFFINITY_NONE) {}
 
   ImmutableString pattern;
   std::vector<DownstreamAddrConfig> addrs;
+  // Bunch of session affinity hash.  Only used if affinity ==
+  // AFFINITY_IP.
+  std::vector<AffinityHash> affinity_hash;
+  // Session affinity
+  shrpx_session_affinity affinity;
 };
 
 struct TicketKey {
@@ -538,6 +577,7 @@ struct Http2Config {
       ev_tstamp settings;
     } timeout;
     nghttp2_option *option;
+    nghttp2_option *alt_mode_option;
     nghttp2_session_callbacks *callbacks;
     size_t window_bits;
     size_t connection_window_bits;
@@ -581,6 +621,53 @@ struct RateLimitConfig {
   size_t burst;
 };
 
+// Wildcard host pattern routing.  We strips left most '*' from host
+// field.  router includes all path patterns sharing the same wildcard
+// host.
+struct WildcardPattern {
+  WildcardPattern(const StringRef &host)
+      : host(std::begin(host), std::end(host)) {}
+
+  ImmutableString host;
+  Router router;
+};
+
+// Configuration to select backend to forward request
+struct RouterConfig {
+  Router router;
+  // Router for reversed wildcard hosts.  Since this router has
+  // wildcard hosts reversed without '*', one should call match()
+  // function with reversed host stripping last character.  This is
+  // because we require at least one character must match for '*'.
+  // The index stored in this router is index of wildcard_patterns.
+  Router rev_wildcard_router;
+  std::vector<WildcardPattern> wildcard_patterns;
+};
+
+struct DownstreamConfig {
+  struct {
+    ev_tstamp read;
+    ev_tstamp write;
+    ev_tstamp idle_read;
+    // The maximum backoff while checking health check for offline
+    // backend or while detaching failed backend from load balancing
+    // group temporarily.
+    ev_tstamp max_backoff;
+  } timeout;
+  RouterConfig router;
+  std::vector<DownstreamAddrGroupConfig> addr_groups;
+  // The index of catch-all group in downstream_addr_groups.
+  size_t addr_group_catch_all;
+  size_t connections_per_host;
+  size_t connections_per_frontend;
+  size_t request_buffer_size;
+  size_t response_buffer_size;
+  // Address family of backend connection.  One of either AF_INET,
+  // AF_INET6 or AF_UNSPEC.  This is ignored if backend connection
+  // is made via Unix domain socket.
+  int family;
+};
+
 struct ConnectionConfig {
   struct {
     struct {
@@ -608,43 +695,24 @@ struct ConnectionConfig {
     bool accept_proxy_protocol;
   } upstream;
 
-  struct {
-    struct {
-      ev_tstamp read;
-      ev_tstamp write;
-      ev_tstamp idle_read;
-    } timeout;
-    std::vector<DownstreamAddrGroupConfig> addr_groups;
-    // The index of catch-all group in downstream_addr_groups.
-    size_t addr_group_catch_all;
-    size_t connections_per_host;
-    size_t connections_per_frontend;
-    size_t request_buffer_size;
-    size_t response_buffer_size;
-    // Address family of backend connection.  One of either AF_INET,
-    // AF_INET6 or AF_UNSPEC.  This is ignored if backend connection
-    // is made via Unix domain socket.
-    int family;
-  } downstream;
+  std::shared_ptr<DownstreamConfig> downstream;
 };
 
-// Wildcard host pattern routing.  We strips left most '*' from host
-// field.  router includes all path pattern sharing same wildcard
-// host.
-struct WildcardPattern {
-  ImmutableString host;
-  Router router;
+struct APIConfig {
+  // Maximum request body size for one API request
+  size_t max_request_body;
+  // true if at least one of UpstreamAddr has api enabled
+  bool enabled;
 };
 
 struct Config {
-  Router router;
-  std::vector<WildcardPattern> wildcard_patterns;
   HttpProxy downstream_http_proxy;
   HttpConfig http;
   Http2Config http2;
   TLSConfig tls;
   LoggingConfig logging;
   ConnectionConfig conn;
+  APIConfig api;
   ImmutableString pid_file;
   ImmutableString conf_path;
   ImmutableString user;
@@ -670,14 +738,158 @@ const Config *get_config();
 Config *mod_config();
 void create_config();
 
+// generated by gennghttpxfun.py
+enum {
+  SHRPX_OPTID_ACCEPT_PROXY_PROTOCOL,
+  SHRPX_OPTID_ACCESSLOG_FILE,
+  SHRPX_OPTID_ACCESSLOG_FORMAT,
+  SHRPX_OPTID_ACCESSLOG_SYSLOG,
+  SHRPX_OPTID_ADD_FORWARDED,
+  SHRPX_OPTID_ADD_REQUEST_HEADER,
+  SHRPX_OPTID_ADD_RESPONSE_HEADER,
+  SHRPX_OPTID_ADD_X_FORWARDED_FOR,
+  SHRPX_OPTID_ALTSVC,
+  SHRPX_OPTID_API_MAX_REQUEST_BODY,
+  SHRPX_OPTID_BACKEND,
+  SHRPX_OPTID_BACKEND_ADDRESS_FAMILY,
+  SHRPX_OPTID_BACKEND_CONNECTIONS_PER_FRONTEND,
+  SHRPX_OPTID_BACKEND_CONNECTIONS_PER_HOST,
+  SHRPX_OPTID_BACKEND_HTTP_PROXY_URI,
+  SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND,
+  SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST,
+  SHRPX_OPTID_BACKEND_HTTP1_TLS,
+  SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS,
+  SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER,
+  SHRPX_OPTID_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS,
+  SHRPX_OPTID_BACKEND_HTTP2_SETTINGS_TIMEOUT,
+  SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS,
+  SHRPX_OPTID_BACKEND_IPV4,
+  SHRPX_OPTID_BACKEND_IPV6,
+  SHRPX_OPTID_BACKEND_KEEP_ALIVE_TIMEOUT,
+  SHRPX_OPTID_BACKEND_MAX_BACKOFF,
+  SHRPX_OPTID_BACKEND_NO_TLS,
+  SHRPX_OPTID_BACKEND_READ_TIMEOUT,
+  SHRPX_OPTID_BACKEND_REQUEST_BUFFER,
+  SHRPX_OPTID_BACKEND_RESPONSE_BUFFER,
+  SHRPX_OPTID_BACKEND_TLS,
+  SHRPX_OPTID_BACKEND_TLS_SNI_FIELD,
+  SHRPX_OPTID_BACKEND_WRITE_TIMEOUT,
+  SHRPX_OPTID_BACKLOG,
+  SHRPX_OPTID_CACERT,
+  SHRPX_OPTID_CERTIFICATE_FILE,
+  SHRPX_OPTID_CIPHERS,
+  SHRPX_OPTID_CLIENT,
+  SHRPX_OPTID_CLIENT_CERT_FILE,
+  SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE,
+  SHRPX_OPTID_CLIENT_PROXY,
+  SHRPX_OPTID_CONF,
+  SHRPX_OPTID_DAEMON,
+  SHRPX_OPTID_DH_PARAM_FILE,
+  SHRPX_OPTID_ERROR_PAGE,
+  SHRPX_OPTID_ERRORLOG_FILE,
+  SHRPX_OPTID_ERRORLOG_SYSLOG,
+  SHRPX_OPTID_FASTOPEN,
+  SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE,
+  SHRPX_OPTID_FORWARDED_BY,
+  SHRPX_OPTID_FORWARDED_FOR,
+  SHRPX_OPTID_FRONTEND,
+  SHRPX_OPTID_FRONTEND_FRAME_DEBUG,
+  SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS,
+  SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER,
+  SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER,
+  SHRPX_OPTID_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS,
+  SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT,
+  SHRPX_OPTID_FRONTEND_HTTP2_SETTINGS_TIMEOUT,
+  SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS,
+  SHRPX_OPTID_FRONTEND_NO_TLS,
+  SHRPX_OPTID_FRONTEND_READ_TIMEOUT,
+  SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT,
+  SHRPX_OPTID_HEADER_FIELD_BUFFER,
+  SHRPX_OPTID_HOST_REWRITE,
+  SHRPX_OPTID_HTTP2_BRIDGE,
+  SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS,
+  SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING,
+  SHRPX_OPTID_HTTP2_PROXY,
+  SHRPX_OPTID_INCLUDE,
+  SHRPX_OPTID_INSECURE,
+  SHRPX_OPTID_LISTENER_DISABLE_TIMEOUT,
+  SHRPX_OPTID_LOG_LEVEL,
+  SHRPX_OPTID_MAX_HEADER_FIELDS,
+  SHRPX_OPTID_MAX_REQUEST_HEADER_FIELDS,
+  SHRPX_OPTID_MAX_RESPONSE_HEADER_FIELDS,
+  SHRPX_OPTID_MRUBY_FILE,
+  SHRPX_OPTID_NO_HOST_REWRITE,
+  SHRPX_OPTID_NO_HTTP2_CIPHER_BLACK_LIST,
+  SHRPX_OPTID_NO_KQUEUE,
+  SHRPX_OPTID_NO_LOCATION_REWRITE,
+  SHRPX_OPTID_NO_OCSP,
+  SHRPX_OPTID_NO_SERVER_PUSH,
+  SHRPX_OPTID_NO_VIA,
+  SHRPX_OPTID_NPN_LIST,
+  SHRPX_OPTID_OCSP_UPDATE_INTERVAL,
+  SHRPX_OPTID_PADDING,
+  SHRPX_OPTID_PID_FILE,
+  SHRPX_OPTID_PRIVATE_KEY_FILE,
+  SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE,
+  SHRPX_OPTID_READ_BURST,
+  SHRPX_OPTID_READ_RATE,
+  SHRPX_OPTID_REQUEST_HEADER_FIELD_BUFFER,
+  SHRPX_OPTID_RESPONSE_HEADER_FIELD_BUFFER,
+  SHRPX_OPTID_RLIMIT_NOFILE,
+  SHRPX_OPTID_STREAM_READ_TIMEOUT,
+  SHRPX_OPTID_STREAM_WRITE_TIMEOUT,
+  SHRPX_OPTID_STRIP_INCOMING_FORWARDED,
+  SHRPX_OPTID_STRIP_INCOMING_X_FORWARDED_FOR,
+  SHRPX_OPTID_SUBCERT,
+  SHRPX_OPTID_SYSLOG_FACILITY,
+  SHRPX_OPTID_TLS_DYN_REC_IDLE_TIMEOUT,
+  SHRPX_OPTID_TLS_DYN_REC_WARMUP_THRESHOLD,
+  SHRPX_OPTID_TLS_PROTO_LIST,
+  SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED,
+  SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY,
+  SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE,
+  SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE,
+  SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS,
+  SHRPX_OPTID_TLS_TICKET_KEY_CIPHER,
+  SHRPX_OPTID_TLS_TICKET_KEY_FILE,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_INTERVAL,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_RETRY,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS,
+  SHRPX_OPTID_USER,
+  SHRPX_OPTID_VERIFY_CLIENT,
+  SHRPX_OPTID_VERIFY_CLIENT_CACERT,
+  SHRPX_OPTID_WORKER_FRONTEND_CONNECTIONS,
+  SHRPX_OPTID_WORKER_READ_BURST,
+  SHRPX_OPTID_WORKER_READ_RATE,
+  SHRPX_OPTID_WORKER_WRITE_BURST,
+  SHRPX_OPTID_WORKER_WRITE_RATE,
+  SHRPX_OPTID_WORKERS,
+  SHRPX_OPTID_WRITE_BURST,
+  SHRPX_OPTID_WRITE_RATE,
+  SHRPX_OPTID_MAXIDX,
+};
+
+// Looks up token for given option name |name| of length |namelen|.
+int option_lookup_token(const char *name, size_t namelen);
+
 // Parses option name |opt| and value |optarg|.  The results are
-// stored into statically allocated Config object. This function
-// returns 0 if it succeeds, or -1.  The |included_set| contains the
-// all paths already included while processing this configuration, to
-// avoid loop in --include option.
-int parse_config(const StringRef &opt, const StringRef &optarg,
+// stored into the object pointed by |config|. This function returns 0
+// if it succeeds, or -1.  The |included_set| contains the all paths
+// already included while processing this configuration, to avoid loop
+// in --include option.
+int parse_config(Config *config, const StringRef &opt, const StringRef &optarg,
                  std::set<StringRef> &included_set);
 
+// Similar to parse_config() above, but additional |optid| which
+// should be the return value of option_lookup_token(opt).
+int parse_config(Config *config, int optid, const StringRef &opt,
+                 const StringRef &optarg, std::set<StringRef> &included_set);
+
 // Loads configurations from |filename| and stores them in statically
 // allocated Config object. This function returns 0 if it succeeds, or
 // -1.  See parse_config() for |include_set|.
@@ -710,6 +922,13 @@ read_tls_ticket_key_file(const std::vector<std::string> &files,
 // Returns string representation of |proto|.
 StringRef strproto(shrpx_proto proto);
 
+int configure_downstream_group(Config *config, bool http2_proxy,
+                               bool numeric_addr_only,
+                               const TLSConfig &tlsconf);
+
+int resolve_hostname(Address *addr, const char *hostname, uint16_t port,
+                     int family, int additional_flags = 0);
+
 } // namespace shrpx
 
 #endif // SHRPX_CONFIG_H

src/shrpx_connect_blocker.cc

@@ -23,6 +23,7 @@
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 #include "shrpx_connect_blocker.h"
+#include "shrpx_config.h"
 
 namespace shrpx {
 
@@ -79,10 +80,15 @@ void ConnectBlocker::on_failure() {
 
   ++fail_count_;
 
-  auto base_backoff = pow(MULTIPLIER, std::min(MAX_BACKOFF_EXP, fail_count_));
+  auto base_backoff =
+      util::int_pow(MULTIPLIER, std::min(MAX_BACKOFF_EXP, fail_count_));
   auto dist = std::uniform_real_distribution<>(-JITTER * base_backoff,
                                                JITTER * base_backoff);
-  auto backoff = base_backoff + dist(gen_);
+
+  auto &downstreamconf = *get_config()->conn.downstream;
+
+  auto backoff =
+      std::min(downstreamconf.timeout.max_backoff, base_backoff + dist(gen_));
 
   LOG(WARN) << "Could not connect " << fail_count_
             << " times in a row; sleep for " << backoff << " seconds";
@@ -112,6 +118,8 @@ void ConnectBlocker::offline() {
 void ConnectBlocker::online() {
   ev_timer_stop(loop_, &timer_);
 
+  call_unblock_func();
+
   fail_count_ = 0;
 
   offline_ = false;

src/shrpx_connect_blocker.h

@@ -66,7 +66,7 @@ public:
   void call_unblock_func();
 
 private:
-  std::mt19937 gen_;
+  std::mt19937 &gen_;
   // Called when blocking is started
   std::function<void()> block_func_;
   // Called when unblocked

src/shrpx_connection.cc

@@ -61,9 +61,6 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl,
     : tls{DefaultMemchunks(mcpool), DefaultPeekMemchunks(mcpool)},
       wlimit(loop, &wev, write_limit.rate, write_limit.burst),
       rlimit(loop, &rev, read_limit.rate, read_limit.burst, this),
-      writecb(writecb),
-      readcb(readcb),
-      timeoutcb(timeoutcb),
       loop(loop),
       data(data),
       fd(fd),

src/shrpx_connection.h

@@ -125,9 +125,6 @@ struct Connection {
   ev_timer rt;
   RateLimit wlimit;
   RateLimit rlimit;
-  IOCb writecb;
-  IOCb readcb;
-  TimerCb timeoutcb;
   struct ev_loop *loop;
   void *data;
   int fd;

src/shrpx_connection_handler.cc

@@ -103,16 +103,20 @@ void thread_join_async_cb(struct ev_loop *loop, ev_async *w, int revent) {
 } // namespace
 
 namespace {
-std::random_device rd;
+void serial_event_async_cb(struct ev_loop *loop, ev_async *w, int revent) {
+  auto h = static_cast<ConnectionHandler *>(w->data);
+
+  h->handle_serial_event();
+}
 } // namespace
 
-ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
-    : gen_(rd()),
+ConnectionHandler::ConnectionHandler(struct ev_loop *loop, std::mt19937 &gen)
+    : gen_(gen),
       single_worker_(nullptr),
       loop_(loop),
       tls_ticket_key_memcached_get_retry_count_(0),
       tls_ticket_key_memcached_fail_count_(0),
-      worker_round_robin_cnt_(0),
+      worker_round_robin_cnt_(get_config()->api.enabled ? 1 : 0),
       graceful_shutdown_(false) {
   ev_timer_init(&disable_acceptor_timer_, acceptor_disable_cb, 0., 0.);
   disable_acceptor_timer_.data = this;
@@ -125,6 +129,11 @@ ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
 
   ev_async_init(&thread_join_asyncev_, thread_join_async_cb);
 
+  ev_async_init(&serial_event_asyncev_, serial_event_async_cb);
+  serial_event_asyncev_.data = this;
+
+  ev_async_start(loop_, &serial_event_asyncev_);
+
   ev_child_init(&ocsp_.chldev, ocsp_chld_cb, 0, 0);
   ocsp_.chldev.data = this;
 
@@ -136,6 +145,7 @@ ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
 
 ConnectionHandler::~ConnectionHandler() {
   ev_child_stop(loop_, &ocsp_.chldev);
+  ev_async_stop(loop_, &serial_event_asyncev_);
   ev_async_stop(loop_, &thread_join_asyncev_);
   ev_io_stop(loop_, &ocsp_.rev);
   ev_timer_stop(loop_, &ocsp_timer_);
@@ -175,9 +185,21 @@ void ConnectionHandler::worker_reopen_log_files() {
   }
 }
 
+void ConnectionHandler::worker_replace_downstream(
+    std::shared_ptr<DownstreamConfig> downstreamconf) {
+  WorkerEvent wev{};
+
+  wev.type = REPLACE_DOWNSTREAM;
+  wev.downstreamconf = std::move(downstreamconf);
+
+  for (auto &worker : workers_) {
+    worker->send(wev);
+  }
+}
+
 int ConnectionHandler::create_single_worker() {
-  auto cert_tree = ssl::create_cert_lookup_tree();
-  auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree
+  cert_tree_ = ssl::create_cert_lookup_tree();
+  auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree_.get()
 #ifdef HAVE_NEVERBLEED
                                                                     ,
                                                   nb_.get()
@@ -207,9 +229,9 @@ int ConnectionHandler::create_single_worker() {
     all_ssl_ctx_.push_back(session_cache_ssl_ctx);
   }
 
-  single_worker_ =
-      make_unique<Worker>(loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx,
-                          cert_tree, ticket_keys_);
+  single_worker_ = make_unique<Worker>(
+      loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
+      ticket_keys_, this, get_config()->conn.downstream);
 #ifdef HAVE_MRUBY
   if (single_worker_->create_mruby_context() != 0) {
     return -1;
@@ -223,8 +245,8 @@ int ConnectionHandler::create_worker_thread(size_t num) {
 #ifndef NOTHREADS
   assert(workers_.size() == 0);
 
-  auto cert_tree = ssl::create_cert_lookup_tree();
-  auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree
+  cert_tree_ = ssl::create_cert_lookup_tree();
+  auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree_.get()
 #ifdef HAVE_NEVERBLEED
                                                                     ,
                                                   nb_.get()
@@ -242,6 +264,12 @@ int ConnectionHandler::create_worker_thread(size_t num) {
 
   auto &tlsconf = get_config()->tls;
   auto &memcachedconf = get_config()->tls.session_cache.memcached;
+  auto &apiconf = get_config()->api;
+
+  // We have dedicated worker for API request processing.
+  if (apiconf.enabled) {
+    ++num;
+  }
 
   for (size_t i = 0; i < num; ++i) {
     auto loop = ev_loop_new(get_config()->ev_loop_flags);
@@ -256,9 +284,9 @@ int ConnectionHandler::create_worker_thread(size_t num) {
           StringRef{memcachedconf.private_key_file}, nullptr);
       all_ssl_ctx_.push_back(session_cache_ssl_ctx);
     }
-    auto worker =
-        make_unique<Worker>(loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx,
-                            cert_tree, ticket_keys_);
+    auto worker = make_unique<Worker>(
+        loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
+        ticket_keys_, this, get_config()->conn.downstream);
 #ifdef HAVE_MRUBY
     if (worker->create_mruby_context() != 0) {
       return -1;
@@ -274,6 +302,7 @@ int ConnectionHandler::create_worker_thread(size_t num) {
   for (auto &worker : workers_) {
     worker->run_async();
   }
+
 #endif // NOTHREADS
 
   return 0;
@@ -358,11 +387,32 @@ int ConnectionHandler::handle_connection(int fd, sockaddr *addr, int addrlen,
     return 0;
   }
 
-  size_t idx = worker_round_robin_cnt_ % workers_.size();
+  Worker *worker;
+
+  if (faddr->alt_mode == ALTMODE_API) {
+    worker = workers_[0].get();
+
     if (LOG_ENABLED(INFO)) {
-    LOG(INFO) << "Dispatch connection to worker #" << idx;
+      LOG(INFO) << "Dispatch connection to API worker #0";
     }
-  ++worker_round_robin_cnt_;
+  } else {
+    worker = workers_[worker_round_robin_cnt_].get();
+
+    if (LOG_ENABLED(INFO)) {
+      LOG(INFO) << "Dispatch connection to worker #" << worker_round_robin_cnt_;
+    }
+
+    if (++worker_round_robin_cnt_ == workers_.size()) {
+      auto &apiconf = get_config()->api;
+
+      if (apiconf.enabled) {
+        worker_round_robin_cnt_ = 1;
+      } else {
+        worker_round_robin_cnt_ = 0;
+      }
+    }
+  }
+
   WorkerEvent wev{};
   wev.type = NEW_CONNECTION;
   wev.client_fd = fd;
@@ -370,7 +420,7 @@ int ConnectionHandler::handle_connection(int fd, sockaddr *addr, int addrlen,
   wev.client_addrlen = addrlen;
   wev.faddr = faddr;
 
-  workers_[idx]->send(wev);
+  worker->send(wev);
 
   return 0;
 }
@@ -682,6 +732,14 @@ ConnectionHandler::get_tls_ticket_key_memcached_dispatcher() const {
   return tls_ticket_key_memcached_dispatcher_.get();
 }
 
+// Use the similar backoff algorithm described in
+// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md
+namespace {
+constexpr size_t MAX_BACKOFF_EXP = 10;
+constexpr auto MULTIPLIER = 3.2;
+constexpr auto JITTER = 0.2;
+} // namespace
+
 void ConnectionHandler::on_tls_ticket_key_network_error(ev_timer *w) {
   if (++tls_ticket_key_memcached_get_retry_count_ >=
       get_config()->tls.ticket.memcached.max_retry) {
@@ -692,15 +750,19 @@ void ConnectionHandler::on_tls_ticket_key_network_error(ev_timer *w) {
     return;
   }
 
-  auto dist = std::uniform_int_distribution<int>(
-      1, std::min(60, 1 << tls_ticket_key_memcached_get_retry_count_));
-  auto t = dist(gen_);
+  auto base_backoff = util::int_pow(
+      MULTIPLIER,
+      std::min(MAX_BACKOFF_EXP, tls_ticket_key_memcached_get_retry_count_));
+  auto dist = std::uniform_real_distribution<>(-JITTER * base_backoff,
+                                               JITTER * base_backoff);
+
+  auto backoff = base_backoff + dist(gen_);
 
   LOG(WARN)
       << "Memcached: tls ticket get failed due to network error, retrying in "
-      << t << " seconds";
+      << backoff << " seconds";
 
-  ev_timer_set(w, t, 0.);
+  ev_timer_set(w, backoff, 0.);
   ev_timer_start(loop_, w);
 }
 
@@ -782,4 +844,50 @@ neverbleed_t *ConnectionHandler::get_neverbleed() const { return nb_.get(); }
 
 #endif // HAVE_NEVERBLEED
 
+void ConnectionHandler::handle_serial_event() {
+  std::vector<SerialEvent> q;
+  {
+    std::lock_guard<std::mutex> g(serial_event_mu_);
+    q.swap(serial_events_);
+  }
+
+  for (auto &sev : q) {
+    switch (sev.type) {
+    case SEV_REPLACE_DOWNSTREAM:
+      // Mmake sure that none of worker uses
+      // get_config()->conn.downstream
+      mod_config()->conn.downstream = sev.downstreamconf;
+
+      if (single_worker_) {
+        single_worker_->replace_downstream_config(sev.downstreamconf);
+
+        break;
+      }
+
+      worker_replace_downstream(sev.downstreamconf);
+
+      break;
+    }
+  }
+}
+
+void ConnectionHandler::send_replace_downstream(
+    const std::shared_ptr<DownstreamConfig> &downstreamconf) {
+  send_serial_event(SerialEvent(SEV_REPLACE_DOWNSTREAM, downstreamconf));
+}
+
+void ConnectionHandler::send_serial_event(SerialEvent ev) {
+  {
+    std::lock_guard<std::mutex> g(serial_event_mu_);
+
+    serial_events_.push_back(std::move(ev));
+  }
+
+  ev_async_send(loop_, &serial_event_asyncev_);
+}
+
+SSL_CTX *ConnectionHandler::get_ssl_ctx(size_t idx) const {
+  return all_ssl_ctx_[idx];
+}
+
 } // namespace shrpx

src/shrpx_connection_handler.h

@@ -32,6 +32,7 @@
 #include <sys/socket.h>
 #endif // HAVE_SYS_SOCKET_H
 
+#include <mutex>
 #include <memory>
 #include <vector>
 #include <random>
@@ -48,6 +49,7 @@
 #endif // HAVE_NEVERBLEED
 
 #include "shrpx_downstream_connection_pool.h"
+#include "shrpx_config.h"
 
 namespace shrpx {
 
@@ -60,6 +62,12 @@ struct TicketKeys;
 class MemcachedDispatcher;
 struct UpstreamAddr;
 
+namespace ssl {
+
+class CertLookupTree;
+
+} // namespace ssl
+
 struct OCSPUpdateContext {
   // ocsp response buffer
   std::vector<uint8_t> resp;
@@ -76,9 +84,24 @@ struct OCSPUpdateContext {
   pid_t pid;
 };
 
+// SerialEvent is an event sent from Worker thread.
+enum SerialEventType {
+  SEV_NONE,
+  SEV_REPLACE_DOWNSTREAM,
+};
+
+struct SerialEvent {
+  // ctor for event uses DownstreamConfig
+  SerialEvent(int type, const std::shared_ptr<DownstreamConfig> &downstreamconf)
+      : type(type), downstreamconf(downstreamconf) {}
+
+  int type;
+  std::shared_ptr<DownstreamConfig> downstreamconf;
+};
+
 class ConnectionHandler {
 public:
-  ConnectionHandler(struct ev_loop *loop);
+  ConnectionHandler(struct ev_loop *loop, std::mt19937 &gen);
   ~ConnectionHandler();
   int handle_connection(int fd, sockaddr *addr, int addrlen,
                         const UpstreamAddr *faddr);
@@ -130,12 +153,26 @@ public:
                                 ev_timer *w);
   void schedule_next_tls_ticket_key_memcached_get(ev_timer *w);
   SSL_CTX *create_tls_ticket_key_memcached_ssl_ctx();
+  // Returns the SSL_CTX at all_ssl_ctx_[idx].  This does not perform
+  // array bound checking.
+  SSL_CTX *get_ssl_ctx(size_t idx) const;
 
 #ifdef HAVE_NEVERBLEED
   void set_neverbleed(std::unique_ptr<neverbleed_t> nb);
   neverbleed_t *get_neverbleed() const;
 #endif // HAVE_NEVERBLEED
 
+  // Send SerialEvent SEV_REPLACE_DOWNSTREAM to this object.
+  void send_replace_downstream(
+      const std::shared_ptr<DownstreamConfig> &downstreamconf);
+  // Internal function to send |ev| to this object.
+  void send_serial_event(SerialEvent ev);
+  // Handles SerialEvents received.
+  void handle_serial_event();
+  // Sends WorkerEvent to make them replace downstream.
+  void
+  worker_replace_downstream(std::shared_ptr<DownstreamConfig> downstreamconf);
+
 private:
   // Stores all SSL_CTX objects.
   std::vector<SSL_CTX *> all_ssl_ctx_;
@@ -144,10 +181,17 @@ private:
   // ev_loop for each worker
   std::vector<struct ev_loop *> worker_loops_;
   // Worker instances when multi threaded mode (-nN, N >= 2) is used.
+  // If at least one frontend enables API request, we allocate 1
+  // additional worker dedicated to API request .
   std::vector<std::unique_ptr<Worker>> workers_;
+  // mutex for serial event resive buffer handling
+  std::mutex serial_event_mu_;
+  // SerialEvent receive buffer
+  std::vector<SerialEvent> serial_events_;
   // Worker instance used when single threaded mode (-n1) is used.
   // Otherwise, nullptr and workers_ has instances of Worker instead.
   std::unique_ptr<Worker> single_worker_;
+  std::unique_ptr<ssl::CertLookupTree> cert_tree_;
   std::unique_ptr<MemcachedDispatcher> tls_ticket_key_memcached_dispatcher_;
   // Current TLS session ticket keys.  Note that TLS connection does
   // not refer to this field directly.  They use TicketKeys object in
@@ -161,6 +205,7 @@ private:
   ev_timer disable_acceptor_timer_;
   ev_timer ocsp_timer_;
   ev_async thread_join_asyncev_;
+  ev_async serial_event_asyncev_;
 #ifndef NOTHREADS
   std::future<void> thread_join_fut_;
 #endif // NOTHREADS

src/shrpx_downstream.cc

@@ -500,12 +500,21 @@ bool Downstream::get_chunked_request() const { return chunked_request_; }
 void Downstream::set_chunked_request(bool f) { chunked_request_ = f; }
 
 bool Downstream::request_buf_full() {
-  if (dconn_) {
-    return request_buf_.rleft() >=
-           get_config()->conn.downstream.request_buffer_size;
-  } else {
+  auto handler = upstream_->get_client_handler();
+  auto faddr = handler->get_upstream_addr();
+  auto worker = handler->get_worker();
+
+  // We don't check buffer size here for API endpoint.
+  if (faddr->alt_mode == ALTMODE_API) {
     return false;
   }
+
+  if (dconn_) {
+    auto &downstreamconf = *worker->get_downstream_config();
+    return request_buf_.rleft() >= downstreamconf.request_buffer_size;
+  }
+
+  return false;
 }
 
 DefaultMemchunks *Downstream::get_request_buf() { return &request_buf_; }
@@ -521,13 +530,14 @@ int Downstream::push_request_headers() {
 }
 
 int Downstream::push_upload_data_chunk(const uint8_t *data, size_t datalen) {
+  req_.recv_body_length += datalen;
+
   // Assumes that request headers have already been pushed to output
   // buffer using push_request_headers().
   if (!dconn_) {
     DLOG(INFO, this) << "dconn_ is NULL";
     return -1;
   }
-  req_.recv_body_length += datalen;
   if (dconn_->push_upload_data_chunk(data, datalen) != 0) {
     return -1;
   }
@@ -593,11 +603,14 @@ DefaultMemchunks *Downstream::get_response_buf() { return &response_buf_; }
 
 bool Downstream::response_buf_full() {
   if (dconn_) {
-    return response_buf_.rleft() >=
-           get_config()->conn.downstream.response_buffer_size;
-  } else {
-    return false;
+    auto handler = upstream_->get_client_handler();
+    auto worker = handler->get_worker();
+    auto &downstreamconf = *worker->get_downstream_config();
+
+    return response_buf_.rleft() >= downstreamconf.response_buffer_size;
   }
+
+  return false;
 }
 
 bool Downstream::validate_request_recv_body_length() const {

src/shrpx_downstream_connection.h

@@ -35,6 +35,7 @@ class ClientHandler;
 class Upstream;
 class Downstream;
 struct DownstreamAddrGroup;
+struct DownstreamAddr;
 
 class DownstreamConnection {
 public:
@@ -61,6 +62,7 @@ public:
   virtual bool poolable() const = 0;
 
   virtual DownstreamAddrGroup *get_downstream_addr_group() const = 0;
+  virtual DownstreamAddr *get_addr() const = 0;
 
   void set_client_handler(ClientHandler *client_handler);
   ClientHandler *get_client_handler();

src/shrpx_downstream_connection_pool.cc

@@ -29,10 +29,14 @@ namespace shrpx {
 
 DownstreamConnectionPool::DownstreamConnectionPool() {}
 
-DownstreamConnectionPool::~DownstreamConnectionPool() {
+DownstreamConnectionPool::~DownstreamConnectionPool() { remove_all(); }
+
+void DownstreamConnectionPool::remove_all() {
   for (auto dconn : pool_) {
     delete dconn;
   }
+
+  pool_.clear();
 }
 
 void DownstreamConnectionPool::add_downstream_connection(

src/shrpx_downstream_connection_pool.h

@@ -42,6 +42,7 @@ public:
   void add_downstream_connection(std::unique_ptr<DownstreamConnection> dconn);
   std::unique_ptr<DownstreamConnection> pop_downstream_connection();
   void remove_downstream_connection(DownstreamConnection *dconn);
+  void remove_all();
 
 private:
   std::set<DownstreamConnection *> pool_;

src/shrpx_health_monitor_downstream_connection.cc

@@ -0,0 +1,107 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2016 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "shrpx_health_monitor_downstream_connection.h"
+
+#include "shrpx_client_handler.h"
+#include "shrpx_upstream.h"
+#include "shrpx_downstream.h"
+//#include "shrpx_connection_handler.h"
+
+namespace shrpx {
+
+HealthMonitorDownstreamConnection::HealthMonitorDownstreamConnection() {}
+
+HealthMonitorDownstreamConnection::~HealthMonitorDownstreamConnection() {}
+
+int HealthMonitorDownstreamConnection::attach_downstream(
+    Downstream *downstream) {
+  if (LOG_ENABLED(INFO)) {
+    DCLOG(INFO, this) << "Attaching to DOWNSTREAM:" << downstream;
+  }
+
+  downstream_ = downstream;
+
+  return 0;
+}
+
+void HealthMonitorDownstreamConnection::detach_downstream(
+    Downstream *downstream) {
+  if (LOG_ENABLED(INFO)) {
+    DCLOG(INFO, this) << "Detaching from DOWNSTREAM:" << downstream;
+  }
+  downstream_ = nullptr;
+}
+
+int HealthMonitorDownstreamConnection::push_request_headers() { return 0; }
+
+int HealthMonitorDownstreamConnection::push_upload_data_chunk(
+    const uint8_t *data, size_t datalen) {
+  return 0;
+}
+
+int HealthMonitorDownstreamConnection::end_upload_data() {
+  auto upstream = downstream_->get_upstream();
+  auto &resp = downstream_->response();
+
+  resp.http_status = 200;
+
+  resp.fs.add_header_token(StringRef::from_lit("content-length"),
+                           StringRef::from_lit("0"), false,
+                           http2::HD_CONTENT_LENGTH);
+
+  if (upstream->send_reply(downstream_, nullptr, 0) != 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+void HealthMonitorDownstreamConnection::pause_read(IOCtrlReason reason) {}
+
+int HealthMonitorDownstreamConnection::resume_read(IOCtrlReason reason,
+                                                   size_t consumed) {
+  return 0;
+}
+
+void HealthMonitorDownstreamConnection::force_resume_read() {}
+
+int HealthMonitorDownstreamConnection::on_read() { return 0; }
+
+int HealthMonitorDownstreamConnection::on_write() { return 0; }
+
+void HealthMonitorDownstreamConnection::on_upstream_change(Upstream *uptream) {}
+
+bool HealthMonitorDownstreamConnection::poolable() const { return false; }
+
+DownstreamAddrGroup *
+HealthMonitorDownstreamConnection::get_downstream_addr_group() const {
+  return nullptr;
+}
+
+DownstreamAddr *HealthMonitorDownstreamConnection::get_addr() const {
+  return nullptr;
+}
+
+} // namespace shrpx

src/shrpx_health_monitor_downstream_connection.h

@@ -0,0 +1,63 @@
+/*
+ * nghttp2 - HTTP/2 C Library
+ *
+ * Copyright (c) 2016 Tatsuhiro Tsujikawa
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef SHRPX_HEALTH_MONITOR_DOWNSTREAM_CONNECTION_H
+#define SHRPX_HEALTH_MONITOR_DOWNSTREAM_CONNECTION_H
+
+#include "shrpx_downstream_connection.h"
+
+namespace shrpx {
+
+class Worker;
+
+class HealthMonitorDownstreamConnection : public DownstreamConnection {
+public:
+  HealthMonitorDownstreamConnection();
+  virtual ~HealthMonitorDownstreamConnection();
+  virtual int attach_downstream(Downstream *downstream);
+  virtual void detach_downstream(Downstream *downstream);
+
+  virtual int push_request_headers();
+  virtual int push_upload_data_chunk(const uint8_t *data, size_t datalen);
+  virtual int end_upload_data();
+
+  virtual void pause_read(IOCtrlReason reason);
+  virtual int resume_read(IOCtrlReason reason, size_t consumed);
+  virtual void force_resume_read();
+
+  virtual int on_read();
+  virtual int on_write();
+
+  virtual void on_upstream_change(Upstream *uptream);
+
+  // true if this object is poolable.
+  virtual bool poolable() const;
+
+  virtual DownstreamAddrGroup *get_downstream_addr_group() const;
+  virtual DownstreamAddr *get_addr() const;
+};
+
+} // namespace shrpx
+
+#endif // SHRPX_HEALTH_MONITOR_DOWNSTREAM_CONNECTION_H

src/shrpx_http2_downstream_connection.cc

@@ -311,13 +311,6 @@ int Http2DownstreamConnection::push_request_headers() {
 
   http2::copy_headers_to_nva_nocopy(nva, req.fs.headers());
 
-  bool chunked_encoding = false;
-  auto transfer_encoding = req.fs.header(http2::HD_TRANSFER_ENCODING);
-  if (transfer_encoding &&
-      util::strieq_l("chunked", (*transfer_encoding).value)) {
-    chunked_encoding = true;
-  }
-
   if (!http2conf.no_cookie_crumbling) {
     downstream_->crumble_request_cookie(nva);
   }
@@ -426,11 +419,12 @@ int Http2DownstreamConnection::push_request_headers() {
     DCLOG(INFO, this) << "HTTP request headers\n" << ss.str();
   }
 
-  auto content_length = req.fs.header(http2::HD_CONTENT_LENGTH);
-  // TODO check content-length: 0 case
+  auto transfer_encoding = req.fs.header(http2::HD_TRANSFER_ENCODING);
 
-  if (req.method == HTTP_CONNECT || chunked_encoding || content_length ||
-      req.http2_expect_body) {
+  // Add body as long as transfer-encoding is given even if
+  // req.fs.content_length == 0 to forward trailer fields.
+  if (req.method == HTTP_CONNECT || transfer_encoding ||
+      req.fs.content_length > 0 || req.http2_expect_body) {
     // Request-body is expected.
     nghttp2_data_provider data_prd{{}, http2_data_read_callback};
     rv = http2session_->submit_request(this, nva.data(), nva.size(), &data_prd);
@@ -485,8 +479,7 @@ int Http2DownstreamConnection::resume_read(IOCtrlReason reason,
                                            size_t consumed) {
   int rv;
 
-  if (http2session_->get_state() != Http2Session::CONNECTED ||
-      !http2session_->get_flow_control()) {
+  if (http2session_->get_state() != Http2Session::CONNECTED) {
     return 0;
   }
 
@@ -550,4 +543,6 @@ Http2DownstreamConnection::get_downstream_addr_group() const {
   return http2session_->get_downstream_addr_group();
 }
 
+DownstreamAddr *Http2DownstreamConnection::get_addr() const { return nullptr; }
+
 } // namespace shrpx

src/shrpx_http2_downstream_connection.h

@@ -65,6 +65,7 @@ public:
   virtual bool poolable() const { return false; }
 
   virtual DownstreamAddrGroup *get_downstream_addr_group() const;
+  virtual DownstreamAddr *get_addr() const;
 
   int send();
 

src/shrpx_http2_session.cc

@@ -171,15 +171,23 @@ void initiate_connection_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 }
 } // namespace
 
+namespace {
+void prepare_cb(struct ev_loop *loop, ev_prepare *w, int revents) {
+  auto http2session = static_cast<Http2Session *>(w->data);
+  http2session->check_retire();
+}
+} // namespace
+
 Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
-                           Worker *worker, DownstreamAddrGroup *group,
+                           Worker *worker,
+                           const std::shared_ptr<DownstreamAddrGroup> &group,
                            DownstreamAddr *addr)
     : dlnext(nullptr),
       dlprev(nullptr),
       conn_(loop, -1, nullptr, worker->get_mcpool(),
-            get_config()->conn.downstream.timeout.write,
-            get_config()->conn.downstream.timeout.read, {}, {}, writecb, readcb,
-            timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold,
+            worker->get_downstream_config()->timeout.write,
+            worker->get_downstream_config()->timeout.read, {}, {}, writecb,
+            readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold,
             get_config()->tls.dyn_rec.idle_timeout, PROTO_HTTP2),
       wb_(worker->get_mcpool()),
       worker_(worker),
@@ -189,8 +197,7 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
       session_(nullptr),
       state_(DISCONNECTED),
       connection_check_state_(CONNECTION_CHECK_NONE),
-      freelist_zone_(FREELIST_ZONE_NONE),
-      flow_control_(false) {
+      freelist_zone_(FREELIST_ZONE_NONE) {
   read_ = write_ = &Http2Session::noop;
 
   on_read_ = &Http2Session::read_noop;
@@ -210,6 +217,10 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
 
   ev_timer_init(&initiate_connection_timer_, initiate_connection_cb, 0., 0.);
   initiate_connection_timer_.data = this;
+
+  ev_prepare_init(&prep_, prepare_cb);
+  prep_.data = this;
+  ev_prepare_start(loop, &prep_);
 }
 
 Http2Session::~Http2Session() {
@@ -229,6 +240,8 @@ int Http2Session::disconnect(bool hard) {
   conn_.rlimit.stopw();
   conn_.wlimit.stopw();
 
+  ev_prepare_stop(conn_.loop, &prep_);
+
   ev_timer_stop(conn_.loop, &initiate_connection_timer_);
   ev_timer_stop(conn_.loop, &settings_timer_);
   ev_timer_stop(conn_.loop, &connchk_timer_);
@@ -469,8 +482,11 @@ int Http2Session::initiate_connection() {
       ev_timer_again(conn_.loop, &conn_.wt);
     } else {
       conn_.rlimit.startw();
+
+      if (addr_->num_dconn == 0) {
         ev_timer_again(conn_.loop, &conn_.rt);
       }
+    }
 
     return 0;
   }
@@ -590,6 +606,8 @@ int Http2Session::downstream_connect_proxy() {
 void Http2Session::add_downstream_connection(Http2DownstreamConnection *dconn) {
   dconns_.append(dconn);
   ++addr_->num_dconn;
+
+  stop_read_timer();
 }
 
 void Http2Session::remove_downstream_connection(
@@ -598,6 +616,10 @@ void Http2Session::remove_downstream_connection(
   dconns_.remove(dconn);
   dconn->detach_stream_data();
 
+  if (addr_->num_dconn == 0) {
+    repeat_read_timer();
+  }
+
   if (LOG_ENABLED(INFO)) {
     SSLOG(INFO, this) << "Remove downstream";
   }
@@ -661,8 +683,6 @@ int Http2Session::submit_rst_stream(int32_t stream_id, uint32_t error_code) {
 
 nghttp2_session *Http2Session::get_session() const { return session_; }
 
-bool Http2Session::get_flow_control() const { return flow_control_; }
-
 int Http2Session::resume_data(Http2DownstreamConnection *dconn) {
   assert(state_ == CONNECTED);
   auto downstream = dconn->get_downstream();
@@ -1490,6 +1510,7 @@ int Http2Session::connection_made() {
 #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
 
     if (!next_proto) {
+      downstream_failure(addr_);
       return -1;
     }
 
@@ -1498,6 +1519,7 @@ int Http2Session::connection_made() {
       SSLOG(INFO, this) << "Negotiated next protocol: " << proto;
     }
     if (!util::check_h2_is_selected(proto)) {
+      downstream_failure(addr_);
       return -1;
     }
   }
@@ -1511,8 +1533,6 @@ int Http2Session::connection_made() {
     return -1;
   }
 
-  flow_control_ = true;
-
   std::array<nghttp2_settings_entry, 3> entry;
   size_t nentry = 2;
   entry[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
@@ -1533,11 +1553,11 @@ int Http2Session::connection_made() {
     return -1;
   }
 
-  auto connection_window_bits = http2conf.downstream.connection_window_bits;
-  if (connection_window_bits > 16) {
-    int32_t delta = (1 << connection_window_bits) - 1 -
-                    NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE;
-    rv = nghttp2_submit_window_update(session_, NGHTTP2_FLAG_NONE, 0, delta);
+  if (http2conf.downstream.connection_window_bits != 16) {
+    int32_t window_size =
+        (1 << http2conf.downstream.connection_window_bits) - 1;
+    rv = nghttp2_session_set_local_window_size(session_, NGHTTP2_FLAG_NONE, 0,
+                                               window_size);
     if (rv != 0) {
       return -1;
     }
@@ -1819,8 +1839,6 @@ int Http2Session::connected() {
 }
 
 int Http2Session::read_clear() {
-  ev_timer_again(conn_.loop, &conn_.rt);
-
   std::array<uint8_t, 16_k> buf;
 
   for (;;) {
@@ -1841,8 +1859,6 @@ int Http2Session::read_clear() {
 }
 
 int Http2Session::write_clear() {
-  ev_timer_again(conn_.loop, &conn_.rt);
-
   std::array<struct iovec, MAX_WR_IOVCNT> iov;
 
   for (;;) {
@@ -1888,6 +1904,8 @@ int Http2Session::tls_handshake() {
   }
 
   if (rv < 0) {
+    downstream_failure(addr_);
+
     return rv;
   }
 
@@ -1897,6 +1915,8 @@ int Http2Session::tls_handshake() {
 
   if (!get_config()->tls.insecure &&
       ssl::check_cert(conn_.tls.ssl, addr_) != 0) {
+    downstream_failure(addr_);
+
     return -1;
   }
 
@@ -1920,8 +1940,6 @@ int Http2Session::tls_handshake() {
 }
 
 int Http2Session::read_tls() {
-  ev_timer_again(conn_.loop, &conn_.rt);
-
   std::array<uint8_t, 16_k> buf;
 
   ERR_clear_error();
@@ -1944,8 +1962,6 @@ int Http2Session::read_tls() {
 }
 
 int Http2Session::write_tls() {
-  ev_timer_again(conn_.loop, &conn_.rt);
-
   ERR_clear_error();
 
   struct iovec iov;
@@ -2111,7 +2127,7 @@ bool Http2Session::max_concurrency_reached(size_t extra) const {
 }
 
 DownstreamAddrGroup *Http2Session::get_downstream_addr_group() const {
-  return group_;
+  return group_.get();
 }
 
 void Http2Session::add_to_avail_freelist() {
@@ -2120,8 +2136,8 @@ void Http2Session::add_to_avail_freelist() {
   }
 
   if (LOG_ENABLED(INFO)) {
-    SSLOG(INFO, this) << "Append to http2_avail_freelist, group=" << group_
-                      << ", freelist.size="
+    SSLOG(INFO, this) << "Append to http2_avail_freelist, group="
+                      << group_.get() << ", freelist.size="
                       << group_->shared_addr->http2_avail_freelist.size();
   }
 
@@ -2194,4 +2210,24 @@ void Http2Session::on_timeout() {
   }
 }
 
+void Http2Session::check_retire() {
+  if (!group_->retired) {
+    return;
+  }
+
+  ev_prepare_stop(conn_.loop, &prep_);
+
+  auto last_stream_id = nghttp2_session_get_last_proc_stream_id(session_);
+  nghttp2_submit_goaway(session_, NGHTTP2_FLAG_NONE, last_stream_id,
+                        NGHTTP2_NO_ERROR, nullptr, 0);
+
+  signal_write();
+}
+
+void Http2Session::repeat_read_timer() {
+  ev_timer_again(conn_.loop, &conn_.rt);
+}
+
+void Http2Session::stop_read_timer() { ev_timer_stop(conn_.loop, &conn_.rt); }
+
 } // namespace shrpx

src/shrpx_http2_session.h

@@ -73,7 +73,8 @@ enum FreelistZone {
 class Http2Session {
 public:
   Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, Worker *worker,
-               DownstreamAddrGroup *group, DownstreamAddr *addr);
+               const std::shared_ptr<DownstreamAddrGroup> &group,
+               DownstreamAddr *addr);
   ~Http2Session();
 
   // If hard is true, all pending requests are abandoned and
@@ -95,8 +96,6 @@ public:
 
   nghttp2_session *get_session() const;
 
-  bool get_flow_control() const;
-
   int resume_data(Http2DownstreamConnection *dconn);
 
   int connection_made();
@@ -199,6 +198,14 @@ public:
 
   void on_timeout();
 
+  // This is called periodically using ev_prepare watcher, and if
+  // group_ is retired (backend has been replaced), send GOAWAY to
+  // shutdown the connection.
+  void check_retire();
+
+  void repeat_read_timer();
+  void stop_read_timer();
+
   enum {
     // Disconnected
     DISCONNECTED,
@@ -240,6 +247,7 @@ private:
   ev_timer connchk_timer_;
   // timer to initiate connection.  usually, this fires immediately.
   ev_timer initiate_connection_timer_;
+  ev_prepare prep_;
   DList<Http2DownstreamConnection> dconns_;
   DList<StreamData> streams_;
   std::function<int(Http2Session &)> read_, write_;
@@ -250,14 +258,13 @@ private:
   Worker *worker_;
   // NULL if no TLS is configured
   SSL_CTX *ssl_ctx_;
-  DownstreamAddrGroup *group_;
+  std::shared_ptr<DownstreamAddrGroup> group_;
   // Address of remote endpoint
   DownstreamAddr *addr_;
   nghttp2_session *session_;
   int state_;
   int connection_check_state_;
   int freelist_zone_;
-  bool flow_control_;
 };
 
 nghttp2_session_callbacks *create_http2_downstream_callbacks();

src/shrpx_http2_upstream.cc

@@ -138,6 +138,9 @@ int Http2Upstream::upgrade_upstream(HttpsUpstream *http) {
   downstream_queue_.add_pending(std::move(downstream));
   downstream_queue_.mark_active(ptr);
 
+  // TODO This might not be necessary
+  handler_->stop_read_timer();
+
   if (LOG_ENABLED(INFO)) {
     ULOG(INFO, this) << "Connection upgraded to HTTP/2";
   }
@@ -413,6 +416,14 @@ void Http2Upstream::initiate_downstream(Downstream *downstream) {
 
   downstream_queue_.mark_active(downstream);
 
+  auto &req = downstream->request();
+  if (!req.http2_expect_body) {
+    rv = downstream->end_upload_data();
+    if (rv != 0) {
+      rst_stream(downstream, NGHTTP2_INTERNAL_ERROR);
+    }
+  }
+
   return;
 }
 
@@ -423,9 +434,6 @@ int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame,
     verbose_on_frame_recv_callback(session, frame, user_data);
   }
   auto upstream = static_cast<Http2Upstream *>(user_data);
-  auto handler = upstream->get_client_handler();
-
-  handler->signal_reset_upstream_conn_rtimer();
 
   switch (frame->hd.type) {
   case NGHTTP2_DATA: {
@@ -438,7 +446,12 @@ int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame,
     if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
       downstream->disable_upstream_rtimer();
 
-      downstream->end_upload_data();
+      if (downstream->end_upload_data() != 0) {
+        if (downstream->get_response_state() != Downstream::MSG_COMPLETE) {
+          upstream->rst_stream(downstream, NGHTTP2_INTERNAL_ERROR);
+        }
+      }
+
       downstream->set_request_state(Downstream::MSG_COMPLETE);
     }
 
@@ -460,7 +473,12 @@ int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame,
     if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
       downstream->disable_upstream_rtimer();
 
-      downstream->end_upload_data();
+      if (downstream->end_upload_data() != 0) {
+        if (downstream->get_response_state() != Downstream::MSG_COMPLETE) {
+          upstream->rst_stream(downstream, NGHTTP2_INTERNAL_ERROR);
+        }
+      }
+
       downstream->set_request_state(Downstream::MSG_COMPLETE);
     }
 
@@ -508,7 +526,9 @@ int on_data_chunk_recv_callback(nghttp2_session *session, uint8_t flags,
   downstream->reset_upstream_rtimer();
 
   if (downstream->push_upload_data_chunk(data, len) != 0) {
+    if (downstream->get_response_state() != Downstream::MSG_COMPLETE) {
       upstream->rst_stream(downstream, NGHTTP2_INTERNAL_ERROR);
+    }
 
     if (upstream->consume(stream_id, len) != 0) {
       return NGHTTP2_ERR_CALLBACK_FAILURE;
@@ -780,24 +800,26 @@ void Http2Upstream::submit_goaway() {
 
 void Http2Upstream::check_shutdown() {
   int rv;
-  if (shutdown_handled_) {
-    return;
-  }
 
   auto worker = handler_->get_worker();
 
-  if (worker->get_graceful_shutdown()) {
-    shutdown_handled_ = true;
+  if (!worker->get_graceful_shutdown()) {
+    return;
+  }
+
+  ev_prepare_stop(handler_->get_loop(), &prep_);
+
   rv = nghttp2_submit_shutdown_notice(session_);
   if (rv != 0) {
     ULOG(FATAL, this) << "nghttp2_submit_shutdown_notice() failed: "
                       << nghttp2_strerror(rv);
     return;
   }
+
   handler_->signal_write();
+
   ev_timer_start(handler_->get_loop(), &shutdown_timer_);
 }
-}
 
 nghttp2_session_callbacks *create_http2_upstream_callbacks() {
   int rv;
@@ -846,23 +868,34 @@ nghttp2_session_callbacks *create_http2_upstream_callbacks() {
   return callbacks;
 }
 
+namespace {
+size_t downstream_queue_size(Worker *worker) {
+  auto &downstreamconf = *worker->get_downstream_config();
+
+  if (get_config()->http2_proxy) {
+    return downstreamconf.connections_per_host;
+  }
+
+  return downstreamconf.connections_per_frontend;
+}
+} // namespace
+
 Http2Upstream::Http2Upstream(ClientHandler *handler)
     : wb_(handler->get_worker()->get_mcpool()),
-      downstream_queue_(
-          get_config()->http2_proxy
-              ? get_config()->conn.downstream.connections_per_host
-              : get_config()->conn.downstream.connections_per_frontend,
+      downstream_queue_(downstream_queue_size(handler->get_worker()),
                         !get_config()->http2_proxy),
       handler_(handler),
-      session_(nullptr),
-      shutdown_handled_(false) {
-
+      session_(nullptr) {
   int rv;
 
   auto &http2conf = get_config()->http2;
 
-  rv = nghttp2_session_server_new2(&session_, http2conf.upstream.callbacks,
-                                   this, http2conf.upstream.option);
+  auto faddr = handler_->get_upstream_addr();
+
+  rv = nghttp2_session_server_new2(
+      &session_, http2conf.upstream.callbacks, this,
+      faddr->alt_mode ? http2conf.upstream.alt_mode_option
+                      : http2conf.upstream.option);
 
   assert(rv == 0);
 
@@ -874,7 +907,11 @@ Http2Upstream::Http2Upstream(ClientHandler *handler)
   entry[0].value = http2conf.upstream.max_concurrent_streams;
 
   entry[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
+  if (faddr->alt_mode) {
+    entry[1].value = (1u << 31) - 1;
+  } else {
     entry[1].value = (1 << http2conf.upstream.window_bits) - 1;
+  }
 
   rv = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, entry.data(),
                                entry.size());
@@ -883,10 +920,13 @@ Http2Upstream::Http2Upstream(ClientHandler *handler)
                       << nghttp2_strerror(rv);
   }
 
-  if (http2conf.upstream.connection_window_bits > 16) {
-    int32_t delta = (1 << http2conf.upstream.connection_window_bits) - 1 -
-                    NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE;
-    rv = nghttp2_submit_window_update(session_, NGHTTP2_FLAG_NONE, 0, delta);
+  int32_t window_bits =
+      faddr->alt_mode ? 31 : http2conf.upstream.connection_window_bits;
+
+  if (window_bits != 16) {
+    int32_t window_size = (1u << window_bits) - 1;
+    rv = nghttp2_session_set_local_window_size(session_, NGHTTP2_FLAG_NONE, 0,
+                                               window_size);
 
     if (rv != 0) {
       ULOG(ERROR, this) << "nghttp2_submit_window_update() returned error: "
@@ -1334,6 +1374,8 @@ int Http2Upstream::error_reply(Downstream *downstream,
 void Http2Upstream::add_pending_downstream(
     std::unique_ptr<Downstream> downstream) {
   downstream_queue_.add_pending(std::move(downstream));
+
+  handler_->stop_read_timer();
 }
 
 void Http2Upstream::remove_downstream(Downstream *downstream) {
@@ -1349,6 +1391,11 @@ void Http2Upstream::remove_downstream(Downstream *downstream) {
   if (next_downstream) {
     initiate_downstream(next_downstream);
   }
+
+  if (downstream_queue_.get_downstreams() == nullptr) {
+    // There is no downstream at the moment.  Start idle timer now.
+    handler_->repeat_read_timer();
+  }
 }
 
 // WARNING: Never call directly or indirectly nghttp2_session_send or
@@ -1675,6 +1722,12 @@ int Http2Upstream::on_downstream_abort_request(Downstream *downstream,
 int Http2Upstream::consume(int32_t stream_id, size_t len) {
   int rv;
 
+  auto faddr = handler_->get_upstream_addr();
+
+  if (faddr->alt_mode) {
+    return 0;
+  }
+
   rv = nghttp2_session_consume(session_, stream_id, len);
 
   if (rv != 0) {

src/shrpx_http2_upstream.h

@@ -132,7 +132,6 @@ private:
   ClientHandler *handler_;
   nghttp2_session *session_;
   bool flow_control_;
-  bool shutdown_handled_;
 };
 
 nghttp2_session_callbacks *create_http2_upstream_callbacks();

src/shrpx_http_downstream_connection.cc

@@ -147,12 +147,12 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) {
 }
 } // namespace
 
-HttpDownstreamConnection::HttpDownstreamConnection(DownstreamAddrGroup *group,
-                                                   struct ev_loop *loop,
-                                                   Worker *worker)
+HttpDownstreamConnection::HttpDownstreamConnection(
+    const std::shared_ptr<DownstreamAddrGroup> &group, ssize_t initial_addr_idx,
+    struct ev_loop *loop, Worker *worker)
     : conn_(loop, -1, nullptr, worker->get_mcpool(),
-            get_config()->conn.downstream.timeout.write,
-            get_config()->conn.downstream.timeout.read, {}, {}, connectcb,
+            worker->get_downstream_config()->timeout.write,
+            worker->get_downstream_config()->timeout.read, {}, {}, connectcb,
             readcb, connect_timeoutcb, this,
             get_config()->tls.dyn_rec.warmup_threshold,
             get_config()->tls.dyn_rec.idle_timeout, PROTO_HTTP1),
@@ -164,9 +164,14 @@ HttpDownstreamConnection::HttpDownstreamConnection(DownstreamAddrGroup *group,
       group_(group),
       addr_(nullptr),
       ioctrl_(&conn_.rlimit),
-      response_htp_{0} {}
+      response_htp_{0},
+      initial_addr_idx_(initial_addr_idx) {}
 
-HttpDownstreamConnection::~HttpDownstreamConnection() {}
+HttpDownstreamConnection::~HttpDownstreamConnection() {
+  if (LOG_ENABLED(INFO)) {
+    DCLOG(INFO, this) << "Deleted";
+  }
+}
 
 int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
   if (LOG_ENABLED(INFO)) {
@@ -182,12 +187,18 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
     return SHRPX_ERR_NETWORK;
   }
 
-  auto &downstreamconf = get_config()->conn.downstream;
+  auto &downstreamconf = *worker_->get_downstream_config();
 
   if (conn_.fd == -1) {
     auto &shared_addr = group_->shared_addr;
     auto &addrs = shared_addr->addrs;
-    auto &next_downstream = shared_addr->next;
+
+    // If session affinity is enabled, we always start with address at
+    // initial_addr_idx_.
+    size_t temp_idx = initial_addr_idx_;
+
+    auto &next_downstream =
+        shared_addr->affinity == AFFINITY_NONE ? shared_addr->next : temp_idx;
     auto end = next_downstream;
     for (;;) {
       auto &addr = addrs[next_downstream];
@@ -303,7 +314,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
     // we may set read timer cb to idle_timeoutcb.  Reset again.
     conn_.rt.repeat = downstreamconf.timeout.read;
     ev_set_cb(&conn_.rt, timeoutcb);
-    ev_timer_again(conn_.loop, &conn_.rt);
+    ev_timer_stop(conn_.loop, &conn_.rt);
+
     ev_set_cb(&conn_.rev, readcb);
   }
 
@@ -373,9 +385,9 @@ int HttpDownstreamConnection::push_request_headers() {
     buf->append("\r\n");
   }
 
-  if (!connect_method && req.http2_expect_body &&
-      !req.fs.header(http2::HD_CONTENT_LENGTH)) {
-
+  // set transfer-encoding only when content-length is unknown and
+  // request body is expected.
+  if (!connect_method && req.http2_expect_body && req.fs.content_length == -1) {
     downstream_->set_chunked_request(true);
     buf->append("Transfer-Encoding: chunked\r\n");
   }
@@ -551,6 +563,24 @@ int HttpDownstreamConnection::end_upload_data() {
   return 0;
 }
 
+namespace {
+void remove_from_pool(HttpDownstreamConnection *dconn) {
+  auto group = dconn->get_downstream_addr_group();
+  auto &shared_addr = group->shared_addr;
+
+  if (shared_addr->affinity == AFFINITY_NONE) {
+    auto &dconn_pool =
+        dconn->get_downstream_addr_group()->shared_addr->dconn_pool;
+    dconn_pool.remove_downstream_connection(dconn);
+    return;
+  }
+
+  auto addr = dconn->get_addr();
+  auto &dconn_pool = addr->dconn_pool;
+  dconn_pool->remove_downstream_connection(dconn);
+}
+} // namespace
+
 namespace {
 void idle_readcb(struct ev_loop *loop, ev_io *w, int revents) {
   auto conn = static_cast<Connection *>(w->data);
@@ -558,9 +588,8 @@ void idle_readcb(struct ev_loop *loop, ev_io *w, int revents) {
   if (LOG_ENABLED(INFO)) {
     DCLOG(INFO, dconn) << "Idle connection EOF";
   }
-  auto &dconn_pool =
-      dconn->get_downstream_addr_group()->shared_addr->dconn_pool;
-  dconn_pool.remove_downstream_connection(dconn);
+
+  remove_from_pool(dconn);
   // dconn was deleted
 }
 } // namespace
@@ -572,9 +601,8 @@ void idle_timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) {
   if (LOG_ENABLED(INFO)) {
     DCLOG(INFO, dconn) << "Idle connection timeout";
   }
-  auto &dconn_pool =
-      dconn->get_downstream_addr_group()->shared_addr->dconn_pool;
-  dconn_pool.remove_downstream_connection(dconn);
+
+  remove_from_pool(dconn);
   // dconn was deleted
 }
 } // namespace
@@ -588,7 +616,9 @@ void HttpDownstreamConnection::detach_downstream(Downstream *downstream) {
   ev_set_cb(&conn_.rev, idle_readcb);
   ioctrl_.force_resume_read();
 
-  conn_.rt.repeat = get_config()->conn.downstream.timeout.idle_read;
+  auto &downstreamconf = *worker_->get_downstream_config();
+
+  conn_.rt.repeat = downstreamconf.timeout.idle_read;
   ev_set_cb(&conn_.rt, idle_timeoutcb);
   ev_timer_again(conn_.loop, &conn_.rt);
 
@@ -602,8 +632,10 @@ void HttpDownstreamConnection::pause_read(IOCtrlReason reason) {
 
 int HttpDownstreamConnection::resume_read(IOCtrlReason reason,
                                           size_t consumed) {
+  auto &downstreamconf = *worker_->get_downstream_config();
+
   if (downstream_->get_response_buf()->rleft() <=
-      get_config()->conn.downstream.request_buffer_size / 2) {
+      downstreamconf.request_buffer_size / 2) {
     ioctrl_.resume_read(reason);
   }
 
@@ -861,7 +893,6 @@ http_parser_settings htp_hooks = {
 } // namespace
 
 int HttpDownstreamConnection::read_clear() {
-  ev_timer_again(conn_.loop, &conn_.rt);
   std::array<uint8_t, 16_k> buf;
   int rv;
 
@@ -887,8 +918,6 @@ int HttpDownstreamConnection::read_clear() {
 }
 
 int HttpDownstreamConnection::write_clear() {
-  ev_timer_again(conn_.loop, &conn_.rt);
-
   auto upstream = downstream_->get_upstream();
   auto input = downstream_->get_request_buf();
 
@@ -934,6 +963,8 @@ int HttpDownstreamConnection::tls_handshake() {
   }
 
   if (rv < 0) {
+    downstream_failure(addr_);
+
     return rv;
   }
 
@@ -943,6 +974,8 @@ int HttpDownstreamConnection::tls_handshake() {
 
   if (!get_config()->tls.insecure &&
       ssl::check_cert(conn_.tls.ssl, addr_) != 0) {
+    downstream_failure(addr_);
+
     return -1;
   }
 
@@ -972,7 +1005,6 @@ int HttpDownstreamConnection::tls_handshake() {
 int HttpDownstreamConnection::read_tls() {
   ERR_clear_error();
 
-  ev_timer_again(conn_.loop, &conn_.rt);
   std::array<uint8_t, 16_k> buf;
   int rv;
 
@@ -1000,8 +1032,6 @@ int HttpDownstreamConnection::read_tls() {
 int HttpDownstreamConnection::write_tls() {
   ERR_clear_error();
 
-  ev_timer_again(conn_.loop, &conn_.rt);
-
   auto upstream = downstream_->get_upstream();
   auto input = downstream_->get_request_buf();
 
@@ -1164,9 +1194,11 @@ int HttpDownstreamConnection::noop() { return 0; }
 
 DownstreamAddrGroup *
 HttpDownstreamConnection::get_downstream_addr_group() const {
-  return group_;
+  return group_.get();
 }
 
 DownstreamAddr *HttpDownstreamConnection::get_addr() const { return addr_; }
 
+bool HttpDownstreamConnection::poolable() const { return !group_->retired; }
+
 } // namespace shrpx

src/shrpx_http_downstream_connection.h

@@ -42,7 +42,8 @@ struct DownstreamAddr;
 
 class HttpDownstreamConnection : public DownstreamConnection {
 public:
-  HttpDownstreamConnection(DownstreamAddrGroup *group, struct ev_loop *loop,
+  HttpDownstreamConnection(const std::shared_ptr<DownstreamAddrGroup> &group,
+                           ssize_t initial_addr_idx, struct ev_loop *loop,
                            Worker *worker);
   virtual ~HttpDownstreamConnection();
   virtual int attach_downstream(Downstream *downstream);
@@ -61,9 +62,10 @@ public:
 
   virtual void on_upstream_change(Upstream *upstream);
 
-  virtual bool poolable() const { return true; }
+  virtual bool poolable() const;
 
   virtual DownstreamAddrGroup *get_downstream_addr_group() const;
+  virtual DownstreamAddr *get_addr() const;
 
   int read_clear();
   int write_clear();
@@ -79,8 +81,6 @@ public:
 
   int noop();
 
-  DownstreamAddr *get_addr() const;
-
 private:
   Connection conn_;
   std::function<int(HttpDownstreamConnection &)> do_read_, do_write_,
@@ -88,11 +88,12 @@ private:
   Worker *worker_;
   // nullptr if TLS is not used.
   SSL_CTX *ssl_ctx_;
-  DownstreamAddrGroup *group_;
+  const std::shared_ptr<DownstreamAddrGroup> &group_;
   // Address of remote endpoint
   DownstreamAddr *addr_;
   IOControl ioctrl_;
   http_parser response_htp_;
+  ssize_t initial_addr_idx_;
 };
 
 } // namespace shrpx