nghttp2/nghttp2
1
0
Fork 0
mirror of https://github.com/nghttp2/nghttp2.git synced 2025-12-06 18:18:52 +08:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2124 from nghttp2/nghttpx-cid

Browse Source 
nghttpx: Rework Connection ID construction
This commit is contained in:
Tatsuhiro Tsujikawa
2024-03-30 11:31:28 +09:00
committed by GitHub
parent c97376d456 ab4b37afab
commit 4dea77ce39
15 changed files with 223 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
bpf/reuseport_kern.c
View File

@@ -325,7 +325,7 @@ struct {
__uint(max_entries, 255);
__type(key, __u64);
__type(value, __u32);
} cid_prefix_map SEC(".maps");
} worker_id_map SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_REUSEPORT_SOCKARRAY);
@@ -355,11 +355,11 @@ typedef struct quic_hd {
__u8 type;
} quic_hd;
#define SV_DCIDLEN 20
#define SV_DCIDLEN 17
#define MAX_DCIDLEN 20
#define MIN_DCIDLEN 8
#define CID_PREFIXLEN 8
#define CID_PREFIX_OFFSET 1
#define WORKER_IDLEN 8
#define WORKER_ID_OFFSET 1
enum {
NGTCP2_PKT_INITIAL = 0x0,
@@ -483,7 +483,7 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) {
quic_hd qhd;
__u8 qpktbuf[6 + MAX_DCIDLEN];
struct AES_ctx *aes_ctx;
__u8 *cid_prefix;
__u8 *worker_id;
if (bpf_skb_load_bytes(reuse_md, sizeof(struct udphdr), qpktbuf,
sizeof(qpktbuf)) != 0) {
@@ -509,10 +509,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) {
case NGTCP2_PKT_INITIAL:
case NGTCP2_PKT_0RTT:
if (qhd.dcidlen == SV_DCIDLEN) {
cid_prefix = qhd.dcid + CID_PREFIX_OFFSET;
AES_ECB_decrypt(aes_ctx, cid_prefix);
worker_id = qhd.dcid + WORKER_ID_OFFSET;
AES_ECB_decrypt(aes_ctx, worker_id);
psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix);
psk_index = bpf_map_lookup_elem(&worker_id_map, worker_id);
if (psk_index != NULL) {
sk_index = *psk_index;
@@ -529,10 +529,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) {
return SK_DROP;
}
cid_prefix = qhd.dcid + CID_PREFIX_OFFSET;
AES_ECB_decrypt(aes_ctx, cid_prefix);
worker_id = qhd.dcid + WORKER_ID_OFFSET;
AES_ECB_decrypt(aes_ctx, worker_id);
psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix);
psk_index = bpf_map_lookup_elem(&worker_id_map, worker_id);
if (psk_index == NULL) {
sk_index = sk_index_from_dcid(&qhd, reuse_md, *pnum_socks);

2
doc/sources/nghttpx-howto.rst
View File

@@ -546,7 +546,7 @@ keys in order to keep the existing connections alive during reload.
The construction of Connection ID closely follows Block Cipher CID
Algorithm described in `QUIC-LB draft
<https://datatracker.ietf.org/doc/html/draft-ietf-quic-load-balancers>`_.
A Connection ID that nghttpx generates is always 20 bytes long. It
A Connection ID that nghttpx generates is always 17 bytes long. It
uses first 2 bits as a configuration ID. The remaining bits in the
first byte are reserved and random. The next 4 bytes are server ID.
The next 4 bytes are used to route UDP datagram to a correct

116
src/shrpx.cc
View File

@@ -141,11 +141,13 @@ constexpr auto ENV_ACCEPT_PREFIX = StringRef::from_lit("NGHTTPX_ACCEPT_");
constexpr auto ENV_ORIG_PID = StringRef::from_lit("NGHTTPX_ORIG_PID");
// Prefix of environment variables to tell new binary the QUIC IPC
// file descriptor and CID prefix of the lingering worker process.
// The value must be comma separated parameters:
// <FD>,<CID_PREFIX_0>,<CID_PREFIX_1>,... <FD> is the file
// descriptor. <CID_PREFIX_I> is the I-th CID prefix in hex encoded
// string.
// file descriptor and Worker ID of the lingering worker process. The
// value must be comma separated parameters:
//
// <FD>,<WORKER_ID_0>,<WORKER_ID_1>,...,<WORKER_ID_I>
//
// <FD> is the file descriptor. <WORKER_ID_I> is the I-th Worker ID
// in hex encoded string.
constexpr auto ENV_QUIC_WORKER_PROCESS_PREFIX =
StringRef::from_lit("NGHTTPX_QUIC_WORKER_PROCESS_");
@@ -203,9 +205,7 @@ struct WorkerProcess {
WorkerProcess(struct ev_loop *loop, pid_t worker_pid, int ipc_fd
#ifdef ENABLE_HTTP3
,
int quic_ipc_fd,
const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>>
&cid_prefixes
int quic_ipc_fd, std::vector<WorkerID> worker_ids
#endif // ENABLE_HTTP3
)
: loop(loop),
@@ -214,7 +214,7 @@ struct WorkerProcess {
#ifdef ENABLE_HTTP3
,
quic_ipc_fd(quic_ipc_fd),
cid_prefixes(cid_prefixes)
worker_ids(std::move(worker_ids))
#endif // ENABLE_HTTP3
{
ev_child_init(&worker_process_childev, worker_process_child_cb, worker_pid,
@@ -245,7 +245,7 @@ struct WorkerProcess {
std::chrono::steady_clock::time_point termination_deadline;
#ifdef ENABLE_HTTP3
int quic_ipc_fd;
std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes;
std::vector<WorkerID> worker_ids;
#endif // ENABLE_HTTP3
};
@@ -582,9 +582,10 @@ void exec_binary() {
s += util::utos(i + 1);
s += '=';
s += util::utos(wp->quic_ipc_fd);
for (auto &cid_prefix : wp->cid_prefixes) {
for (auto &wid : wp->worker_ids) {
s += ',';
s += util::format_hex(cid_prefix);
s += util::format_hex(reinterpret_cast<const unsigned char *>(&wid),
sizeof(wid));
}
quic_lwps.emplace_back(s);
@@ -1258,26 +1259,27 @@ get_inherited_quic_lingering_worker_process_from_env() {
util::make_socket_closeonexec(fd);
std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes;
std::vector<WorkerID> worker_ids;
auto p = end_fd + 1;
for (;;) {
auto end = std::find(p, envend, ',');
auto hex_cid_prefix = StringRef{p, end};
if (hex_cid_prefix.size() != SHRPX_QUIC_CID_PREFIXLEN * 2 ||
!util::is_hex_string(hex_cid_prefix)) {
LOG(WARN) << "Found invalid CID prefix=" << hex_cid_prefix;
auto hex_wid = StringRef{p, end};
if (hex_wid.size() != SHRPX_QUIC_WORKER_IDLEN * 2 ||
!util::is_hex_string(hex_wid)) {
LOG(WARN) << "Found invalid WorkerID=" << hex_wid;
break;
}
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Inherit worker process CID prefix=" << hex_cid_prefix;
LOG(INFO) << "Inherit worker process WorkerID=" << hex_wid;
}
cid_prefixes.emplace_back();
worker_ids.emplace_back();
util::decode_hex(std::begin(cid_prefixes.back()), hex_cid_prefix);
util::decode_hex(reinterpret_cast<uint8_t *>(&worker_ids.back()),
hex_wid);
if (end == envend) {
break;
@@ -1286,7 +1288,7 @@ get_inherited_quic_lingering_worker_process_from_env() {
p = end + 1;
}
iwps.emplace_back(std::move(cid_prefixes), fd);
iwps.emplace_back(std::move(worker_ids), fd);
}
return iwps;
@@ -1418,30 +1420,29 @@ int create_quic_ipc_socket(std::array<int, 2> &quic_ipc_fd) {
} // namespace
namespace {
int generate_cid_prefix(
std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> &cid_prefixes,
const Config *config) {
int generate_worker_id(std::vector<WorkerID> &worker_ids,
const Config *config) {
auto &apiconf = config->api;
auto &quicconf = config->quic;
size_t num_cid_prefix;
size_t num_wid;
if (config->single_thread) {
num_cid_prefix = 1;
num_wid = 1;
} else {
num_cid_prefix = config->num_worker;
num_wid = config->num_worker;
// API endpoint occupies the one dedicated worker thread.
// Although such worker never gets QUIC traffic, we create CID
// prefix for it to make code a bit simpler.
// Although such worker never gets QUIC traffic, we create Worker
// ID for it to make code a bit simpler.
if (apiconf.enabled) {
++num_cid_prefix;
++num_wid;
}
}
cid_prefixes.resize(num_cid_prefix);
worker_ids.resize(num_wid);
for (auto &cid_prefix : cid_prefixes) {
if (create_cid_prefix(cid_prefix.data(), quicconf.server_id.data()) != 0) {
for (auto &wid : worker_ids) {
if (create_worker_id(wid, quicconf.server_id) != 0) {
return -1;
}
}
@@ -1458,7 +1459,7 @@ collect_quic_lingering_worker_processes() {
std::end(inherited_quic_lingering_worker_processes)};
for (auto &wp : worker_processes) {
quic_lwps.emplace_back(wp->cid_prefixes, wp->quic_ipc_fd);
quic_lwps.emplace_back(wp->worker_ids, wp->quic_ipc_fd);
}
return quic_lwps;
@@ -1596,19 +1597,17 @@ namespace {
// |main_ipc_fd|. In child process, we will close file descriptors
// which are inherited from previous configuration/process, but not
// used in the current configuration.
pid_t fork_worker_process(
int &main_ipc_fd
pid_t fork_worker_process(int &main_ipc_fd
#ifdef ENABLE_HTTP3
,
int &wp_quic_ipc_fd
,
int &wp_quic_ipc_fd
#endif // ENABLE_HTTP3
,
const std::vector<InheritedAddr> &iaddrs
,
const std::vector<InheritedAddr> &iaddrs
#ifdef ENABLE_HTTP3
,
const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>>
&cid_prefixes,
const std::vector<QUICLingeringWorkerProcess> &quic_lwps
,
std::vector<WorkerID> worker_ids,
std::vector<QUICLingeringWorkerProcess> quic_lwps
#endif // ENABLE_HTTP3
) {
std::array<char, STRERROR_BUFSIZE> errbuf;
@@ -1714,9 +1713,9 @@ pid_t fork_worker_process(
.ipc_fd = ipc_fd[0],
.ready_ipc_fd = worker_process_ready_ipc_fd[1],
#ifdef ENABLE_HTTP3
.cid_prefixes = cid_prefixes,
.worker_ids = std::move(worker_ids),
.quic_ipc_fd = quic_ipc_fd[0],
.quic_lingering_worker_processes = quic_lwps,
.quic_lingering_worker_processes = std::move(quic_lwps),
#endif // ENABLE_HTTP3
};
rv = worker_process_event_loop(&wpconf);
@@ -1835,9 +1834,9 @@ int event_loop() {
auto quic_lwps = collect_quic_lingering_worker_processes();
std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes;
std::vector<WorkerID> worker_ids;
if (generate_cid_prefix(cid_prefixes, config) != 0) {
if (generate_worker_id(worker_ids, config) != 0) {
return -1;
}
#endif // ENABLE_HTTP3
@@ -1858,7 +1857,7 @@ int event_loop() {
{}
#ifdef ENABLE_HTTP3
,
cid_prefixes, quic_lwps
worker_ids, std::move(quic_lwps)
#endif // ENABLE_HTTP3
);
@@ -1872,9 +1871,10 @@ int event_loop() {
worker_process_add(std::make_unique<WorkerProcess>(loop, pid, ipc_fd
#ifdef ENABLE_HTTP3
,
quic_ipc_fd, cid_prefixes
quic_ipc_fd,
std::move(worker_ids)
#endif // ENABLE_HTTP3
));
));
// Write PID file when we are ready to accept connection from peer.
// This makes easier to write restart script for nghttpx. Because
@@ -2089,7 +2089,8 @@ void fill_default_config(Config *config) {
static_cast<ev_tstamp>(NGTCP2_DEFAULT_INITIAL_RTT) / NGTCP2_SECONDS;
}
if (RAND_bytes(quicconf.server_id.data(), quicconf.server_id.size()) != 1) {
if (RAND_bytes(reinterpret_cast<unsigned char *>(&quicconf.server_id),
sizeof(quicconf.server_id)) != 1) {
assert(0);
abort();
}
@@ -4003,9 +4004,9 @@ void reload_config() {
auto quic_lwps = collect_quic_lingering_worker_processes();
std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes;
std::vector<WorkerID> worker_ids;
if (generate_cid_prefix(cid_prefixes, new_config.get()) != 0) {
if (generate_worker_id(worker_ids, new_config.get()) != 0) {
close_not_inherited_fd(new_config.get(), iaddrs);
return;
}
@@ -4026,7 +4027,7 @@ void reload_config() {
iaddrs
#ifdef ENABLE_HTTP3
,
cid_prefixes, quic_lwps
worker_ids, std::move(quic_lwps)
#endif // ENABLE_HTTP3
);
@@ -4044,9 +4045,10 @@ void reload_config() {
worker_process_add(std::make_unique<WorkerProcess>(loop, pid, ipc_fd
#ifdef ENABLE_HTTP3
,
quic_ipc_fd, cid_prefixes
quic_ipc_fd,
std::move(worker_ids)
#endif // ENABLE_HTTP3
));
));
worker_process_adjust_limit();

11
src/shrpx_config.cc
View File

@@ -4144,12 +4144,13 @@ int parse_config(Config *config, int optid, const StringRef &opt,
return 0;
case SHRPX_OPTID_QUIC_SERVER_ID:
#ifdef ENABLE_HTTP3
if (optarg.size() != config->quic.server_id.size() * 2 ||
if (optarg.size() != sizeof(config->quic.server_id) * 2 ||
!util::is_hex_string(optarg)) {
LOG(ERROR) << opt << ": must be a hex-string";
return -1;
}
util::decode_hex(std::begin(config->quic.server_id), optarg);
util::decode_hex(reinterpret_cast<uint8_t *>(&config->quic.server_id),
optarg);
#endif // ENABLE_HTTP3
return 0;
@@ -4718,6 +4719,7 @@ int resolve_hostname(Address *addr, const char *hostname, uint16_t port,
#ifdef ENABLE_HTTP3
QUICKeyingMaterial::QUICKeyingMaterial(QUICKeyingMaterial &&other) noexcept
: cid_encryption_ctx{std::exchange(other.cid_encryption_ctx, nullptr)},
cid_decryption_ctx{std::exchange(other.cid_decryption_ctx, nullptr)},
reserved{other.reserved},
secret{other.secret},
salt{other.salt},
@@ -4728,11 +4730,16 @@ QUICKeyingMaterial::~QUICKeyingMaterial() noexcept {
if (cid_encryption_ctx) {
EVP_CIPHER_CTX_free(cid_encryption_ctx);
}
if (cid_decryption_ctx) {
EVP_CIPHER_CTX_free(cid_decryption_ctx);
}
}
QUICKeyingMaterial &
QUICKeyingMaterial::operator=(QUICKeyingMaterial &&other) noexcept {
cid_encryption_ctx = std::exchange(other.cid_encryption_ctx, nullptr);
cid_decryption_ctx = std::exchange(other.cid_decryption_ctx, nullptr);
reserved = other.reserved;
secret = other.secret;
salt = other.salt;

3
src/shrpx_config.h
View File

@@ -647,6 +647,7 @@ struct QUICKeyingMaterial {
~QUICKeyingMaterial() noexcept;
QUICKeyingMaterial &operator=(QUICKeyingMaterial &&other) noexcept;
EVP_CIPHER_CTX *cid_encryption_ctx;
EVP_CIPHER_CTX *cid_decryption_ctx;
std::array<uint8_t, SHRPX_QUIC_SECRET_RESERVEDLEN> reserved;
std::array<uint8_t, SHRPX_QUIC_SECRETLEN> secret;
std::array<uint8_t, SHRPX_QUIC_SALTLEN> salt;
@@ -821,7 +822,7 @@ struct QUICConfig {
StringRef prog_file;
bool disabled;
} bpf;
std::array<uint8_t, SHRPX_QUIC_SERVER_IDLEN> server_id;
uint32_t server_id;
};
struct Http3Config {

51
src/shrpx_connection_handler.cc
View File

@@ -278,15 +278,14 @@ int ConnectionHandler::create_single_worker() {
#endif // ENABLE_HTTP3 && HAVE_LIBBPF
#ifdef ENABLE_HTTP3
assert(cid_prefixes_.size() == 1);
const auto &cid_prefix = cid_prefixes_[0];
assert(worker_ids_.size() == 1);
const auto &wid = worker_ids_[0];
#endif // ENABLE_HTTP3
single_worker_ = std::make_unique<Worker>(
loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
#ifdef ENABLE_HTTP3
quic_sv_ssl_ctx, quic_cert_tree_.get(), cid_prefix.data(),
cid_prefix.size(),
quic_sv_ssl_ctx, quic_cert_tree_.get(), wid,
# ifdef HAVE_LIBBPF
/* index = */ 0,
# endif // HAVE_LIBBPF
@@ -376,21 +375,20 @@ int ConnectionHandler::create_worker_thread(size_t num) {
}
# ifdef ENABLE_HTTP3
assert(cid_prefixes_.size() == num);
assert(worker_ids_.size() == num);
# endif // ENABLE_HTTP3
for (size_t i = 0; i < num; ++i) {
auto loop = ev_loop_new(config->ev_loop_flags);
# ifdef ENABLE_HTTP3
const auto &cid_prefix = cid_prefixes_[i];
const auto &wid = worker_ids_[i];
# endif // ENABLE_HTTP3
auto worker = std::make_unique<Worker>(
loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
# ifdef ENABLE_HTTP3
quic_sv_ssl_ctx, quic_cert_tree_.get(), cid_prefix.data(),
cid_prefix.size(),
quic_sv_ssl_ctx, quic_cert_tree_.get(), wid,
# ifdef HAVE_LIBBPF
i,
# endif // HAVE_LIBBPF
@@ -1008,13 +1006,12 @@ void ConnectionHandler::set_enable_acceptor_on_ocsp_completion(bool f) {
#ifdef ENABLE_HTTP3
int ConnectionHandler::forward_quic_packet(
const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const ngtcp2_pkt_info &pi,
const uint8_t *cid_prefix, const uint8_t *data, size_t datalen) {
const Address &local_addr, const ngtcp2_pkt_info &pi, const WorkerID &wid,
const uint8_t *data, size_t datalen) {
assert(!get_config()->single_thread);
for (auto &worker : workers_) {
if (!std::equal(cid_prefix, cid_prefix + SHRPX_QUIC_CID_PREFIXLEN,
worker->get_cid_prefix())) {
if (wid != worker->get_worker_id()) {
continue;
}
@@ -1041,20 +1038,16 @@ ConnectionHandler::get_quic_keying_materials() const {
return quic_keying_materials_;
}
void ConnectionHandler::set_cid_prefixes(
const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>>
&cid_prefixes) {
cid_prefixes_ = cid_prefixes;
void ConnectionHandler::set_worker_ids(std::vector<WorkerID> worker_ids) {
worker_ids_ = std::move(worker_ids);
}
QUICLingeringWorkerProcess *
ConnectionHandler::match_quic_lingering_worker_process_cid_prefix(
const uint8_t *dcid, size_t dcidlen) {
assert(dcidlen >= SHRPX_QUIC_CID_PREFIXLEN);
ConnectionHandler::match_quic_lingering_worker_process_worker_id(
const WorkerID &wid) {
for (auto &lwps : quic_lingering_worker_processes_) {
for (auto &cid_prefix : lwps.cid_prefixes) {
if (std::equal(std::begin(cid_prefix), std::end(cid_prefix), dcid)) {
for (auto &lwid : lwps.worker_ids) {
if (wid == lwid) {
return &lwps;
}
}
@@ -1275,18 +1268,16 @@ int ConnectionHandler::quic_ipc_read() {
auto &qkm = quic_keying_materials_->keying_materials.front();
std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
ConnectionID decrypted_dcid;
if (decrypt_quic_connection_id(decrypted_dcid.data(),
vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
qkm.cid_encryption_ctx) != 0) {
if (decrypt_quic_connection_id(decrypted_dcid,
vc.dcid + SHRPX_QUIC_CID_WORKER_ID_OFFSET,
qkm.cid_decryption_ctx) != 0) {
return -1;
}
for (auto &worker : workers_) {
if (!std::equal(std::begin(decrypted_dcid),
std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
worker->get_cid_prefix())) {
if (decrypted_dcid.worker != worker->get_worker_id()) {
continue;
}
@@ -1300,7 +1291,7 @@ int ConnectionHandler::quic_ipc_read() {
}
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "No worker to match CID prefix";
LOG(INFO) << "No worker to match Worker ID";
}
return 0;

26
src/shrpx_connection_handler.h
View File

@@ -108,7 +108,7 @@ struct SerialEvent {
struct BPFRef {
bpf_object *obj;
bpf_map *reuseport_array;
bpf_map *cid_prefix_map;
bpf_map *worker_id_map;
};
# endif // HAVE_LIBBPF
@@ -121,12 +121,10 @@ enum class QUICIPCType {
// WorkerProcesses which are in graceful shutdown period.
struct QUICLingeringWorkerProcess {
QUICLingeringWorkerProcess(
std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes,
int quic_ipc_fd)
: cid_prefixes{std::move(cid_prefixes)}, quic_ipc_fd{quic_ipc_fd} {}
QUICLingeringWorkerProcess(std::vector<WorkerID> worker_ids, int quic_ipc_fd)
: worker_ids{std::move(worker_ids)}, quic_ipc_fd{quic_ipc_fd} {}
std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes;
std::vector<WorkerID> worker_ids;
// Socket to send QUIC IPC message to this worker process.
int quic_ipc_fd;
};
@@ -197,25 +195,22 @@ public:
int forward_quic_packet(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const ngtcp2_pkt_info &pi,
const uint8_t *cid_prefix, const uint8_t *data,
const WorkerID &wid, const uint8_t *data,
size_t datalen);
void set_quic_keying_materials(std::shared_ptr<QUICKeyingMaterials> qkms);
const std::shared_ptr<QUICKeyingMaterials> &get_quic_keying_materials() const;
void set_cid_prefixes(
const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>>
&cid_prefixes);
void set_worker_ids(std::vector<WorkerID> worker_ids);
void set_quic_lingering_worker_processes(
const std::vector<QUICLingeringWorkerProcess> &quic_lwps);
// Return matching QUICLingeringWorkerProcess which has a CID prefix
// Return matching QUICLingeringWorkerProcess which has a Worker ID
// such that |dcid| starts with it. If no such
// QUICLingeringWorkerProcess, it returns nullptr.
QUICLingeringWorkerProcess *
match_quic_lingering_worker_process_cid_prefix(const uint8_t *dcid,
size_t dcidlen);
match_quic_lingering_worker_process_worker_id(const WorkerID &wid);
int forward_quic_packet_to_lingering_worker_process(
QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr,
@@ -260,9 +255,8 @@ private:
// and signature algorithm presented by client.
std::vector<std::vector<SSL_CTX *>> indexed_ssl_ctx_;
#ifdef ENABLE_HTTP3
std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes_;
std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>>
lingering_cid_prefixes_;
std::vector<WorkerID> worker_ids_;
std::vector<WorkerID> lingering_worker_ids_;
int quic_ipc_fd_;
std::vector<QUICLingeringWorkerProcess> quic_lingering_worker_processes_;
# ifdef HAVE_LIBBPF

9
src/shrpx_http3_upstream.cc
View File

@@ -211,8 +211,10 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token,
auto &qkms = conn_handler->get_quic_keying_materials();
auto &qkm = qkms->keying_materials.front();
if (generate_quic_connection_id(*cid, cidlen, worker->get_cid_prefix(),
qkm.id, qkm.cid_encryption_ctx) != 0) {
assert(SHRPX_QUIC_SCIDLEN == cidlen);
if (generate_quic_connection_id(*cid, worker->get_worker_id(), qkm.id,
qkm.cid_encryption_ctx) != 0) {
return NGTCP2_ERR_CALLBACK_FAILURE;
}
@@ -609,8 +611,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
ngtcp2_cid scid;
if (generate_quic_connection_id(scid, SHRPX_QUIC_SCIDLEN,
worker->get_cid_prefix(), qkm.id,
if (generate_quic_connection_id(scid, worker->get_worker_id(), qkm.id,
qkm.cid_encryption_ctx) != 0) {
return -1;
}

39
src/shrpx_quic.cc
View File

@@ -173,42 +173,34 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
return 0;
}
int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
const uint8_t *server_id, uint8_t km_id,
EVP_CIPHER_CTX *ctx) {
assert(cidlen == SHRPX_QUIC_SCIDLEN);
if (RAND_bytes(cid.data, cidlen) != 1) {
int generate_quic_retry_connection_id(ngtcp2_cid &cid, uint32_t server_id,
uint8_t km_id, EVP_CIPHER_CTX *ctx) {
if (RAND_bytes(cid.data, SHRPX_QUIC_SCIDLEN) != 1) {
return -1;
}
cid.datalen = cidlen;
cid.datalen = SHRPX_QUIC_SCIDLEN;
cid.data[0] = (cid.data[0] & 0x3f) | km_id;
auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET;
auto p = cid.data + SHRPX_QUIC_CID_WORKER_ID_OFFSET;
std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p);
std::copy_n(reinterpret_cast<uint8_t *>(&server_id), sizeof(server_id), p);
return encrypt_quic_connection_id(p, p, ctx);
}
int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
const uint8_t *cid_prefix, uint8_t km_id,
EVP_CIPHER_CTX *ctx) {
assert(cidlen == SHRPX_QUIC_SCIDLEN);
if (RAND_bytes(cid.data, cidlen) != 1) {
int generate_quic_connection_id(ngtcp2_cid &cid, const WorkerID &wid,
uint8_t km_id, EVP_CIPHER_CTX *ctx) {
if (RAND_bytes(cid.data, SHRPX_QUIC_SCIDLEN) != 1) {
return -1;
}
cid.datalen = cidlen;
cid.datalen = SHRPX_QUIC_SCIDLEN;
cid.data[0] = (cid.data[0] & 0x3f) | km_id;
auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET;
auto p = cid.data + SHRPX_QUIC_CID_WORKER_ID_OFFSET;
std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p);
std::copy_n(reinterpret_cast<const uint8_t *>(&wid), sizeof(wid), p);
return encrypt_quic_connection_id(p, p, ctx);
}
@@ -225,12 +217,13 @@ int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,
return 0;
}
int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,
int decrypt_quic_connection_id(ConnectionID &dest, const uint8_t *src,
EVP_CIPHER_CTX *ctx) {
int len;
auto p = reinterpret_cast<uint8_t *>(&dest);
if (!EVP_EncryptUpdate(ctx, dest, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) ||
!EVP_EncryptFinal_ex(ctx, dest + len, &len)) {
if (!EVP_DecryptUpdate(ctx, p, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) ||
!EVP_DecryptFinal_ex(ctx, p + len, &len)) {
return -1;
}

52
src/shrpx_quic.h
View File

@@ -65,12 +65,16 @@ struct UpstreamAddr;
struct QUICKeyingMaterials;
struct QUICKeyingMaterial;
constexpr size_t SHRPX_QUIC_SCIDLEN = 20;
constexpr size_t SHRPX_QUIC_CID_WORKER_ID_OFFSET = 1;
constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 4;
// SHRPX_QUIC_CID_PREFIXLEN includes SHRPX_QUIC_SERVER_IDLEN.
constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8;
constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1;
constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16;
constexpr size_t SHRPX_QUIC_SOCK_IDLEN = 4;
constexpr size_t SHRPX_QUIC_WORKER_IDLEN =
SHRPX_QUIC_SERVER_IDLEN + SHRPX_QUIC_SOCK_IDLEN;
constexpr size_t SHRPX_QUIC_CLIENT_IDLEN = 8;
constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN =
SHRPX_QUIC_WORKER_IDLEN + SHRPX_QUIC_CLIENT_IDLEN;
constexpr size_t SHRPX_QUIC_SCIDLEN =
SHRPX_QUIC_CID_WORKER_ID_OFFSET + SHRPX_QUIC_DECRYPTED_DCIDLEN;
constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16;
constexpr size_t SHRPX_QUIC_CONN_CLOSE_PKTLEN = 256;
constexpr size_t SHRPX_QUIC_STATELESS_RESET_BURST = 100;
@@ -79,6 +83,32 @@ constexpr size_t SHRPX_QUIC_SECRETLEN = 32;
constexpr size_t SHRPX_QUIC_SALTLEN = 32;
constexpr uint8_t SHRPX_QUIC_DCID_KM_ID_MASK = 0xc0;
struct WorkerID {
union {
struct {
uint32_t server;
uint32_t thread;
};
uint64_t worker;
};
};
static_assert(sizeof(WorkerID) == SHRPX_QUIC_WORKER_IDLEN,
"WorkerID length assertion failure");
inline bool operator==(const WorkerID &lhd, const WorkerID &rhd) {
return lhd.worker == rhd.worker;
}
inline bool operator!=(const WorkerID &lhd, const WorkerID &rhd) {
return lhd.worker != rhd.worker;
}
struct ConnectionID {
WorkerID worker;
uint64_t client;
};
ngtcp2_tstamp quic_timestamp();
int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
@@ -86,18 +116,16 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
size_t local_salen, const ngtcp2_pkt_info &pi,
const uint8_t *data, size_t datalen, size_t gso_size);
int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
const uint8_t *server_id, uint8_t km_id,
EVP_CIPHER_CTX *ctx);
int generate_quic_retry_connection_id(ngtcp2_cid &cid, uint32_t server_id,
uint8_t km_id, EVP_CIPHER_CTX *ctx);
int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
const uint8_t *cid_prefix, uint8_t km_id,
EVP_CIPHER_CTX *ctx);
int generate_quic_connection_id(ngtcp2_cid &cid, const WorkerID &wid,
uint8_t km_id, EVP_CIPHER_CTX *ctx);
int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,
EVP_CIPHER_CTX *ctx);
int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,
int decrypt_quic_connection_id(ConnectionID &dest, const uint8_t *src,
EVP_CIPHER_CTX *ctx);
int generate_quic_hashed_connection_id(ngtcp2_cid &dest,

41
src/shrpx_quic_connection_handler.cc
View File

@@ -123,7 +123,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
}
if (it == std::end(connections_)) {
std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
ConnectionID decrypted_dcid;
auto &qkms = conn_handler->get_quic_keying_materials();
const QUICKeyingMaterial *qkm = nullptr;
@@ -132,19 +132,17 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
qkm = select_quic_keying_material(
*qkms.get(), vc.dcid[0] & SHRPX_QUIC_DCID_KM_ID_MASK);
if (decrypt_quic_connection_id(decrypted_dcid.data(),
vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
qkm->cid_encryption_ctx) != 0) {
if (decrypt_quic_connection_id(decrypted_dcid,
vc.dcid + SHRPX_QUIC_CID_WORKER_ID_OFFSET,
qkm->cid_decryption_ctx) != 0) {
return 0;
}
if (qkm != &qkms->keying_materials.front() ||
!std::equal(std::begin(decrypted_dcid),
std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
worker_->get_cid_prefix())) {
decrypted_dcid.worker != worker_->get_worker_id()) {
auto quic_lwp =
conn_handler->match_quic_lingering_worker_process_cid_prefix(
decrypted_dcid.data(), decrypted_dcid.size());
conn_handler->match_quic_lingering_worker_process_worker_id(
decrypted_dcid.worker);
if (quic_lwp) {
if (conn_handler->forward_quic_packet_to_lingering_worker_process(
quic_lwp, remote_addr, local_addr, pi, data, datalen) == 0) {
@@ -177,23 +175,21 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
switch (ngtcp2_accept(&hd, data, datalen)) {
case 0: {
// If we get Initial and it has the CID prefix of this worker,
// it is likely that client is intentionally use the prefix.
// Just drop it.
// If we get Initial and it has the Worker ID of this worker, it
// is likely that client is intentionally use the prefix. Just
// drop it.
if (vc.dcidlen == SHRPX_QUIC_SCIDLEN) {
if (qkm != &qkms->keying_materials.front()) {
qkm = &qkms->keying_materials.front();
if (decrypt_quic_connection_id(decrypted_dcid.data(),
vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
qkm->cid_encryption_ctx) != 0) {
if (decrypt_quic_connection_id(
decrypted_dcid, vc.dcid + SHRPX_QUIC_CID_WORKER_ID_OFFSET,
qkm->cid_decryption_ctx) != 0) {
return 0;
}
}
if (std::equal(std::begin(decrypted_dcid),
std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
worker_->get_cid_prefix())) {
if (decrypted_dcid.worker == worker_->get_worker_id()) {
return 0;
}
}
@@ -325,12 +321,10 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
}
default:
if (!(data[0] & 0x80) && vc.dcidlen == SHRPX_QUIC_SCIDLEN &&
!std::equal(std::begin(decrypted_dcid),
std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
worker_->get_cid_prefix())) {
decrypted_dcid.worker != worker_->get_worker_id()) {
if (!config->single_thread &&
conn_handler->forward_quic_packet(faddr, remote_addr, local_addr,
pi, decrypted_dcid.data(), data,
pi, decrypted_dcid.worker, data,
datalen) == 0) {
return 0;
}
@@ -477,8 +471,7 @@ int QUICConnectionHandler::send_retry(
ngtcp2_cid retry_scid;
if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN,
quicconf.server_id.data(), qkm.id,
if (generate_quic_retry_connection_id(retry_scid, quicconf.server_id, qkm.id,
qkm.cid_encryption_ctx) != 0) {
return -1;
}

28
src/shrpx_worker.cc
View File

@@ -148,7 +148,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
tls::CertLookupTree *cert_tree,
#ifdef ENABLE_HTTP3
SSL_CTX *quic_sv_ssl_ctx, tls::CertLookupTree *quic_cert_tree,
const uint8_t *cid_prefix, size_t cid_prefixlen,
WorkerID wid,
# ifdef HAVE_LIBBPF
size_t index,
# endif // HAVE_LIBBPF
@@ -164,6 +164,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
worker_stat_{},
dns_tracker_(loop, get_config()->conn.downstream->family),
#ifdef ENABLE_HTTP3
worker_id_{std::move(wid)},
quic_upstream_addrs_{get_config()->conn.quic_listener.addrs},
#endif // ENABLE_HTTP3
loop_(loop),
@@ -180,10 +181,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
connect_blocker_(
std::make_unique<ConnectBlocker>(randgen_, loop_, nullptr, nullptr)),
graceful_shutdown_(false) {
#ifdef ENABLE_HTTP3
std::copy_n(cid_prefix, cid_prefixlen, std::begin(cid_prefix_));
#endif // ENABLE_HTTP3
ev_async_init(&w_, eventcb);
w_.data = this;
ev_async_start(loop_, &w_);
@@ -1071,10 +1068,10 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
return -1;
}
ref.cid_prefix_map = bpf_object__find_map_by_name(obj, "cid_prefix_map");
if (!ref.cid_prefix_map) {
ref.worker_id_map = bpf_object__find_map_by_name(obj, "worker_id_map");
if (!ref.worker_id_map) {
auto error = errno;
LOG(FATAL) << "Failed to get cid_prefix_map: "
LOG(FATAL) << "Failed to get worker_id_map: "
<< xsi_strerror(error, errbuf.data(), errbuf.size());
close(fd);
return -1;
@@ -1155,12 +1152,12 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
return -1;
}
rv = bpf_map__update_elem(ref.cid_prefix_map, cid_prefix_.data(),
cid_prefix_.size(), &sk_index, sizeof(sk_index),
rv = bpf_map__update_elem(ref.worker_id_map, &worker_id_,
sizeof(worker_id_), &sk_index, sizeof(sk_index),
BPF_NOEXIST);
if (rv != 0) {
auto error = errno;
LOG(FATAL) << "Failed to update cid_prefix_map: "
LOG(FATAL) << "Failed to update worker_id_map: "
<< xsi_strerror(error, errbuf.data(), errbuf.size());
close(fd);
return -1;
@@ -1187,7 +1184,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
return 0;
}
const uint8_t *Worker::get_cid_prefix() const { return cid_prefix_.data(); }
const WorkerID &Worker::get_worker_id() const { return worker_id_; }
const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) {
std::array<char, NI_MAXHOST> host;
@@ -1445,10 +1442,11 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr) {
}
#ifdef ENABLE_HTTP3
int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id) {
auto p = std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, cid_prefix);
int create_worker_id(WorkerID &dest, uint32_t server_id) {
dest.server = server_id;
if (RAND_bytes(p, SHRPX_QUIC_CID_PREFIXLEN - SHRPX_QUIC_SERVER_IDLEN) != 1) {
if (RAND_bytes(reinterpret_cast<unsigned char *>(&dest.thread),
sizeof(dest.thread)) != 1) {
return -1;
}

13
src/shrpx_worker.h
View File

@@ -312,7 +312,7 @@ public:
tls::CertLookupTree *cert_tree,
#ifdef ENABLE_HTTP3
SSL_CTX *quic_sv_ssl_ctx, tls::CertLookupTree *quic_cert_tree,
const uint8_t *cid_prefix, size_t cid_prefixlen,
WorkerID wid,
# ifdef HAVE_LIBBPF
size_t index,
# endif // HAVE_LIBBPF
@@ -377,7 +377,7 @@ public:
int setup_quic_server_socket();
const uint8_t *get_cid_prefix() const;
const WorkerID &get_worker_id() const;
# ifdef HAVE_LIBBPF
bool should_attach_bpf() const;
@@ -414,7 +414,7 @@ private:
DNSTracker dns_tracker_;
#ifdef ENABLE_HTTP3
std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN> cid_prefix_;
WorkerID worker_id_;
std::vector<UpstreamAddr> quic_upstream_addrs_;
std::vector<std::unique_ptr<QUICListener>> quic_listeners_;
#endif // ENABLE_HTTP3
@@ -469,10 +469,9 @@ size_t match_downstream_addr_group(
void downstream_failure(DownstreamAddr *addr, const Address *raddr);
#ifdef ENABLE_HTTP3
// Creates unpredictable SHRPX_QUIC_CID_PREFIXLEN bytes sequence which
// is used as a prefix of QUIC Connection ID. This function returns
// -1 on failure. |server_id| must be 2 bytes long.
int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id);
// Creates WorkerID used as a prefix of QUIC Connection ID. This
// function returns -1 on failure.
int create_worker_id(WorkerID &dest, uint32_t server_id);
#endif // ENABLE_HTTP3
} // namespace shrpx

12
src/shrpx_worker_process.cc
View File

@@ -593,11 +593,21 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
}
EVP_CIPHER_CTX_set_padding(qkm.cid_encryption_ctx, 0);
qkm.cid_decryption_ctx = EVP_CIPHER_CTX_new();
if (!EVP_DecryptInit_ex(qkm.cid_decryption_ctx, EVP_aes_128_ecb(), nullptr,
qkm.cid_encryption_key.data(), nullptr)) {
LOG(ERROR)
<< "Failed to initialize QUIC Connection ID decryption context";
return -1;
}
EVP_CIPHER_CTX_set_padding(qkm.cid_decryption_ctx, 0);
}
conn_handler->set_quic_keying_materials(std::move(qkms));
conn_handler->set_cid_prefixes(wpconf->cid_prefixes);
conn_handler->set_worker_ids(wpconf->worker_ids);
conn_handler->set_quic_lingering_worker_processes(
wpconf->quic_lingering_worker_processes);
#endif // ENABLE_HTTP3

4
src/shrpx_worker_process.h
View File

@@ -49,8 +49,8 @@ struct WorkerProcessConfig {
// IPv6 socket, or -1 if not used
int server_fd6;
#ifdef ENABLE_HTTP3
// CID prefixes for the new worker process.
std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes;
// Worker IDs for the new worker process.
std::vector<WorkerID> worker_ids;
// IPC socket to read forwarded QUIC UDP datagram from the current
// worker process.
int quic_ipc_fd;