You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
226 lines
10 KiB
226 lines
10 KiB
https://github.com/h2o/h2o/pull/3293
|
|
|
|
From 770208bbe3955c47e005a1e8cb08266e4a8dfc9a Mon Sep 17 00:00:00 2001
|
|
From: Remi Gacogne <remi.gacogne@powerdns.com>
|
|
Date: Tue, 10 Oct 2023 15:47:57 +0200
|
|
Subject: [PATCH] [http2] delay processing requests upon observing suspicious
|
|
behavior
|
|
|
|
Backport of 94fbc54b6c9309912fe3d53e7b63408bbe9a1b0d to v2.2.x
|
|
---
|
|
include/h2o.h | 8 +++++++
|
|
include/h2o/http2_internal.h | 8 +++++++
|
|
lib/core/config.c | 1 +
|
|
lib/core/configurator.c | 9 ++++++++
|
|
lib/core/context.c | 2 ++
|
|
lib/http2/connection.c | 41 ++++++++++++++++++++++++++++++++----
|
|
6 files changed, 65 insertions(+), 4 deletions(-)
|
|
|
|
diff --git a/include/h2o.h b/include/h2o.h
|
|
index 57877bd12c..409cd5c21c 100644
|
|
--- a/include/h2o.h
|
|
+++ b/include/h2o.h
|
|
@@ -378,6 +378,10 @@ struct st_h2o_globalconf_t {
|
|
* list of callbacks
|
|
*/
|
|
h2o_protocol_callbacks_t callbacks;
|
|
+ /**
|
|
+ * milliseconds to delay processing requests when suspicious behavior is detected
|
|
+ */
|
|
+ uint64_t dos_delay;
|
|
} http2;
|
|
|
|
struct {
|
|
@@ -590,6 +594,10 @@ struct st_h2o_context_t {
|
|
* timeout entry used for graceful shutdown
|
|
*/
|
|
h2o_timeout_entry_t _graceful_shutdown_timeout;
|
|
+ /*
|
|
+ * dos timeout
|
|
+ */
|
|
+ h2o_timeout_t dos_delay_timeout;
|
|
struct {
|
|
/**
|
|
* counter for http2 errors internally emitted by h2o
|
|
diff --git a/include/h2o/http2_internal.h b/include/h2o/http2_internal.h
|
|
index 5cfc4d8204..b9cf400929 100644
|
|
--- a/include/h2o/http2_internal.h
|
|
+++ b/include/h2o/http2_internal.h
|
|
@@ -179,6 +179,7 @@ struct st_h2o_http2_stream_t {
|
|
h2o_linklist_t link;
|
|
h2o_http2_scheduler_openref_t scheduler;
|
|
} _refs;
|
|
+ unsigned reset_by_peer : 1;
|
|
h2o_send_state_t send_state; /* state of the ostream, only used in push mode */
|
|
/* placed at last since it is large and has it's own ctor */
|
|
h2o_req_t req;
|
|
@@ -232,6 +233,13 @@ struct st_h2o_http2_conn_t {
|
|
} _write;
|
|
h2o_cache_t *push_memo;
|
|
h2o_http2_casper_t *casper;
|
|
+ /**
|
|
+ * DoS mitigation; the idea here is to delay processing requests when observing suspicious behavior
|
|
+ */
|
|
+ struct {
|
|
+ h2o_timeout_entry_t process_delay;
|
|
+ size_t reset_budget; /* RST_STREAM frames are considered suspicious when this value goes down to zero */
|
|
+ } dos_mitigation;
|
|
};
|
|
|
|
int h2o_http2_update_peer_settings(h2o_http2_settings_t *settings, const uint8_t *src, size_t len, const char **err_desc);
|
|
diff --git a/lib/core/config.c b/lib/core/config.c
|
|
index ce1d320183..08e43a6d30 100644
|
|
--- a/lib/core/config.c
|
|
+++ b/lib/core/config.c
|
|
@@ -189,6 +189,7 @@ void h2o_config_init(h2o_globalconf_t *config)
|
|
config->http2.latency_optimization.min_rtt = 50; // milliseconds
|
|
config->http2.latency_optimization.max_additional_delay = 10;
|
|
config->http2.latency_optimization.max_cwnd = 65535;
|
|
+ config->http2.dos_delay = 100; /* 100ms processing delay when observing suspicious behavior */
|
|
config->http2.callbacks = H2O_HTTP2_CALLBACKS;
|
|
config->mimemap = h2o_mimemap_create();
|
|
|
|
diff --git a/lib/core/configurator.c b/lib/core/configurator.c
|
|
index 891770cc2d..4731ba2707 100644
|
|
--- a/lib/core/configurator.c
|
|
+++ b/lib/core/configurator.c
|
|
@@ -531,6 +531,12 @@ static int on_config_http2_casper(h2o_configurator_command_t *cmd, h2o_configura
|
|
return 0;
|
|
}
|
|
|
|
+
|
|
+static int on_config_http2_dos_delay(h2o_configurator_command_t *cmd, h2o_configurator_context_t *ctx, yoml_t *node)
|
|
+{
|
|
+ return config_timeout(cmd, node, &ctx->globalconf->http2.dos_delay);
|
|
+}
|
|
+
|
|
static int assert_is_mimetype(h2o_configurator_command_t *cmd, yoml_t *node)
|
|
{
|
|
if (node->type != YOML_TYPE_SCALAR) {
|
|
@@ -910,6 +916,9 @@ void h2o_configurator__init_core(h2o_globalconf_t *conf)
|
|
on_config_http2_push_preload);
|
|
h2o_configurator_define_command(&c->super, "http2-casper", H2O_CONFIGURATOR_FLAG_GLOBAL | H2O_CONFIGURATOR_FLAG_HOST,
|
|
on_config_http2_casper);
|
|
+ h2o_configurator_define_command(&c->super, "http2-dos-delay",
|
|
+ H2O_CONFIGURATOR_FLAG_GLOBAL | H2O_CONFIGURATOR_FLAG_EXPECT_SCALAR,
|
|
+ on_config_http2_dos_delay);
|
|
h2o_configurator_define_command(&c->super, "file.mime.settypes",
|
|
(H2O_CONFIGURATOR_FLAG_ALL_LEVELS & ~H2O_CONFIGURATOR_FLAG_EXTENSION) |
|
|
H2O_CONFIGURATOR_FLAG_EXPECT_MAPPING,
|
|
diff --git a/lib/core/context.c b/lib/core/context.c
|
|
index 8d11013810..ac4b0aaf08 100644
|
|
--- a/lib/core/context.c
|
|
+++ b/lib/core/context.c
|
|
@@ -101,6 +101,7 @@ void h2o_context_init(h2o_context_t *ctx, h2o_loop_t *loop, h2o_globalconf_t *co
|
|
h2o_linklist_init_anchor(&ctx->http1._conns);
|
|
h2o_timeout_init(ctx->loop, &ctx->http2.idle_timeout, config->http2.idle_timeout);
|
|
h2o_timeout_init(ctx->loop, &ctx->http2.graceful_shutdown_timeout, config->http2.graceful_shutdown_timeout);
|
|
+ h2o_timeout_init(ctx->loop, &ctx->http2.dos_delay_timeout, config->http2.dos_delay);
|
|
h2o_linklist_init_anchor(&ctx->http2._conns);
|
|
ctx->proxy.client_ctx.loop = loop;
|
|
h2o_timeout_init(ctx->loop, &ctx->proxy.io_timeout, config->proxy.io_timeout);
|
|
@@ -146,6 +147,7 @@ void h2o_context_dispose(h2o_context_t *ctx)
|
|
h2o_timeout_dispose(ctx->loop, &ctx->http1.req_timeout);
|
|
h2o_timeout_dispose(ctx->loop, &ctx->http2.idle_timeout);
|
|
h2o_timeout_dispose(ctx->loop, &ctx->http2.graceful_shutdown_timeout);
|
|
+ h2o_timeout_dispose(ctx->loop, &ctx->http2.dos_delay_timeout);
|
|
h2o_timeout_dispose(ctx->loop, &ctx->proxy.io_timeout);
|
|
/* what should we do here? assert(!h2o_linklist_is_empty(&ctx->http2._conns); */
|
|
|
|
diff --git a/lib/http2/connection.c b/lib/http2/connection.c
|
|
index e2da293043..4910e33098 100644
|
|
--- a/lib/http2/connection.c
|
|
+++ b/lib/http2/connection.c
|
|
@@ -161,7 +161,6 @@ static void update_idle_timeout(h2o_http2_conn_t *conn)
|
|
h2o_timeout_unlink(&conn->_timeout_entry);
|
|
|
|
if (conn->num_streams.pull.half_closed + conn->num_streams.push.half_closed == 0) {
|
|
- assert(h2o_linklist_is_empty(&conn->_pending_reqs));
|
|
conn->_timeout_entry.cb = on_idle_timeout;
|
|
h2o_timeout_link(conn->super.ctx->loop, &conn->super.ctx->http2.idle_timeout, &conn->_timeout_entry);
|
|
}
|
|
@@ -175,6 +174,9 @@ static int can_run_requests(h2o_http2_conn_t *conn)
|
|
|
|
static void run_pending_requests(h2o_http2_conn_t *conn)
|
|
{
|
|
+ if (h2o_timeout_is_linked(&conn->dos_mitigation.process_delay))
|
|
+ return;
|
|
+
|
|
while (!h2o_linklist_is_empty(&conn->_pending_reqs) && can_run_requests(conn)) {
|
|
/* fetch and detach a pending stream */
|
|
h2o_http2_stream_t *stream = H2O_STRUCT_FROM_MEMBER(h2o_http2_stream_t, _refs.link, conn->_pending_reqs.next);
|
|
@@ -226,6 +228,16 @@ void h2o_http2_conn_unregister_stream(h2o_http2_conn_t *conn, h2o_http2_stream_t
|
|
assert(h2o_http2_scheduler_is_open(&stream->_refs.scheduler));
|
|
h2o_http2_scheduler_close(&stream->_refs.scheduler);
|
|
|
|
+ /* Decrement reset_budget if the stream was reset by peer, otherwise increment. By doing so, we penalize connections that
|
|
+ * generate resets for >50% of requests. */
|
|
+ if (stream->reset_by_peer) {
|
|
+ if (conn->dos_mitigation.reset_budget > 0)
|
|
+ --conn->dos_mitigation.reset_budget;
|
|
+ } else {
|
|
+ if (conn->dos_mitigation.reset_budget < conn->super.ctx->globalconf->http2.max_concurrent_requests_per_connection)
|
|
+ ++conn->dos_mitigation.reset_budget;
|
|
+ }
|
|
+
|
|
switch (stream->state) {
|
|
case H2O_HTTP2_STREAM_STATE_IDLE:
|
|
case H2O_HTTP2_STREAM_STATE_RECV_HEADERS:
|
|
@@ -272,6 +284,8 @@ void close_connection_now(h2o_http2_conn_t *conn)
|
|
h2o_hpack_dispose_header_table(&conn->_output_header_table);
|
|
assert(h2o_linklist_is_empty(&conn->_pending_reqs));
|
|
h2o_timeout_unlink(&conn->_timeout_entry);
|
|
+ if (h2o_timeout_is_linked(&conn->dos_mitigation.process_delay))
|
|
+ h2o_timeout_unlink(&conn->dos_mitigation.process_delay);
|
|
h2o_buffer_dispose(&conn->_write.buf);
|
|
if (conn->_write.buf_in_flight != NULL)
|
|
h2o_buffer_dispose(&conn->_write.buf_in_flight);
|
|
@@ -797,11 +811,19 @@ static int handle_rst_stream_frame(h2o_http2_conn_t *conn, h2o_http2_frame_t *fr
|
|
return H2O_HTTP2_ERROR_PROTOCOL;
|
|
}
|
|
|
|
- stream = h2o_http2_conn_get_stream(conn, frame->stream_id);
|
|
- if (stream != NULL) {
|
|
+ if ((stream = h2o_http2_conn_get_stream(conn, frame->stream_id)) == NULL)
|
|
+ return 0;
|
|
+
|
|
/* reset the stream */
|
|
+ stream->reset_by_peer = 1;
|
|
h2o_http2_stream_reset(conn, stream);
|
|
- }
|
|
+
|
|
+ /* setup process delay if we've just ran out of reset budget */
|
|
+ if (conn->dos_mitigation.reset_budget == 0 && conn->super.ctx->globalconf->http2.dos_delay != 0 &&
|
|
+ !h2o_timeout_is_linked(&conn->dos_mitigation.process_delay))
|
|
+ h2o_timeout_link(conn->super.ctx->loop, &conn->super.ctx->http2.dos_delay_timeout,
|
|
+ &conn->dos_mitigation.process_delay);
|
|
+
|
|
/* TODO log */
|
|
|
|
return 0;
|
|
@@ -1204,6 +1226,14 @@ static h2o_iovec_t log_priority_actual_weight(h2o_req_t *req)
|
|
return h2o_iovec_init(s, len);
|
|
}
|
|
|
|
+static void on_dos_process_delay(h2o_timeout_entry_t *timer)
|
|
+{
|
|
+ h2o_http2_conn_t *conn = H2O_STRUCT_FROM_MEMBER(h2o_http2_conn_t, dos_mitigation.process_delay, timer);
|
|
+
|
|
+ assert(!h2o_timeout_is_linked(&conn->dos_mitigation.process_delay));
|
|
+ run_pending_requests(conn);
|
|
+}
|
|
+
|
|
static h2o_http2_conn_t *create_conn(h2o_context_t *ctx, h2o_hostconf_t **hosts, h2o_socket_t *sock, struct timeval connected_at)
|
|
{
|
|
static const h2o_conn_callbacks_t callbacks = {
|
|
@@ -1240,6 +1270,9 @@ static h2o_http2_conn_t *create_conn(h2o_context_t *ctx, h2o_hostconf_t **hosts,
|
|
conn->_write.timeout_entry.cb = emit_writereq;
|
|
h2o_http2_window_init(&conn->_write.window, &conn->peer_settings);
|
|
|
|
+ conn->dos_mitigation.process_delay.cb = on_dos_process_delay;
|
|
+ conn->dos_mitigation.reset_budget = conn->super.ctx->globalconf->http2.max_concurrent_requests_per_connection;
|
|
+
|
|
return conn;
|
|
}
|
|
|