Download raw body.
improve gotwebd accept behaviour under load
The gotwebd sockets process no longer needs 6 file descriptors per open connection. I'm not even sure where this number came from. httpd's limit is higher than this, which means it will try to queue more requests than gotwebd will accept and start returning status 500 when it can no longer connect to gotwebd. At some point I tweaked the accept timout to retry exactly once per second, by adding a check for an already pending timeout. To match httpd's behaviour, reschedule the timeout for an entire second each time we hit EMFILE. Not sure if this is very important but it is nice to be consistent. Testing this diff while got.g.o is being hammered with requests I see gotwebd handle requests slowly but gracefully under load. without the diff it quickly falls into an endless EMFILE accept loop which results in httpd returning status 500 imediately. I don't think this will be a 100% solution since our request timeout is quite long (120 seconds), giving httpd plenty of time to queue up too many requests. Unfortunately some requests can really take a long time (blaming huge files with deep history) so I do not want to lower this timeout. In the future, perhaps we should make this timeout configurable and/or adjust it based on the nature of the request. Also drop the separate accept timeout handler. It is redundant since it does the same as the EV_TIMEOUT case at the top of the regular handler. ok? M gotwebd/gotwebd.h | 0+ 1- M gotwebd/sockets.c | 3+ 14- 2 files changed, 3 insertions(+), 15 deletions(-) commit - 12c1bbcab3809ec364d34a8280dfb318a2968da6 commit + 8d39e68c89e9b071c7add3674a580b99e065e76d blob - 58211c1296e9a12d3d7be515900874e2cfc64c6c blob + 53ad046c22bd7e8a385afcc21d0390211c2fca07 --- gotwebd/gotwebd.h +++ gotwebd/gotwebd.h @@ -94,7 +94,6 @@ (((n) + (FCGI_ALIGNMENT - 1)) & ~(FCGI_ALIGNMENT - 1)) #define FD_RESERVE 5 -#define FD_NEEDED 6 #define FCGI_BEGIN_REQUEST 1 #define FCGI_ABORT_REQUEST 2 blob - ab9b70f72224ae1422ba89b2c55c42cd3bcd29fb blob + ac432c97b2feeadb639545d4fa5bd3ea8c66af67 --- gotwebd/sockets.c +++ gotwebd/sockets.c @@ -69,7 +69,6 @@ static struct timeval timeout = { TIMEOUT_DEFAULT, 0 } static void sockets_sighdlr(int, short, void *); static void sockets_shutdown(void); static void sockets_launch(struct gotwebd *); -static void sockets_accept_paused(int, short, void *); static void sockets_dispatch_main(int, short, void *); static int sockets_unix_socket_listen(struct gotwebd *, struct socket *, uid_t, gid_t); @@ -359,7 +358,7 @@ sockets_launch(struct gotwebd *env) if (event_add(&sock->ev, NULL)) fatalx("event add sock"); - evtimer_set(&sock->pause, sockets_accept_paused, sock); + evtimer_set(&sock->pause, sockets_socket_accept, sock); log_info("%s: running socket listener %d", __func__, sock->conf.id); @@ -986,8 +985,7 @@ sockets_accept_reserve(int sockfd, struct sockaddr *ad { int ret; - if (getdtablecount() + reserve + - ((*counter + 1) * FD_NEEDED) >= getdtablesize()) { + if (getdtablecount() + reserve + *counter + 1 >= getdtablesize()) { log_warnx("inflight fds exceeded"); errno = EMFILE; return -1; @@ -1002,14 +1000,6 @@ sockets_accept_reserve(int sockfd, struct sockaddr *ad return ret; } -static void -sockets_accept_paused(int fd, short events, void *arg) -{ - struct socket *sock = (struct socket *)arg; - - event_add(&sock->ev, NULL); -} - static int parse_params(struct request *c, uint8_t *record, size_t record_len) { @@ -1196,8 +1186,7 @@ sockets_socket_accept(int fd, short event, void *arg) case ENFILE: log_warn("accept"); event_del(&sock->ev); - if (!evtimer_pending(&sock->pause, NULL)) - evtimer_add(&sock->pause, &backoff); + evtimer_add(&sock->pause, &backoff); return; default: log_warn("%s: accept", __func__);
improve gotwebd accept behaviour under load