Improve gotweb.c worker selection.

Instead of distributing requests across workers in a round-robin fashion
keep track of how many requests a given worker has queued and send new
requests to the worker which has the least amount of work queued for it.

This makes gotwebd much more responsive than stupid round-robin, even
though the new strategy is still not optimal because different types of
requests carry different cost. E.g. blaming a file is usually heavier than
listing trees. But it might be good enough, for now. Let's try and see.

This applies on top of the "run just one gotwebd process which listens"
patch I sent earlier.


M  gotwebd/gotwebd.c  |   7+  0-
M  gotwebd/gotwebd.h  |   2+  0-
M  gotwebd/sockets.c  |  36+  5-

3 files changed, 45 insertions(+), 5 deletions(-)

commit - 968aa2bf0321618be8cfe7a56f986fc176fd3736
commit + 6b69d5ce1682c758e4bb6b27b780b094d35e156f
blob - ecc7fd3254c6a09eb4b56485c161763b345e14a1
blob + 17338707c2666ca04c1d07ca753f94cd50762191
--- gotwebd/gotwebd.c
+++ gotwebd/gotwebd.c
@@ -500,6 +500,13 @@ main(int argc, char **argv)
 			www_username = env->www_user;
 	}
 
+	if (proc_type == GOTWEBD_PROC_SERVER) {
+		env->worker_load = calloc(env->prefork,
+		    sizeof(env->worker_load[0]));
+		if (env->worker_load == NULL)
+			fatal("calloc");
+	}
+
 	pw = getpwnam(www_username);
 	if (pw == NULL)
 		fatalx("unknown user %s", www_username);
blob - 0719d555453958214b9038725b19d44c068a5c05
blob + 02bc544637cc043a756a9a812d12548947cd9442
--- gotwebd/gotwebd.h
+++ gotwebd/gotwebd.h
@@ -273,6 +273,7 @@ struct request {
 	struct event			*resp_event;
 	int				 sock_id;
 	uint32_t			 request_id;
+	int				 worker_idx;
 
 	uint8_t				 *buf;
 	size_t				 buf_len;
@@ -391,6 +392,7 @@ struct gotwebd {
 	uint16_t	 prefork;
 	int		 gotweb_pending;
 	int		 gotweb_cur;
+	int		 *worker_load;
 
 	int		 server_cnt;
 
blob - f2cc7a4eb8d7c88ef03073b64897e2a73685cc29
blob + 0c92e34a59abbd18afc609df8ab37edb345a7a71
--- gotwebd/sockets.c
+++ gotwebd/sockets.c
@@ -134,8 +134,16 @@ find_request(uint32_t request_id)
 static void
 cleanup_request(struct request *c)
 {
+	struct gotwebd *env = gotwebd_env;
+
 	cgi_inflight--;
 
+	if (c->worker_idx != -1) {
+		if (env->worker_load[c->worker_idx] <= 0)
+			fatalx("request in flight on worker with zero load");
+		env->worker_load[c->worker_idx]--;
+	}
+
 	del_request(c);
 
 	event_add(&c->sock->ev, NULL);
@@ -468,6 +476,29 @@ recv_gotweb_pipe(struct gotwebd *env, struct imsg *ims
 	env->gotweb_pending--;
 }
 
+static struct imsgev *
+select_worker(struct request *c)
+{
+	struct gotwebd *env = gotwebd_env;
+	int i, least_busy_worker_idx, min_load;
+
+	min_load = env->worker_load[0];
+	least_busy_worker_idx = 0;
+	for (i = 1; i < env->prefork; i++) {
+		if (env->worker_load[i] > min_load) 
+			continue;
+
+		min_load = env->worker_load[i];
+		least_busy_worker_idx = i;
+	}
+
+	log_debug("dispatching request %u to gotweb %d",
+	    c->request_id, least_busy_worker_idx);
+
+	c->worker_idx = least_busy_worker_idx;
+	return &env->iev_gotweb[least_busy_worker_idx];
+}
+
 static int
 process_request(struct request *c)
 {
@@ -491,19 +522,18 @@ process_request(struct request *c)
 		ic.priv_fd[i] = -1;
 	ic.fd = -1;
 
-	/* Round-robin requests across gotweb processes. */
-	iev_gotweb = &env->iev_gotweb[env->gotweb_cur];
-	env->gotweb_cur = (env->gotweb_cur + 1) % env->prefork;
-
+	iev_gotweb = select_worker(c);
 	ret = imsg_compose_event(iev_gotweb, GOTWEBD_IMSG_REQ_PROCESS,
 	    GOTWEBD_PROC_SERVER, -1, c->fd, &ic, sizeof(ic));
 	if (ret == -1) {
 		log_warn("imsg_compose_event");
+		c->worker_idx = -1;
 		return -1;
 	}
 
 	c->fd = -1;
 	c->client_status = CLIENT_REQUEST;
+	env->worker_load[c->worker_idx]++;
 	return 0;
 }
 
@@ -782,7 +812,7 @@ sockets_shutdown(void)
 	for (i = 0; i < gotwebd_env->prefork; i++)
 		imsgbuf_clear(&gotwebd_env->iev_gotweb[i].ibuf);
 	free(gotwebd_env->iev_gotweb);
-
+	free(gotwebd_env->worker_load);
 	free(gotwebd_env);
 
 	exit(0);
@@ -1165,6 +1195,7 @@ sockets_socket_accept(int fd, short event, void *arg)
 	c->buf_len = 0;
 	c->client_status = CLIENT_CONNECT;
 	c->request_id = get_request_id();
+	c->worker_idx = -1;
 
 	event_set(&c->ev, s, EV_READ, read_fcgi_records, c);
 	event_add(&c->ev, NULL);