From: Omar Polo <op@omarpolo.com>
Subject: Re: gotwebd: get rid of proc.[ch]
To: gameoftrees@openbsd.org
Date: Wed, 15 Nov 2023 19:46:57 +0100

On 2023/11/15 18:42:14 +0100, Omar Polo <op@omarpolo.com> wrote:
> proc.c really shines when there's a net of different kinds of processes
> with potentially many instances that needs to talk.  gotwebd instead is
> just a main process for the privileged operations plus a bunch of
> unpriviled workers.
> 
> using proc.c here only really caused some headaces, so here's a proposal
> to remove it.  it needs some testing, but some preliminary testing
> locally promises good :)
> 
> There's something regarding the verbose logging that I still need to
> figure out exactly, but otherwise seems to be working just as before.

Found it.  The logging handling in the child was wrong, plus in
log_debug() we log only if verbose > 1.  Now, we don't do anything
special for the case verbose == 1 so I just 'lowered' log_debug() to one
-v.

> There are now two main routines that I expect will be used to send
> messages:
> 
>  - main_compose_sockets(): send an imsg (with optional fd) to the
>    sockets processes.  The file descriptor is dup'ed as needed.
>    This alone semplifies most of the loops in config.c
> 
>  - sockets_compose_main(): to send messages the other ways.
> 
> There is no iov variant since we don't need them so far, will be easy to
> add eventually.
> 
> The imsgev structs now live in the main gotwebd struct.  The parent
> allocates enough entries for the children it needs to run, while each
> child only initialize one to talk with the parent.
> 
> Maybe my hands slipped and there's more fixing that strictly needed...
> 
> I've omitted `got rm proc.[ch]' to keep the diff short, but I intend to
> do so before committing.

updated diff, this includes the fix to the daemon() call too.

diff 5abbba2d467df0d641100b74fbe24428fbb1c2c6 c85d8cd6af267cd4529f13315a1e00a416621d5f
commit - 5abbba2d467df0d641100b74fbe24428fbb1c2c6
commit + c85d8cd6af267cd4529f13315a1e00a416621d5f
blob - 8a18107cdc7ef357964ae5a8c38a6e1205874e12
blob + 258caeda8f63d90382ebfba64f426fcbc541a389
--- gotwebd/Makefile
+++ gotwebd/Makefile
@@ -7,7 +7,7 @@ SUBDIR = libexec
 .include "Makefile.inc"
 
 PROG =		gotwebd
-SRCS =		config.c sockets.c log.c gotwebd.c parse.y proc.c \
+SRCS =		config.c sockets.c log.c gotwebd.c parse.y \
 		fcgi.c gotweb.c got_operations.c tmpl.c pages.c
 SRCS +=		blame.c commit_graph.c delta.c diff.c \
 		diffreg.c error.c object.c object_cache.c \
blob - 1c8057cb99a030e7a3b45871f35124b82101e918
blob + 775d88bd837ed1c57363aea14c95b1793c999afa
--- gotwebd/config.c
+++ gotwebd/config.c
@@ -39,31 +39,18 @@
 #include "got_opentemp.h"
 #include "got_reference.h"
 
-#include "proc.h"
 #include "gotwebd.h"
 
 int
 config_init(struct gotwebd *env)
 {
-	struct privsep *ps = env->gotwebd_ps;
-	unsigned int what;
-
 	strlcpy(env->httpd_chroot, D_HTTPD_CHROOT, sizeof(env->httpd_chroot));
 
-	/* Global configuration. */
-	if (privsep_process == PROC_GOTWEBD)
-		env->prefork_gotwebd = GOTWEBD_NUMPROC;
+	env->prefork_gotwebd = GOTWEBD_NUMPROC;
+	env->server_cnt = 0;
+	TAILQ_INIT(&env->servers);
+	TAILQ_INIT(&env->sockets);
 
-	ps->ps_what[PROC_GOTWEBD] = CONFIG_ALL;
-	ps->ps_what[PROC_SOCKS] = CONFIG_SOCKS;
-
-	/* Other configuration. */
-	what = ps->ps_what[privsep_process];
-	if (what & CONFIG_SOCKS) {
-		env->server_cnt = 0;
-		TAILQ_INIT(&env->servers);
-		TAILQ_INIT(&env->sockets);
-	}
 	return 0;
 }
 
@@ -71,23 +58,17 @@ int
 config_getcfg(struct gotwebd *env, struct imsg *imsg)
 {
 	/* nothing to do but tell gotwebd configuration is done */
-	if (privsep_process != PROC_GOTWEBD)
-		proc_compose(env->gotwebd_ps, PROC_GOTWEBD,
-		    IMSG_CFG_DONE, NULL, 0);
-
+	if (sockets_compose_main(env, IMSG_CFG_DONE, NULL, 0) == -1)
+		fatal("sockets_compose_main IMSG_CFG_DONE");
 	return 0;
 }
 
 int
 config_setserver(struct gotwebd *env, struct server *srv)
 {
-	struct server ssrv;
-	struct privsep *ps = env->gotwebd_ps;
-
-	memcpy(&ssrv, srv, sizeof(ssrv));
-	if (proc_compose(ps, PROC_SOCKS, IMSG_CFG_SRV, &ssrv, sizeof(ssrv))
+	if (main_compose_sockets(env, IMSG_CFG_SRV, -1, srv, sizeof(*srv))
 	    == -1)
-		fatal("proc_compose");
+		fatal("main_compose_sockets IMSG_CFG_SRV");
 	return 0;
 }
 
@@ -97,17 +78,11 @@ config_getserver(struct gotwebd *env, struct imsg *ims
 	struct server *srv;
 	uint8_t *p = imsg->data;
 
-	IMSG_SIZE_CHECK(imsg, &srv);
-
 	srv = calloc(1, sizeof(*srv));
 	if (srv == NULL)
 		fatalx("%s: calloc", __func__);
 
-	if (IMSG_DATA_SIZE(imsg) != sizeof(*srv)) {
-		log_debug("%s: imsg size error", __func__);
-		free(srv);
-		return 1;
-	}
+	IMSG_SIZE_CHECK(imsg, srv);
 
 	memcpy(srv, p, sizeof(*srv));
 	srv->cached_repos = calloc(GOTWEBD_REPO_CACHESIZE,
@@ -129,62 +104,15 @@ config_getserver(struct gotwebd *env, struct imsg *ims
 int
 config_setsock(struct gotwebd *env, struct socket *sock)
 {
-	struct privsep *ps = env->gotwebd_ps;
-	struct socket_conf s;
-	int id;
-	int fd = -1, n, m;
-	struct iovec iov[6];
-	size_t c;
-	unsigned int what;
-
 	/* open listening sockets */
 	if (sockets_privinit(env, sock) == -1)
 		return -1;
 
-	for (id = 0; id < PROC_MAX; id++) {
-		what = ps->ps_what[id];
+	if (main_compose_sockets(env, IMSG_CFG_SOCK, sock->fd,
+	    &sock->conf, sizeof(sock->conf)) == -1)
+		fatal("main_compose_sockets IMSG_CFG_SOCK");
 
-		if ((what & CONFIG_SOCKS) == 0 || id == privsep_process)
-			continue;
-
-		memcpy(&s, &sock->conf, sizeof(s));
-
-		c = 0;
-		iov[c].iov_base = &s;
-		iov[c++].iov_len = sizeof(s);
-
-		if (id == PROC_SOCKS) {
-			/* XXX imsg code will close the fd after 1st call */
-			n = -1;
-			proc_range(ps, id, &n, &m);
-			for (n = 0; n < m; n++) {
-				if (sock->fd == -1)
-					fd = -1;
-				else if ((fd = dup(sock->fd)) == -1)
-					return 1;
-				if (proc_composev_imsg(ps, id, n, IMSG_CFG_SOCK,
-				    -1, fd, iov, c) != 0) {
-					log_warn("%s: failed to compose "
-					    "IMSG_CFG_SOCK imsg",
-					    __func__);
-					return 1;
-				}
-				if (proc_flush_imsg(ps, id, n) == -1) {
-					log_warn("%s: failed to flush "
-					    "IMSG_CFG_SOCK imsg",
-					    __func__);
-					return 1;
-				}
-			}
-		}
-	}
-
-	/* Close socket early to prevent fd exhaustion in gotwebd. */
-	if (sock->fd != -1) {
-		close(sock->fd);
-		sock->fd = -1;
-	}
-
+	sock->fd = -1;
 	return 0;
 }
 
@@ -237,60 +165,20 @@ config_getsock(struct gotwebd *env, struct imsg *imsg)
 int
 config_setfd(struct gotwebd *env, struct socket *sock)
 {
-	struct privsep *ps = env->gotwebd_ps;
-	int id, s;
-	int fd = -1, n, m, j;
-	struct iovec iov[6];
-	size_t c;
-	unsigned int what;
+	int i, fd;
 
 	log_debug("%s: Allocating %d file descriptors",
 	    __func__, PRIV_FDS__MAX + GOTWEB_PACK_NUM_TEMPFILES);
 
-	for (j = 0; j < PRIV_FDS__MAX + GOTWEB_PACK_NUM_TEMPFILES; j++) {
-		for (id = 0; id < PROC_MAX; id++) {
-			what = ps->ps_what[id];
-
-			if ((what & CONFIG_SOCKS) == 0 || id == privsep_process)
-				continue;
-
-			s = sock->conf.id;
-			c = 0;
-			iov[c].iov_base = &s;
-			iov[c++].iov_len = sizeof(s);
-
-			if (id == PROC_SOCKS) {
-				/*
-				 * XXX imsg code will close the fd
-				 * after 1st call
-				 */
-				n = -1;
-				proc_range(ps, id, &n, &m);
-				for (n = 0; n < m; n++) {
-					fd = got_opentempfd();
-					if (fd == -1)
-						return 1;
-					if (proc_composev_imsg(ps, id, n,
-					    IMSG_CFG_FD, -1, fd, iov, c) != 0) {
-						log_warn("%s: failed to compose "
-						    "IMSG_CFG_FD imsg",
-						    __func__);
-						return 1;
-					}
-					if (proc_flush_imsg(ps, id, n) == -1) {
-						log_warn("%s: failed to flush "
-						    "IMSG_CFG_FD imsg",
-						    __func__);
-						return 1;
-					}
-				}
-			}
-		}
-
-		/* Close fd early to prevent fd exhaustion in gotwebd. */
-		if (fd != -1)
-			close(fd);
+	for (i = 0; i < PRIV_FDS__MAX + GOTWEB_PACK_NUM_TEMPFILES; i++) {
+		fd = got_opentempfd();
+		if (fd == -1)
+			fatal("got_opentemp");
+		if (main_compose_sockets(env, IMSG_CFG_FD, fd,
+		    &sock->conf.id, sizeof(sock->conf.id)) == -1)
+			fatal("main_compose_sockets IMSG_CFG_FD");
 	}
+
 	return 0;
 }
 
blob - b6905f7fda842f60988300371ff61ab63db68a4c
blob + 202f1af662fe84b6759cc1a1c53efca43470da98
--- gotwebd/fcgi.c
+++ gotwebd/fcgi.c
@@ -35,7 +35,6 @@
 #include "got_error.h"
 #include "got_reference.h"
 
-#include "proc.h"
 #include "gotwebd.h"
 #include "tmpl.h"
 
blob - 6d9d5cf8b5507c4d545eca925df5fe128789c73a
blob + 45a8a36ea39f6f705bfc81c145d550389dd82872
--- gotwebd/got_operations.c
+++ gotwebd/got_operations.c
@@ -39,7 +39,6 @@
 #include "got_blame.h"
 #include "got_privsep.h"
 
-#include "proc.h"
 #include "gotwebd.h"
 
 static const struct got_error *got_init_repo_commit(struct repo_commit **);
blob - 5d1283460327c8a36e76c5f8ed29b5fd573a5ce8
blob + 228d0e727516198db3aa33c2653f669ee179b8d6
--- gotwebd/gotweb.c
+++ gotwebd/gotweb.c
@@ -49,7 +49,6 @@
 #include "got_blame.h"
 #include "got_privsep.h"
 
-#include "proc.h"
 #include "gotwebd.h"
 #include "tmpl.h"
 
blob - 9b7b6853658f6996141e0355f9a0a6f08fca0c7c
blob + 4017a8fd50fbe849af03accbf3ef5e16ba4eb232
--- gotwebd/gotwebd.c
+++ gotwebd/gotwebd.c
@@ -42,7 +42,6 @@
 #include "got_opentemp.h"
 #include "got_reference.h"
 
-#include "proc.h"
 #include "gotwebd.h"
 
 __dead void usage(void);
@@ -52,40 +51,138 @@ int	 gotwebd_configure(struct gotwebd *);
 void	 gotwebd_configure_done(struct gotwebd *);
 void	 gotwebd_sighdlr(int sig, short event, void *arg);
 void	 gotwebd_shutdown(void);
-int	 gotwebd_dispatch_sockets(int, struct privsep_proc *, struct imsg *);
+void	 gotwebd_dispatch_sockets(int, short, void *);
 
 struct gotwebd	*gotwebd_env;
 
-static struct privsep_proc procs[] = {
-	{ "sockets",	PROC_SOCKS,	gotwebd_dispatch_sockets, sockets,
-	    sockets_shutdown },
-};
+void
+imsg_event_add(struct imsgev *iev)
+{
+	if (iev->handler == NULL) {
+		imsg_flush(&iev->ibuf);
+		return;
+	}
 
+	iev->events = EV_READ;
+	if (iev->ibuf.w.queued)
+		iev->events |= EV_WRITE;
+
+	event_del(&iev->ev);
+	event_set(&iev->ev, iev->ibuf.fd, iev->events, iev->handler, iev->data);
+	event_add(&iev->ev, NULL);
+}
+
 int
-gotwebd_dispatch_sockets(int fd, struct privsep_proc *p, struct imsg *imsg)
+imsg_compose_event(struct imsgev *iev, uint16_t type, uint32_t peerid,
+    pid_t pid, int fd, const void *data, uint16_t datalen)
 {
-	struct privsep		*ps = p->p_ps;
-	struct gotwebd		*env = ps->ps_env;
+	int ret;
 
-	switch (imsg->hdr.type) {
-	case IMSG_CFG_DONE:
-		gotwebd_configure_done(env);
-		break;
-	default:
-		return (-1);
+	ret = imsg_compose(&iev->ibuf, type, peerid, pid, fd, data, datalen);
+	if (ret == -1)
+		return (ret);
+	imsg_event_add(iev);
+	return (ret);
+}
+
+int
+main_compose_sockets(struct gotwebd *env, uint32_t type, int fd,
+    const void *data, uint16_t len)
+{
+	size_t	 i;
+	int	 ret, d;
+
+	for (i = 0; i < env->nserver; ++i) {
+		d = -1;
+		if (fd != -1 && (d = dup(fd)) == -1)
+			return (-1);
+
+		ret = imsg_compose_event(&env->iev_server[i], type, 0, -1,
+		    d, data, len);
+		if (ret == -1)
+			return (-1);
+
+		/* prevent fd exhaustion */
+		if (d != -1) {
+			do {
+				ret = imsg_flush(&env->iev_server[i].ibuf);
+			} while (ret == -1 && errno == EAGAIN);
+			if (ret == -1)
+				return (-1);
+			imsg_event_add(&env->iev_server[i]);
+		}
 	}
 
-	return (0);
+	if (fd != -1)
+		close(fd);
+
+	return 0;
 }
 
+int
+sockets_compose_main(struct gotwebd *env, uint32_t type, const void *d,
+    uint16_t len)
+{
+	return (imsg_compose_event(env->iev_parent, type, 0, -1, -1, d, len));
+}
+
 void
+gotwebd_dispatch_sockets(int fd, short event, void *arg)
+{
+	struct imsgev		*iev = arg;
+	struct imsgbuf		*ibuf;
+	struct imsg		 imsg;
+	struct gotwebd		*env = gotwebd_env;
+	ssize_t			 n;
+	int			 shut = 0;
+
+	ibuf = &iev->ibuf;
+
+	if (event & EV_READ) {
+		if ((n = imsg_read(ibuf)) == -1 && errno != EAGAIN)
+			fatal("imsg_read error");
+		if (n == 0)	/* Connection closed */
+			shut = 1;
+	}
+	if (event & EV_WRITE) {
+		if ((n = msgbuf_write(&ibuf->w)) == -1 && errno != EAGAIN)
+			fatal("msgbuf_write");
+		if (n == 0)	/* Connection closed */
+			shut = 1;
+	}
+
+	for (;;) {
+		if ((n = imsg_get(ibuf, &imsg)) == -1)
+			fatal("imsg_get");
+		if (n == 0)	/* No more messages. */
+			break;
+
+		switch (imsg.hdr.type) {
+		case IMSG_CFG_DONE:
+			gotwebd_configure_done(env);
+			break;
+		default:
+			fatalx("%s: unknown imsg type %d", __func__,
+			    imsg.hdr.type);
+		}
+
+		imsg_free(&imsg);
+	}
+
+	if (!shut)
+		imsg_event_add(iev);
+	else {
+		/* This pipe is dead.  Remove its event handler */
+		event_del(&iev->ev);
+		event_loopexit(NULL);
+	}
+}
+
+void
 gotwebd_sighdlr(int sig, short event, void *arg)
 {
 	/* struct privsep	*ps = arg; */
 
-	if (privsep_process != PROC_GOTWEBD)
-		return;
-
 	switch (sig) {
 	case SIGHUP:
 		log_info("%s: ignoring SIGHUP", __func__);
@@ -105,6 +202,54 @@ gotwebd_sighdlr(int sig, short event, void *arg)
 	}
 }
 
+static int
+spawn_socket_process(struct gotwebd *env, const char *argv0, int n)
+{
+	const char	*argv[5];
+	int		 argc = 0;
+	int		 p[2];
+	pid_t		 pid;
+
+	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, p) == -1)
+		fatal("socketpair");
+
+	switch (pid = fork()) {
+	case -1:
+		fatal("fork");
+	case 0:		/* child */
+		break;
+	default:	/* parent */
+		close(p[0]);
+		imsg_init(&env->iev_server[n].ibuf, p[1]);
+		env->iev_server[n].handler = gotwebd_dispatch_sockets;
+		env->iev_server[n].data = &env->iev_server[n];
+		event_set(&env->iev_server[n].ev, p[1], EV_READ,
+		    gotwebd_dispatch_sockets, &env->iev_server[n]);
+		event_add(&env->iev_server[n].ev, NULL);
+		return 0;
+	}
+
+	close(p[1]);
+
+	argv[argc++] = argv0;
+	argv[argc++] = "-S";
+	if (env->gotwebd_debug)
+		argv[argc++] = "-d";
+	if (env->gotwebd_verbose)
+		argv[argc++] = "-v";
+	argv[argc] = NULL;
+
+	if (p[0] != 3) {
+		if (dup2(p[0], 3) == -1)
+			fatal("dup2");
+	} else if (fcntl(p[0], F_SETFD, 0) == -1)
+		fatal("fcntl");
+
+	/* obnoxious cast */
+	execvp(argv0, (char * const *)argv);
+	fatal("execvp %s", argv0);
+}
+
 __dead void
 usage(void)
 {
@@ -116,25 +261,27 @@ usage(void)
 int
 main(int argc, char **argv)
 {
-	struct gotwebd *env;
-	struct privsep *ps;
-	unsigned int proc;
-	int ch;
-	const char *conffile = GOTWEBD_CONF;
-	enum privsep_procid proc_id = PROC_GOTWEBD;
-	int proc_instance = 0;
-	const char *errp, *title = NULL;
-	int argc0 = argc;
+	struct event		 sigint, sigterm, sighup, sigpipe, sigusr1;
+	struct gotwebd		*env;
+	struct passwd		*pw;
+	int			 ch, i;
+	int			 no_action = 0;
+	int			 server_proc = 0;
+	const char		*conffile = GOTWEBD_CONF;
+	const char		*argv0;
 
+	if ((argv0 = argv[0]) == NULL)
+		argv0 = "gotwebd";
+
 	env = calloc(1, sizeof(*env));
 	if (env == NULL)
 		fatal("%s: calloc", __func__);
+	config_init(env);
 
 	/* log to stderr until daemonized */
 	log_init(1, LOG_DAEMON);
 
-	/* XXX: add s and S for both sockets */
-	while ((ch = getopt(argc, argv, "D:df:I:nP:v")) != -1) {
+	while ((ch = getopt(argc, argv, "D:df:nSv")) != -1) {
 		switch (ch) {
 		case 'D':
 			if (cmdline_symset(optarg) < 0)
@@ -142,26 +289,16 @@ main(int argc, char **argv)
 				    optarg);
 			break;
 		case 'd':
-			env->gotwebd_debug = 2;
+			env->gotwebd_debug = 1;
 			break;
 		case 'f':
 			conffile = optarg;
 			break;
-		case 'I':
-			proc_instance = strtonum(optarg, 0,
-			    PROC_MAX_INSTANCES, &errp);
-			if (errp)
-				fatalx("invalid process instance");
-			break;
 		case 'n':
-			env->gotwebd_debug = 2;
-			env->gotwebd_noaction = 1;
+			no_action = 1;
 			break;
-		case 'P':
-			title = optarg;
-			proc_id = proc_getid(procs, nitems(procs), title);
-			if (proc_id == PROC_MAX)
-				fatalx("invalid process name");
+		case 'S':
+			server_proc = 1;
 			break;
 		case 'v':
 			env->gotwebd_verbose++;
@@ -175,72 +312,77 @@ main(int argc, char **argv)
 	if (argc > 0)
 		usage();
 
-	ps = calloc(1, sizeof(*ps));
-	if (ps == NULL)
-		fatal("%s: calloc:", __func__);
-
 	gotwebd_env = env;
-	env->gotwebd_ps = ps;
-	ps->ps_env = env;
 	env->gotwebd_conffile = conffile;
 
 	if (parse_config(env->gotwebd_conffile, env) == -1)
 		exit(1);
 
-	if (env->gotwebd_noaction && !env->gotwebd_debug)
-		env->gotwebd_debug = 1;
+	if (no_action) {
+		fprintf(stderr, "configuration OK\n");
+		exit(0);
+	}
 
 	/* check for root privileges */
-	if (env->gotwebd_noaction == 0) {
-		if (geteuid())
-			fatalx("need root privileges");
-	}
+	if (geteuid())
+		fatalx("need root privileges");
 
-	ps->ps_pw = getpwnam(GOTWEBD_USER);
-	if (ps->ps_pw == NULL)
+	pw = getpwnam(GOTWEBD_USER);
+	if (pw == NULL)
 		fatalx("unknown user %s", GOTWEBD_USER);
+	env->pw = pw;
 
 	log_init(env->gotwebd_debug, LOG_DAEMON);
 	log_setverbose(env->gotwebd_verbose);
 
-	if (env->gotwebd_noaction)
-		ps->ps_noaction = 1;
+	if (server_proc) {
+		setproctitle("sockets");
+		log_procinit("sockets");
 
-	ps->ps_instances[PROC_SOCKS] = env->prefork_gotwebd;
-	ps->ps_instance = proc_instance;
-	if (title != NULL)
-		ps->ps_title[proc_id] = title;
+		if (chroot(pw->pw_dir) == -1)
+			fatal("chroot %s", pw->pw_dir);
+		if (chdir("/") == -1)
+			fatal("chdir /");
+		if (setgroups(1, &pw->pw_gid) == -1 ||
+		    setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) == -1 ||
+		    setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid) == -1)
+			fatal("failed to drop privileges");
 
-	for (proc = 0; proc < nitems(procs); proc++)
-		procs[proc].p_chroot = env->httpd_chroot;
+		sockets(env, 3);
+		return 1;
+	}
 
-	/* only the gotwebd returns */
-	proc_init(ps, procs, nitems(procs), argc0, argv, proc_id);
-
-	log_procinit("gotwebd");
 	if (!env->gotwebd_debug && daemon(0, 0) == -1)
-		fatal("can't daemonize");
+		fatal("daemon");
 
-	if (ps->ps_noaction == 0)
-		log_info("%s startup", getprogname());
-
 	event_init();
 
-	signal_set(&ps->ps_evsigint, SIGINT, gotwebd_sighdlr, ps);
-	signal_set(&ps->ps_evsigterm, SIGTERM, gotwebd_sighdlr, ps);
-	signal_set(&ps->ps_evsighup, SIGHUP, gotwebd_sighdlr, ps);
-	signal_set(&ps->ps_evsigpipe, SIGPIPE, gotwebd_sighdlr, ps);
-	signal_set(&ps->ps_evsigusr1, SIGUSR1, gotwebd_sighdlr, ps);
+	env->nserver = env->prefork_gotwebd;
+	env->iev_server = calloc(env->nserver, sizeof(*env->iev_server));
+	if (env->iev_server == NULL)
+		fatal("calloc");
 
-	signal_add(&ps->ps_evsigint, NULL);
-	signal_add(&ps->ps_evsigterm, NULL);
-	signal_add(&ps->ps_evsighup, NULL);
-	signal_add(&ps->ps_evsigpipe, NULL);
-	signal_add(&ps->ps_evsigusr1, NULL);
+	for (i = 0; i < env->nserver; ++i) {
+		if (spawn_socket_process(env, argv0, i) == -1)
+			fatal("spawn_socket_process");
+	}
 
-	if (!env->gotwebd_noaction)
-		proc_connect(ps);
+	log_procinit("gotwebd");
 
+	log_info("%s startup", getprogname());
+
+	signal_set(&sigint, SIGINT, gotwebd_sighdlr, env);
+	signal_set(&sigterm, SIGTERM, gotwebd_sighdlr, env);
+	signal_set(&sighup, SIGHUP, gotwebd_sighdlr, env);
+	signal_set(&sigpipe, SIGPIPE, gotwebd_sighdlr, env);
+	signal_set(&sigusr1, SIGUSR1, gotwebd_sighdlr, env);
+
+	signal_add(&sigint, NULL);
+	signal_add(&sigterm, NULL);
+	signal_add(&sighup, NULL);
+	signal_add(&sigpipe, NULL);
+	signal_add(&sigusr1, NULL);
+
 	if (gotwebd_configure(env) == -1)
 		fatalx("configuration failed");
 
@@ -275,14 +417,7 @@ gotwebd_configure(struct gotwebd *env)
 {
 	struct server *srv;
 	struct socket *sock;
-	int id;
 
-	if (env->gotwebd_noaction) {
-		fprintf(stderr, "configuration OK\n");
-		proc_kill(env->gotwebd_ps);
-		exit(0);
-	}
-
 	/* gotweb need to reload its config. */
 	env->gotwebd_reload = env->prefork_gotwebd;
 
@@ -300,11 +435,8 @@ gotwebd_configure(struct gotwebd *env)
 			fatalx("%s: send priv_fd error", __func__);
 	}
 
-	for (id = 0; id < PROC_MAX; id++) {
-		if (id == privsep_process)
-			continue;
-		proc_compose(env->gotwebd_ps, id, IMSG_CFG_DONE, NULL, 0);
-	}
+	if (main_compose_sockets(env, IMSG_CFG_DONE, -1, NULL, 0) == -1)
+		fatal("main_compose_sockets IMSG_CFG_DONE");
 
 	return (0);
 }
@@ -312,31 +444,44 @@ gotwebd_configure(struct gotwebd *env)
 void
 gotwebd_configure_done(struct gotwebd *env)
 {
-	int id;
-
 	if (env->gotwebd_reload == 0) {
 		log_warnx("%s: configuration already finished", __func__);
 		return;
 	}
 
 	env->gotwebd_reload--;
-	if (env->gotwebd_reload == 0) {
-		for (id = 0; id < PROC_MAX; id++) {
-			if (id == privsep_process)
-				continue;
-			proc_compose(env->gotwebd_ps, id, IMSG_CTL_START,
-			    NULL, 0);
-		}
-	}
+	if (env->gotwebd_reload == 0 &&
+	    main_compose_sockets(env, IMSG_CTL_START, -1, NULL, 0) == -1)
+		fatal("main_compose_sockets IMSG_CTL_START");
 }
 
 void
 gotwebd_shutdown(void)
 {
-	proc_kill(gotwebd_env->gotwebd_ps);
+	struct gotwebd	*env = gotwebd_env;
+	pid_t		 pid;
+	int		 i, status;
 
-	/* unlink(gotwebd_env->gotweb->gotweb_conf.gotweb_unix_socket_name); */
-	/* free(gotwebd_env->gotweb); */
+	for (i = 0; i < env->nserver; ++i) {
+		event_del(&env->iev_server[i].ev);
+		imsg_clear(&env->iev_server[i].ibuf);
+		close(env->iev_server[i].ibuf.fd);
+		env->iev_server[i].ibuf.fd = -1;
+	}
+
+	do {
+		pid = waitpid(WAIT_ANY, &status, 0);
+		if (pid <= 0)
+			continue;
+
+		if (WIFSIGNALED(status))
+			log_warnx("lost child: pid %u terminated; signal %d",
+			    pid, WTERMSIG(status));
+		else if (WIFEXITED(status) && WEXITSTATUS(status) != 0)
+			log_warnx("lost child: pid %u exited abnormally",
+			    pid);
+	} while (pid != -1 || (pid == -1 && errno == EINTR));
+
 	free(gotwebd_env);
 
 	log_warnx("gotwebd terminating");
blob - 44293b62e594c517072cd3268cb34949d7393bd0
blob + fd1c0c99231219c08e4fc5bf185228b9925892c3
--- gotwebd/gotwebd.h
+++ gotwebd/gotwebd.h
@@ -49,6 +49,8 @@
 #define GOTWEBD_NUMPROC		 3
 #define GOTWEBD_REPO_CACHESIZE	 4
 
+#define PROC_MAX_INSTANCES	 32
+
 /* GOTWEB DEFAULTS */
 #define MAX_QUERYSTRING		 2048
 #define MAX_DOCUMENT_URI	 255
@@ -120,13 +122,28 @@ struct got_tree_entry;
 struct got_reflist_head;
 
 enum imsg_type {
-	IMSG_CFG_SRV = IMSG_PROC_MAX,
+	IMSG_CFG_SRV,
 	IMSG_CFG_SOCK,
 	IMSG_CFG_FD,
 	IMSG_CFG_DONE,
 	IMSG_CTL_START,
 };
 
+struct imsgev {
+	struct imsgbuf		 ibuf;
+	void			(*handler)(int, short, void *);
+	struct event		 ev;
+	void			*data;
+	short			 events;
+};
+
+#define IMSG_SIZE_CHECK(imsg, p) do {					\
+	if (IMSG_DATA_SIZE(imsg) < sizeof(*p))				\
+		fatalx("bad length imsg received (%s)",	#p);		\
+} while (0)
+
+#define IMSG_DATA_SIZE(imsg)	((imsg)->hdr.len - IMSG_HEADER_SIZE)
+
 struct env_val {
 	SLIST_ENTRY(env_val)	 entry;
 	char			*val;
@@ -345,17 +362,22 @@ struct socket {
 };
 TAILQ_HEAD(socketlist, socket);
 
+struct passwd;
 struct gotwebd {
 	struct serverlist	servers;
 	struct socketlist	sockets;
 
-	struct privsep	*gotwebd_ps;
 	const char	*gotwebd_conffile;
 
 	int		 gotwebd_debug;
 	int		 gotwebd_verbose;
-	int		 gotwebd_noaction;
 
+	struct imsgev	*iev_parent;
+	struct imsgev	*iev_server;
+	size_t		 nserver;
+
+	struct passwd	*pw;
+
 	uint16_t	 prefork_gotwebd;
 	int		 gotwebd_reload;
 
@@ -442,9 +464,17 @@ extern struct gotwebd	*gotwebd_env;
 typedef int (*got_render_blame_line_cb)(struct template *, const char *,
     struct blame_line *, int, int);
 
+/* gotwebd.c */
+void	 imsg_event_add(struct imsgev *);
+int	 imsg_compose_event(struct imsgev *, uint16_t, uint32_t,
+	    pid_t, int, const void *, uint16_t);
+int	 main_compose_sockets(struct gotwebd *, uint32_t, int,
+	    const void *, uint16_t);
+int	 sockets_compose_main(struct gotwebd *, uint32_t,
+	    const void *, uint16_t);
+
 /* sockets.c */
-void sockets(struct privsep *, struct privsep_proc *);
-void sockets_shutdown(void);
+void sockets(struct gotwebd *, int);
 void sockets_parse_sockets(struct gotwebd *);
 void sockets_socket_accept(int, short, void *);
 int sockets_privinit(struct gotwebd *, struct socket *);
@@ -519,3 +549,25 @@ int config_setfd(struct gotwebd *, struct socket *);
 int config_getfd(struct gotwebd *, struct imsg *);
 int config_getcfg(struct gotwebd *, struct imsg *);
 int config_init(struct gotwebd *);
+
+/* log.c */
+void	log_init(int, int);
+void	log_procinit(const char *);
+void	log_setverbose(int);
+int	log_getverbose(void);
+void	log_warn(const char *, ...)
+	    __attribute__((__format__ (printf, 1, 2)));
+void	log_warnx(const char *, ...)
+	    __attribute__((__format__ (printf, 1, 2)));
+void	log_info(const char *, ...)
+	    __attribute__((__format__ (printf, 1, 2)));
+void	log_debug(const char *, ...)
+	    __attribute__((__format__ (printf, 1, 2)));
+void	logit(int, const char *, ...)
+	    __attribute__((__format__ (printf, 2, 3)));
+void	vlog(int, const char *, va_list)
+	    __attribute__((__format__ (printf, 2, 0)));
+__dead void fatal(const char *, ...)
+	    __attribute__((__format__ (printf, 1, 2)));
+__dead void fatalx(const char *, ...)
+	    __attribute__((__format__ (printf, 1, 2)));
blob - 79d3d334581eddd8ffcc0f02f067e5c64d0be72e
blob + 73cbf47a766ba764769018c3e5833cb0414ed1f8
--- gotwebd/log.c
+++ gotwebd/log.c
@@ -164,7 +164,7 @@ log_debug(const char *emsg, ...)
 {
 	va_list ap;
 
-	if (verbose > 1) {
+	if (verbose) {
 		va_start(ap, emsg);
 		vlog(LOG_DEBUG, emsg, ap);
 		va_end(ap);
blob - a51f74abb8dfc67479ed3558a8c7efd83400e068
blob + d04e49122cc951bb2815072249af493a694624f3
--- gotwebd/pages.tmpl
+++ gotwebd/pages.tmpl
@@ -34,8 +34,6 @@
 #include "got_object.h"
 #include "got_reference.h"
 
-#include "proc.h"
-
 #include "gotwebd.h"
 #include "tmpl.h"
 
blob - b02dbb86b14c946386f468402580e3119a1f1e61
blob + 207d9f20026688bdc8849f0125e73b94a285504e
--- gotwebd/parse.y
+++ gotwebd/parse.y
@@ -49,7 +49,6 @@
 
 #include "got_reference.h"
 
-#include "proc.h"
 #include "gotwebd.h"
 
 TAILQ_HEAD(files, file)		 files = TAILQ_HEAD_INITIALIZER(files);
blob - 16a74076b94ffabf45821936ff713dacb0e1dff0
blob + dd100773cb3593bb4f04e31d024acf711bdc912e
--- gotwebd/sockets.c
+++ gotwebd/sockets.c
@@ -55,28 +55,25 @@
 #include "got_repository.h"
 #include "got_privsep.h"
 
-#include "proc.h"
 #include "gotwebd.h"
 #include "tmpl.h"
 
 #define SOCKS_BACKLOG 5
 #define MAXIMUM(a, b)	(((a) > (b)) ? (a) : (b))
 
-
 volatile int client_cnt;
 
 static struct timeval	timeout = { TIMEOUT_DEFAULT, 0 };
 
 static void	 sockets_sighdlr(int, short, void *);
-static void	 sockets_run(struct privsep *, struct privsep_proc *, void *);
+static void	 sockets_shutdown(void);
 static void	 sockets_launch(void);
 static void	 sockets_purge(struct gotwebd *);
 static void	 sockets_accept_paused(int, short, void *);
 static void	 sockets_rlimit(int);
 
-static int	 sockets_dispatch_gotwebd(int, struct privsep_proc *,
-		    struct imsg *);
-static int	 sockets_unix_socket_listen(struct privsep *, struct socket *);
+static void	 sockets_dispatch_main(int, short, void *);
+static int	 sockets_unix_socket_listen(struct gotwebd *, struct socket *);
 static int	 sockets_create_socket(struct address *);
 static int	 sockets_accept_reserve(int, struct sockaddr *, socklen_t *,
 		    int, volatile int *);
@@ -88,35 +85,44 @@ static struct socket *sockets_conf_new_socket_fcgi(str
 
 int cgi_inflight = 0;
 
-static struct privsep_proc procs[] = {
-	{ "gotwebd",	PROC_GOTWEBD,	sockets_dispatch_gotwebd  },
-};
-
 void
-sockets(struct privsep *ps, struct privsep_proc *p)
+sockets(struct gotwebd *env, int fd)
 {
-	proc_run(ps, p, procs, nitems(procs), sockets_run, NULL);
-}
+	struct event	 sighup, sigpipe, sigusr1, sigchld;
 
-static void
-sockets_run(struct privsep *ps, struct privsep_proc *p, void *arg)
-{
-	if (config_init(ps->ps_env) == -1)
-		fatal("failed to initialize configuration");
+	event_init();
 
-	p->p_shutdown = sockets_shutdown;
-
 	sockets_rlimit(-1);
 
-	signal_del(&ps->ps_evsigchld);
-	signal_set(&ps->ps_evsigchld, SIGCHLD, sockets_sighdlr, ps);
-	signal_add(&ps->ps_evsigchld, NULL);
+	if (config_init(env) == -1)
+		fatal("failed to initialize configuration");
 
+	if ((env->iev_parent = malloc(sizeof(*env->iev_parent))) == NULL)
+		fatal("malloc");
+	imsg_init(&env->iev_parent->ibuf, fd);
+	env->iev_parent->handler = sockets_dispatch_main;
+	env->iev_parent->data = env->iev_parent;
+	event_set(&env->iev_parent->ev, fd, EV_READ, sockets_dispatch_main,
+	    env->iev_parent);
+	event_add(&env->iev_parent->ev, NULL);
+
+	signal_set(&sighup, SIGCHLD, sockets_sighdlr, env);
+	signal_add(&sighup, NULL);
+	signal_set(&sigpipe, SIGCHLD, sockets_sighdlr, env);
+	signal_add(&sigpipe, NULL);
+	signal_set(&sigusr1, SIGCHLD, sockets_sighdlr, env);
+	signal_add(&sigusr1, NULL);
+	signal_set(&sigchld, SIGCHLD, sockets_sighdlr, env);
+	signal_add(&sigchld, NULL);
+
 #ifndef PROFILE
 	if (pledge("stdio rpath inet recvfd proc exec sendfd unveil",
 	    NULL) == -1)
 		fatal("pledge");
 #endif
+
+	event_dispatch();
+	sockets_shutdown();
 }
 
 void
@@ -301,48 +307,68 @@ sockets_purge(struct gotwebd *env)
 	}
 }
 
-static int
-sockets_dispatch_gotwebd(int fd, struct privsep_proc *p, struct imsg *imsg)
+static void
+sockets_dispatch_main(int fd, short event, void *arg)
 {
-	struct privsep *ps = p->p_ps;
-	int res = 0, cmd = 0, verbose;
+	struct imsgev		*iev = arg;
+	struct imsgbuf		*ibuf;
+	struct imsg		 imsg;
+	struct gotwebd		*env = gotwebd_env;
+	ssize_t			 n;
+	int			 shut = 0;
 
-	switch (imsg->hdr.type) {
-	case IMSG_CFG_SRV:
-		config_getserver(gotwebd_env, imsg);
-		break;
-	case IMSG_CFG_SOCK:
-		config_getsock(gotwebd_env, imsg);
-		break;
-	case IMSG_CFG_FD:
-		config_getfd(gotwebd_env, imsg);
-		break;
-	case IMSG_CFG_DONE:
-		config_getcfg(gotwebd_env, imsg);
-		break;
-	case IMSG_CTL_START:
-		sockets_launch();
-		break;
-	case IMSG_CTL_VERBOSE:
-		IMSG_SIZE_CHECK(imsg, &verbose);
-		memcpy(&verbose, imsg->data, sizeof(verbose));
-		log_setverbose(verbose);
-		break;
-	default:
-		return -1;
+	ibuf = &iev->ibuf;
+
+	if (event & EV_READ) {
+		if ((n = imsg_read(ibuf)) == -1 && errno != EAGAIN)
+			fatal("imsg_read error");
+		if (n == 0)	/* Connection closed */
+			shut = 1;
 	}
+	if (event & EV_WRITE) {
+		if ((n = msgbuf_write(&ibuf->w)) == -1 && errno != EAGAIN)
+			fatal("msgbuf_write");
+		if (n == 0)	/* Connection closed */
+			shut = 1;
+	}
 
-	switch (cmd) {
-	case 0:
-		break;
-	default:
-		if (proc_compose_imsg(ps, PROC_GOTWEBD, -1, cmd,
-		    imsg->hdr.peerid, -1, &res, sizeof(res)) == -1)
-			return -1;
-		break;
+	for (;;) {
+		if ((n = imsg_get(ibuf, &imsg)) == -1)
+			fatal("imsg_get");
+		if (n == 0)	/* No more messages. */
+			break;
+
+		switch (imsg.hdr.type) {
+		case IMSG_CFG_SRV:
+			config_getserver(env, &imsg);
+			break;
+		case IMSG_CFG_SOCK:
+			config_getsock(env, &imsg);
+			break;
+		case IMSG_CFG_FD:
+			config_getfd(env, &imsg);
+			break;
+		case IMSG_CFG_DONE:
+			config_getcfg(env, &imsg);
+			break;
+		case IMSG_CTL_START:
+			sockets_launch();
+			break;
+		default:
+			fatalx("%s: unknown imsg type %d", __func__,
+			    imsg.hdr.type);
+		}
+
+		imsg_free(&imsg);
 	}
 
-	return 0;
+	if (!shut)
+		imsg_event_add(iev);
+	else {
+		/* This pipe is dead.  Remove its event handler */
+		event_del(&iev->ev);
+		event_loopexit(NULL);
+	}
 }
 
 static void
@@ -366,7 +392,7 @@ sockets_sighdlr(int sig, short event, void *arg)
 	}
 }
 
-void
+static void
 sockets_shutdown(void)
 {
 	struct server *srv, *tsrv;
@@ -395,12 +421,10 @@ sockets_shutdown(void)
 int
 sockets_privinit(struct gotwebd *env, struct socket *sock)
 {
-	struct privsep *ps = env->gotwebd_ps;
-
 	if (sock->conf.af_type == AF_UNIX) {
 		log_debug("%s: initializing unix socket %s", __func__,
 		    sock->conf.unix_socket_name);
-		sock->fd = sockets_unix_socket_listen(ps, sock);
+		sock->fd = sockets_unix_socket_listen(env, sock);
 		if (sock->fd == -1) {
 			log_warnx("%s: create unix socket failed", __func__);
 			return -1;
@@ -422,9 +446,8 @@ sockets_privinit(struct gotwebd *env, struct socket *s
 }
 
 static int
-sockets_unix_socket_listen(struct privsep *ps, struct socket *sock)
+sockets_unix_socket_listen(struct gotwebd *env, struct socket *sock)
 {
-	struct gotwebd *env = ps->ps_env;
 	struct sockaddr_un sun;
 	struct socket *tsock;
 	int u_fd = -1;
@@ -480,8 +503,8 @@ sockets_unix_socket_listen(struct privsep *ps, struct 
 		return -1;
 	}
 
-	if (chown(sock->conf.unix_socket_name, ps->ps_pw->pw_uid,
-	    ps->ps_pw->pw_gid) == -1) {
+	if (chown(sock->conf.unix_socket_name, env->pw->pw_uid,
+	    env->pw->pw_gid) == -1) {
 		log_warn("%s: chown", __func__);
 		close(u_fd);
 		(void)unlink(sock->conf.unix_socket_name);