"GOT", but the "O" is a cute, smiling sun Index | Thread

From:
Stefan Sperling <stsp@stsp.name>
Subject:
split up pack_create.c
To:
gameoftrees@openbsd.org
Date:
Sun, 16 Oct 2022 12:31:59 +0200

Download raw body.

Some code in pack_create.c depends on privsep.c, which makes
future gotd(8) error out during the build like this:

ld: error: undefined symbol: got_privsep_recv_reused_deltas
ld: error: undefined symbol: got_privsep_init_commit_painting
ld: error: undefined symbol: got_privsep_send_painting_commits_done
ld: error: undefined symbol: got_privsep_send_painting_request
ld: error: undefined symbol: got_privsep_recv_painted_commits
ld: error: undefined symbol: got_privsep_send_object_idlist
ld: error: undefined symbol: got_privsep_send_object_idlist_done
ld: error: undefined symbol: got_privsep_send_raw_delta_outfd
ld: error: undefined symbol: got_privsep_send_delta_reuse_req

To allow me to fix this, move relevant code to a new file, called
pack_create_privsep.c, and link to this file where neeed.

gotd will then have to implement its own versions of got_pack_search_deltas(),
got_pack_paint_commits(), and got_pack_load_packed_object_ids(), avoiding
the privsep dance (instead, gotd reads objects directly inside a chroot).

ok?

diff a6072ec90f252faf1f1256591fb0d4623f4bc0c2 a4df42eded0178f40d762a4cf5239dcb95a02d42
commit - a6072ec90f252faf1f1256591fb0d4623f4bc0c2
commit + a4df42eded0178f40d762a4cf5239dcb95a02d42
blob - 2491213e9e4a591c7ea3ebd247528bf036f39e53
blob + 652bddb959b1bd1a528daf84da62d9a2f52d5d51
--- got/Makefile
+++ got/Makefile
@@ -15,7 +15,7 @@ SRCS=		got.c blame.c commit_graph.c delta.c diff.c \
 		diff_patience.c send.c deltify.c pack_create.c dial.c \
 		bloom.c murmurhash2.c ratelimit.c patch.c sigs.c date.c \
 		object_open_privsep.c read_gitconfig_privsep.c \
-		read_gotconfig_privsep.c
+		read_gotconfig_privsep.c pack_create_privsep.c
 
 MAN =		${PROG}.1 got-worktree.5 git-repository.5 got.conf.5
 
blob - 6ffb2c64697c223811c72f8cf386b645cb1bc757
blob + 00e5e77e780621968aa2a18606882840fb9f4d2c
--- gotadmin/Makefile
+++ gotadmin/Makefile
@@ -10,7 +10,8 @@ SRCS=		gotadmin.c \
 		path.c privsep.c reference.c repository.c repository_admin.c \
 		worktree_open.c sha1.c bloom.c murmurhash2.c ratelimit.c \
 		sigs.c buf.c date.c object_open_privsep.c \
-		read_gitconfig_privsep.c read_gotconfig_privsep.c
+		read_gitconfig_privsep.c read_gotconfig_privsep.c \
+		pack_create_privsep.c
 MAN =		${PROG}.1
 
 CPPFLAGS = -I${.CURDIR}/../include -I${.CURDIR}/../lib
blob - bc54ae58a9ac550958c259f684367576576d5209
blob + eccf16c525cb2cec8d54d8ed5220540f001ddb78
--- lib/got_lib_pack_create.h
+++ lib/got_lib_pack_create.h
@@ -27,3 +27,120 @@ const struct got_error *got_pack_create(uint8_t *pack_
     struct got_repository *repo, int loose_obj_only, int allow_empty,
     got_pack_progress_cb progress_cb, void *progress_arg,
     got_cancel_cb cancel_cb, void *cancel_arg);
+
+const struct got_error *
+got_pack_cache_pack_for_packidx(struct got_pack **pack,
+    struct got_packidx *packidx, struct got_repository *repo);
+
+const struct got_error *
+got_pack_find_pack_for_commit_painting(struct got_packidx **best_packidx,
+    struct got_object_id_queue *ids, int nids, struct got_repository *repo);
+const struct got_error *got_pack_find_pack_for_reuse(
+    struct got_packidx **best_packidx, struct got_repository *repo);
+
+struct got_ratelimit;
+const struct got_error *got_pack_paint_commits(int *ncolored,
+    struct got_object_id_queue *ids, int nids,
+    struct got_object_idset *keep, struct got_object_idset *drop,
+    struct got_object_idset *skip, struct got_repository *repo,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg);
+
+enum got_pack_findtwixt_color {
+	COLOR_KEEP = 0,
+	COLOR_DROP,
+	COLOR_SKIP,
+	COLOR_MAX,
+};
+
+const struct got_error *got_pack_paint_commit(struct got_object_qid *qid,
+    intptr_t color);
+const struct got_error *got_pack_queue_commit_id(
+    struct got_object_id_queue *ids, struct got_object_id *id, intptr_t color,
+    struct got_repository *repo);
+
+struct got_pack_metavec {
+	struct got_pack_meta **meta;
+	int nmeta;
+	int metasz;
+};
+
+struct got_pack_meta {
+	struct got_object_id id;
+	uint32_t path_hash;
+	int	obj_type;
+	off_t	size;
+	time_t	mtime;
+
+	/* The best delta we picked */
+	struct got_pack_meta *head;
+	struct got_pack_meta *prev;
+	unsigned char *delta_buf; /* if encoded in memory (compressed) */
+	off_t	delta_offset;	/* offset in delta cache file (compressed) */
+	off_t	delta_len;	/* encoded delta length */
+	off_t	delta_compressed_len; /* encoded+compressed delta length */
+	int	nchain;
+
+	off_t   reused_delta_offset; /* offset of delta in reused pack file */
+	struct got_object_id *base_obj_id;
+
+	/* Only used for delta window */
+	struct got_delta_table *dtab;
+
+	/* Only used for writing offset deltas */
+	off_t	off;
+};
+
+const struct got_error *got_pack_add_meta(struct got_pack_meta *m,
+    struct got_pack_metavec *v);
+
+const struct got_error *
+got_pack_search_deltas(struct got_pack_metavec *v,
+    struct got_object_idset *idset, int delta_cache_fd,
+    int ncolored, int nfound, int ntrees, int ncommits,
+    struct got_repository *repo,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg);
+
+const struct got_error *
+got_pack_report_progress(got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, int ncolored, int nfound, int ntrees,
+    off_t packfile_size, int ncommits, int nobj_total, int obj_deltify,
+    int nobj_written);
+
+const struct got_error *
+got_pack_load_packed_object_ids(int *found_all_objects,
+    struct got_object_id **ours, int nours,
+    struct got_object_id **theirs, int ntheirs,
+    int want_meta, uint32_t seed, struct got_object_idset *idset,
+    struct got_object_idset *idset_exclude, int loose_obj_only,
+    struct got_repository *repo, struct got_packidx *packidx,
+    int *ncolored, int *nfound, int *ntrees,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg);
+
+const struct got_error *
+got_pack_load_tree_entries(struct got_object_id_queue *ids, int want_meta,
+    struct got_object_idset *idset, struct got_object_idset *idset_exclude,
+    struct got_tree_object *tree,
+    const char *dpath, time_t mtime, uint32_t seed, struct got_repository *repo,
+    int loose_obj_only, int *ncolored, int *nfound, int *ntrees,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg);
+
+const struct got_error *
+got_pack_load_tree(int want_meta, struct got_object_idset *idset,
+    struct got_object_idset *idset_exclude,
+    struct got_object_id *tree_id, const char *dpath, time_t mtime,
+    uint32_t seed, struct got_repository *repo, int loose_obj_only,
+    int *ncolored, int *nfound, int *ntrees,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg);
+
+const struct got_error *
+got_pack_add_object(int want_meta, struct got_object_idset *idset,
+    struct got_object_id *id, const char *path, int obj_type,
+    time_t mtime, uint32_t seed, int loose_obj_only,
+    struct got_repository *repo, int *ncolored, int *nfound, int *ntrees,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl);
blob - a3679a51361ead3fefdf763047f2b5756aaa45ee
blob + 5c304b81b40e2f353d56b5acb1e76b45f3448678
--- lib/pack_create.c
+++ lib/pack_create.c
@@ -52,7 +52,6 @@
 #include "got_lib_deflate.h"
 #include "got_lib_pack.h"
 #include "got_lib_pack_create.h"
-#include "got_lib_privsep.h"
 #include "got_lib_repository.h"
 #include "got_lib_ratelimit.h"
 #include "got_lib_inflate.h"
@@ -71,38 +70,6 @@ struct got_pack_meta {
 #define nitems(_a)	(sizeof((_a)) / sizeof((_a)[0]))
 #endif
 
-struct got_pack_meta {
-	struct got_object_id id;
-	uint32_t path_hash;
-	int	obj_type;
-	off_t	size;
-	time_t	mtime;
-
-	/* The best delta we picked */
-	struct got_pack_meta *head;
-	struct got_pack_meta *prev;
-	unsigned char *delta_buf; /* if encoded in memory (compressed) */
-	off_t	delta_offset;	/* offset in delta cache file (compressed) */
-	off_t	delta_len;	/* encoded delta length */
-	off_t	delta_compressed_len; /* encoded+compressed delta length */
-	int	nchain;
-
-	off_t   reused_delta_offset; /* offset of delta in reused pack file */
-	struct got_object_id *base_obj_id;
-
-	/* Only used for delta window */
-	struct got_delta_table *dtab;
-
-	/* Only used for writing offset deltas */
-	off_t	off;
-};
-
-struct got_pack_metavec {
-	struct got_pack_meta **meta;
-	int nmeta;
-	int metasz;
-};
-
 static const struct got_error *
 alloc_meta(struct got_pack_meta **new, struct got_object_id *id,
     const char *path, int obj_type, time_t mtime, uint32_t seed)
@@ -466,8 +433,8 @@ static const struct got_error *
 	return err;
 }
 
-static const struct got_error *
-report_progress(got_pack_progress_cb progress_cb, void *progress_arg,
+const struct got_error *
+got_pack_report_progress(got_pack_progress_cb progress_cb, void *progress_arg,
     struct got_ratelimit *rl, int ncolored, int nfound, int ntrees,
     off_t packfile_size, int ncommits, int nobj_total, int obj_deltify,
     int nobj_written)
@@ -486,8 +453,8 @@ static const struct got_error *
 	    packfile_size, ncommits, nobj_total, obj_deltify, nobj_written);
 }
 
-static const struct got_error *
-add_meta(struct got_pack_meta *m, struct got_pack_metavec *v)
+const struct got_error *
+got_pack_add_meta(struct got_pack_meta *m, struct got_pack_metavec *v)
 {
 	if (v->nmeta == v->metasz){
 		size_t newsize = 2 * v->metasz;
@@ -503,8 +470,8 @@ static const struct got_error *
 	return NULL;
 }
 
-static const struct got_error *
-find_pack_for_reuse(struct got_packidx **best_packidx,
+const struct got_error *
+got_pack_find_pack_for_reuse(struct got_packidx **best_packidx,
     struct got_repository *repo)
 {
 	const struct got_error *err = NULL;
@@ -538,89 +505,11 @@ struct send_id_arg {
 	return err;
 }
 
-struct send_id_arg {
-	struct imsgbuf *ibuf;
-	struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS];
-	size_t nids;
-};
-
-static const struct got_error *
-send_id(struct got_object_id *id, void *data, void *arg)
+const struct got_error *
+got_pack_cache_pack_for_packidx(struct got_pack **pack,
+    struct got_packidx *packidx, struct got_repository *repo)
 {
-	const struct got_error *err = NULL;
-	struct send_id_arg *a = arg;
-
-	a->ids[a->nids++] = id;
-
-	if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) {
-		err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids);
-		if (err)
-			return err;
-		a->nids = 0;
-	}
-
-	return NULL;
-}
-
-static const struct got_error *
-send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset)
-{
 	const struct got_error *err;
-	struct send_id_arg sia;
-
-	memset(&sia, 0, sizeof(sia));
-	sia.ibuf = ibuf;
-	err = got_object_idset_for_each(idset, send_id, &sia);
-	if (err)
-		return err;
-
-	if (sia.nids > 0) {
-		err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids);
-		if (err)
-			return err;
-	}
-
-	return got_privsep_send_object_idlist_done(ibuf);
-}
-
-
-static const struct got_error *
-recv_reused_delta(struct got_imsg_reused_delta *delta,
-    struct got_object_idset *idset, struct got_pack_metavec *v)
-{
-	struct got_pack_meta *m, *base;
-
-	if (delta->delta_offset + delta->delta_size < delta->delta_offset ||
-	    delta->delta_offset +
-	    delta->delta_compressed_size < delta->delta_offset)
-		return got_error(GOT_ERR_BAD_PACKFILE);
-
-	m = got_object_idset_get(idset, &delta->id);
-	if (m == NULL)
-		return got_error(GOT_ERR_NO_OBJ);
-
-	base = got_object_idset_get(idset, &delta->base_id);
-	if (base == NULL)
-		return got_error(GOT_ERR_NO_OBJ);
-
-	m->delta_len = delta->delta_size;
-	m->delta_compressed_len = delta->delta_compressed_size;
-	m->delta_offset = delta->delta_out_offset;
-	m->prev = base;
-	m->size = delta->result_size;
-	m->reused_delta_offset = delta->delta_offset;
-	m->base_obj_id = got_object_id_dup(&delta->base_id);
-	if (m->base_obj_id == NULL)
-		return got_error_from_errno("got_object_id_dup");
-
-	return add_meta(m, v);
-}
-
-static const struct got_error *
-cache_pack_for_packidx(struct got_pack **pack, struct got_packidx *packidx,
-    struct got_repository *repo)
-{
-	const struct got_error *err;
 	char *path_packfile = NULL;
 
 	err = got_packidx_get_packfile_path(&path_packfile,
@@ -634,106 +523,12 @@ cache_pack_for_packidx(struct got_pack **pack, struct 
 		if (err)
 			goto done;
 	}
-	if ((*pack)->privsep_child == NULL) {
-		err = got_pack_start_privsep_child(*pack, packidx);
-		if (err)
-			goto done;
-	}
 done:
 	free(path_packfile);
 	return err;
 }
 
 static const struct got_error *
-prepare_delta_reuse(struct got_pack *pack, struct got_packidx *packidx,
-    int delta_outfd, struct got_repository *repo)
-{
-	const struct got_error *err = NULL;
-
-	if (!pack->child_has_delta_outfd) {
-		int outfd_child;
-		outfd_child = dup(delta_outfd);
-		if (outfd_child == -1) {
-			err = got_error_from_errno("dup");
-			goto done;
-		}
-		err = got_privsep_send_raw_delta_outfd(
-		    pack->privsep_child->ibuf, outfd_child);
-		if (err)
-			goto done;
-		pack->child_has_delta_outfd = 1;
-	}
-
-	err = got_privsep_send_delta_reuse_req(pack->privsep_child->ibuf);
-done:
-	return err;
-}
-
-
-static const struct got_error *
-search_deltas(struct got_pack_metavec *v, struct got_object_idset *idset,
-    int delta_cache_fd, int ncolored, int nfound, int ntrees, int ncommits,
-    struct got_repository *repo,
-    got_pack_progress_cb progress_cb, void *progress_arg,
-    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
-{
-	const struct got_error *err = NULL;
-	struct got_packidx *packidx;
-	struct got_pack *pack;
-	struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
-	size_t ndeltas, i;
-
-	err = find_pack_for_reuse(&packidx, repo);
-	if (err)
-		return err;
-
-	if (packidx == NULL)
-		return NULL;
-
-	err = cache_pack_for_packidx(&pack, packidx, repo);
-	if (err)
-		return err;
-
-	err = prepare_delta_reuse(pack, packidx, delta_cache_fd, repo);
-	if (err)
-		return err;
-
-	err = send_idset(pack->privsep_child->ibuf, idset);
-	if (err)
-		return err;
-
-	for (;;) {
-		int done = 0;
-
-		if (cancel_cb) {
-			err = (*cancel_cb)(cancel_arg);
-			if (err)
-				break;
-		}
-
-		err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas,
-		    pack->privsep_child->ibuf);
-		if (err || done)
-			break;
-
-		for (i = 0; i < ndeltas; i++) {
-			struct got_imsg_reused_delta *delta = &deltas[i];
-			err = recv_reused_delta(delta, idset, v);
-			if (err)
-				goto done;
-		}
-
-		err = report_progress(progress_cb, progress_arg, rl,
-		    ncolored, nfound, ntrees, 0L, ncommits,
-		    got_object_idset_num_elements(idset), v->nmeta, 0);
-		if (err)
-			break;
-	}
-done:
-	return err;
-}
-
-static const struct got_error *
 pick_deltas(struct got_pack_meta **meta, int nmeta, int ncolored,
     int nfound, int ntrees, int ncommits, int nreused, FILE *delta_cache,
     struct got_repository *repo,
@@ -761,7 +556,7 @@ pick_deltas(struct got_pack_meta **meta, int nmeta, in
 			if (err)
 				break;
 		}
-		err = report_progress(progress_cb, progress_arg, rl,
+		err = got_pack_report_progress(progress_cb, progress_arg, rl,
 		    ncolored, nfound, ntrees, 0L, ncommits, nreused + nmeta,
 		    nreused + i, 0);
 		if (err)
@@ -924,8 +719,8 @@ static const struct got_error *
 	return err;
 }
 
-static const struct got_error *
-add_object(int want_meta, struct got_object_idset *idset,
+const struct got_error *
+got_pack_add_object(int want_meta, struct got_object_idset *idset,
     struct got_object_id *id, const char *path, int obj_type,
     time_t mtime, uint32_t seed, int loose_obj_only,
     struct got_repository *repo, int *ncolored, int *nfound, int *ntrees,
@@ -950,7 +745,7 @@ add_object(int want_meta, struct got_object_idset *ids
 			return err;
 
 		(*nfound)++;
-		err = report_progress(progress_cb, progress_arg, rl,
+		err = got_pack_report_progress(progress_cb, progress_arg, rl,
 		    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
 		if (err) {
 			clear_meta(m);
@@ -967,8 +762,8 @@ static const struct got_error *
 	return err;
 }
 
-static const struct got_error *
-load_tree_entries(struct got_object_id_queue *ids, int want_meta,
+const struct got_error *
+got_pack_load_tree_entries(struct got_object_id_queue *ids, int want_meta,
     struct got_object_idset *idset, struct got_object_idset *idset_exclude,
     struct got_tree_object *tree,
     const char *dpath, time_t mtime, uint32_t seed, struct got_repository *repo,
@@ -981,7 +776,7 @@ load_tree_entries(struct got_object_id_queue *ids, int
 	int i;
 
 	(*ntrees)++;
-	err = report_progress(progress_cb, progress_arg, rl,
+	err = got_pack_report_progress(progress_cb, progress_arg, rl,
 	    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
 	if (err)
 		return err;
@@ -1025,7 +820,7 @@ load_tree_entries(struct got_object_id_queue *ids, int
 			p = NULL;
 			STAILQ_INSERT_TAIL(ids, qid, entry);
 		} else if (S_ISREG(mode) || S_ISLNK(mode)) {
-			err = add_object(want_meta,
+			err = got_pack_add_object(want_meta,
 			    want_meta ? idset : idset_exclude, id, p,
 			    GOT_OBJ_TYPE_BLOB, mtime, seed, loose_obj_only,
 			    repo, ncolored, nfound, ntrees,
@@ -1044,8 +839,8 @@ static const struct got_error *
 	return err;
 }
 
-static const struct got_error *
-load_tree(int want_meta, struct got_object_idset *idset,
+const struct got_error *
+got_pack_load_tree(int want_meta, struct got_object_idset *idset,
     struct got_object_idset *idset_exclude,
     struct got_object_id *tree_id, const char *dpath, time_t mtime,
     uint32_t seed, struct got_repository *repo, int loose_obj_only,
@@ -1094,7 +889,8 @@ load_tree(int want_meta, struct got_object_idset *idse
 			continue;
 		}
 
-		err = add_object(want_meta, want_meta ? idset : idset_exclude,
+		err = got_pack_add_object(want_meta,
+		    want_meta ? idset : idset_exclude,
 		    &qid->id, path, GOT_OBJ_TYPE_TREE,
 		    mtime, seed, loose_obj_only, repo,
 		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl);
@@ -1111,7 +907,7 @@ load_tree(int want_meta, struct got_object_idset *idse
 			break;
 		}
 
-		err = load_tree_entries(&tree_ids, want_meta, idset,
+		err = got_pack_load_tree_entries(&tree_ids, want_meta, idset,
 		    idset_exclude, tree, path, mtime, seed, repo,
 		    loose_obj_only, ncolored, nfound, ntrees,
 		    progress_cb, progress_arg, rl,
@@ -1161,15 +957,15 @@ load_commit(int want_meta, struct got_object_idset *id
 	if (err)
 		return err;
 
-	err = add_object(want_meta, want_meta ? idset : idset_exclude,
-	    id, "", GOT_OBJ_TYPE_COMMIT,
+	err = got_pack_add_object(want_meta,
+	    want_meta ? idset : idset_exclude, id, "", GOT_OBJ_TYPE_COMMIT,
 	    got_object_commit_get_committer_time(commit), seed,
 	    loose_obj_only, repo,
 	    ncolored, nfound, ntrees, progress_cb, progress_arg, rl);
 	if (err)
 		goto done;
 
-	err = load_tree(want_meta, idset, idset_exclude,
+	err = got_pack_load_tree(want_meta, idset, idset_exclude,
 	    got_object_commit_get_tree_id(commit),
 	    "", got_object_commit_get_committer_time(commit), seed,
 	    repo, loose_obj_only, ncolored, nfound, ntrees,
@@ -1207,8 +1003,8 @@ load_tag(int want_meta, struct got_object_idset *idset
 	if (err)
 		return err;
 
-	err = add_object(want_meta, want_meta ? idset : idset_exclude,
-	    id, "", GOT_OBJ_TYPE_TAG,
+	err = got_pack_add_object(want_meta,
+	    want_meta ? idset : idset_exclude, id, "", GOT_OBJ_TYPE_TAG,
 	    got_object_tag_get_tagger_time(tag), seed, loose_obj_only, repo,
 	    ncolored, nfound, ntrees, progress_cb, progress_arg, rl);
 	if (err)
@@ -1222,7 +1018,7 @@ load_tag(int want_meta, struct got_object_idset *idset
 		    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
 		break;
 	case GOT_OBJ_TYPE_TREE:
-		err = load_tree(want_meta, idset, idset_exclude,
+		err = got_pack_load_tree(want_meta, idset, idset_exclude,
 		    got_object_tag_get_object_id(tag), "",
 		    got_object_tag_get_tagger_time(tag), seed, repo,
 		    loose_obj_only, ncolored, nfound, ntrees,
@@ -1237,15 +1033,8 @@ enum findtwixt_color {
 	return err;
 }
 
-enum findtwixt_color {
-	COLOR_KEEP = 0,
-	COLOR_DROP,
-	COLOR_SKIP,
-	COLOR_MAX,
-};
-
-static const struct got_error *
-paint_commit(struct got_object_qid *qid, intptr_t color)
+const struct got_error *
+got_pack_paint_commit(struct got_object_qid *qid, intptr_t color)
 {
 	if (color < 0 || color >= COLOR_MAX)
 		return got_error(GOT_ERR_RANGE);
@@ -1254,9 +1043,9 @@ static const struct got_error *
 	return NULL;
 }
 
-static const struct got_error *
-queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id,
-    intptr_t color, struct got_repository *repo)
+const struct got_error *
+got_pack_queue_commit_id(struct got_object_id_queue *ids,
+    struct got_object_id *id, intptr_t color, struct got_repository *repo)
 {
 	const struct got_error *err;
 	struct got_object_qid *qid;
@@ -1266,7 +1055,7 @@ queue_commit_id(struct got_object_id_queue *ids, struc
 		return err;
 
 	STAILQ_INSERT_TAIL(ids, qid, entry);
-	return paint_commit(qid, color);
+	return got_pack_paint_commit(qid, color);
 }
 
 struct append_id_arg {
@@ -1313,7 +1102,7 @@ queue_commit_or_tag_id(struct got_object_id *id, intpt
 	}
 
 	if (obj_type == GOT_OBJ_TYPE_COMMIT) {
-		err = queue_commit_id(ids, id, color, repo);
+		err = got_pack_queue_commit_id(ids, id, color, repo);
 		if (err)
 			goto done;
 	}
@@ -1323,148 +1112,8 @@ struct recv_painted_commit_arg {
 	return err;
 }
 
-struct recv_painted_commit_arg {
-	int *ncolored;
-	int *nqueued;
-	int *nskip;
-	struct got_object_id_queue *ids;
-	struct got_object_idset *keep;
-	struct got_object_idset *drop;
-	struct got_object_idset *skip;
-	got_pack_progress_cb progress_cb;
-	void *progress_arg;
-	struct got_ratelimit *rl;
-	got_cancel_cb cancel_cb;
-	void *cancel_arg;
-};
-
-static const struct got_error *
-recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color)
-{
-	const struct got_error *err = NULL;
-	struct recv_painted_commit_arg *a = arg;
-	struct got_object_qid *qid, *tmp;
-
-	if (a->cancel_cb) {
-		err = a->cancel_cb(a->cancel_arg);
-		if (err)
-			return err;
-	}
-
-	switch (color) {
-	case COLOR_KEEP:
-		err = got_object_idset_add(a->keep, id, NULL);
-		if (err)
-			return err;
-		(*a->ncolored)++;
-		break;
-	case COLOR_DROP:
-		err = got_object_idset_add(a->drop, id, NULL);
-		if (err)
-			return err;
-		(*a->ncolored)++;
-		break;
-	case COLOR_SKIP:
-		err = got_object_idset_add(a->skip, id, NULL);
-		if (err)
-			return err;
-		break;
-	default:
-		/* should not happen */
-		return got_error_fmt(GOT_ERR_NOT_IMPL,
-		    "%s invalid commit color %"PRIdPTR, __func__, color);
-	}
-
-	STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) {
-		if (got_object_id_cmp(&qid->id, id) != 0)
-			continue;
-		STAILQ_REMOVE(a->ids, qid, got_object_qid, entry);
-		color = (intptr_t)qid->data;
-		got_object_qid_free(qid);
-		(*a->nqueued)--;
-		if (color == COLOR_SKIP)
-			(*a->nskip)--;
-		break;
-	}
-
-	return report_progress(a->progress_cb, a->progress_arg, a->rl,
-	    *a->ncolored, 0, 0, 0L, 0, 0, 0, 0);
-}
-
-static const struct got_error *
-paint_packed_commits(struct got_pack *pack, struct got_object_id *id,
-    int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip,
-    struct got_object_id_queue *ids,
-    struct got_object_idset *keep, struct got_object_idset *drop,
-    struct got_object_idset *skip, struct got_repository *repo,
-    got_pack_progress_cb progress_cb, void *progress_arg,
-    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
-{
-	const struct got_error *err = NULL;
-	struct got_object_id_queue next_ids;
-	struct got_object_qid *qid, *tmp;
-	struct recv_painted_commit_arg arg;
-
-	STAILQ_INIT(&next_ids);
-
-	err = got_privsep_send_painting_request(pack->privsep_child->ibuf,
-	    idx, id, color);
-	if (err)
-		return err;
-
-	arg.ncolored = ncolored;
-	arg.nqueued = nqueued;
-	arg.nskip = nskip;
-	arg.ids = ids;
-	arg.keep = keep;
-	arg.drop = drop;
-	arg.skip = skip;
-	arg.progress_cb = progress_cb;
-	arg.progress_arg = progress_arg;
-	arg.rl = rl;
-	arg.cancel_cb = cancel_cb;
-	arg.cancel_arg = cancel_arg;
-	err = got_privsep_recv_painted_commits(&next_ids,
-	    recv_painted_commit, &arg, pack->privsep_child->ibuf);
-	if (err)
-		return err;
-
-	STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) {
-		struct got_object_qid *old_id;
-		intptr_t qcolor, ocolor;
-		STAILQ_FOREACH(old_id, ids, entry) {
-			if (got_object_id_cmp(&qid->id, &old_id->id))
-				continue;
-			qcolor = (intptr_t)qid->data;
-			ocolor = (intptr_t)old_id->data;
-			STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry);
-			got_object_qid_free(qid);
-			qid = NULL;
-			if (qcolor != ocolor) {
-				paint_commit(old_id, qcolor);
-				if (ocolor == COLOR_SKIP)
-					(*nskip)--;
-				else if (qcolor == COLOR_SKIP)
-					(*nskip)++;
-			}
-			break;
-		}
-	}
-	while (!STAILQ_EMPTY(&next_ids)) {
-		qid = STAILQ_FIRST(&next_ids);
-		STAILQ_REMOVE_HEAD(&next_ids, entry);
-		paint_commit(qid, color);
-		STAILQ_INSERT_TAIL(ids, qid, entry);
-		(*nqueued)++;
-		if (color == COLOR_SKIP)
-			(*nskip)++;
-	}
-
-	return err;
-}
-
-static const struct got_error *
-find_pack_for_commit_painting(struct got_packidx **best_packidx,
+const struct got_error *
+got_pack_find_pack_for_commit_painting(struct got_packidx **best_packidx,
     struct got_object_id_queue *ids, int nids, struct got_repository *repo)
 {
 	const struct got_error *err = NULL;
@@ -1514,191 +1163,6 @@ paint_commits(int *ncolored, struct got_object_id_queu
 }
 
 static const struct got_error *
-paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids,
-    struct got_object_idset *keep, struct got_object_idset *drop,
-    struct got_object_idset *skip, struct got_repository *repo,
-    got_pack_progress_cb progress_cb, void *progress_arg,
-    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
-{
-	const struct got_error *err = NULL;
-	struct got_commit_object *commit = NULL;
-	struct got_packidx *packidx = NULL;
-	struct got_pack *pack = NULL;
-	const struct got_object_id_queue *parents;
-	struct got_object_qid *qid = NULL;
-	int nqueued = nids, nskip = 0;
-	int idx;
-
-	while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
-		intptr_t color;
-
-		if (cancel_cb) {
-			err = cancel_cb(cancel_arg);
-			if (err)
-				break;
-		}
-
-		qid = STAILQ_FIRST(ids);
-		STAILQ_REMOVE_HEAD(ids, entry);
-		nqueued--;
-		color = (intptr_t)qid->data;
-		if (color == COLOR_SKIP)
-			nskip--;
-
-		if (got_object_idset_contains(skip, &qid->id)) {
-			got_object_qid_free(qid);
-			qid = NULL;
-			continue;
-		}
-		if (color == COLOR_KEEP &&
-		    got_object_idset_contains(keep, &qid->id)) {
-			got_object_qid_free(qid);
-			qid = NULL;
-			continue;
-		}
-		if (color == COLOR_DROP &&
-		    got_object_idset_contains(drop, &qid->id)) {
-			got_object_qid_free(qid);
-			qid = NULL;
-			continue;
-		}
-
-		/* Pinned pack may have moved to different cache slot. */
-		pack = got_repo_get_pinned_pack(repo);
-
-		if (packidx && pack) {
-			idx = got_packidx_get_object_idx(packidx, &qid->id);
-			if (idx != -1) {
-				err = paint_packed_commits(pack, &qid->id,
-				    idx, color, ncolored, &nqueued, &nskip,
-				    ids, keep, drop, skip, repo,
-				    progress_cb, progress_arg, rl,
-				    cancel_cb, cancel_arg);
-				if (err)
-					break;
-				got_object_qid_free(qid);
-				qid = NULL;
-				continue;
-			}
-		}
-
-		switch (color) {
-		case COLOR_KEEP:
-			if (got_object_idset_contains(drop, &qid->id)) {
-				err = paint_commit(qid, COLOR_SKIP);
-				if (err)
-					goto done;
-			} else
-				(*ncolored)++;
-			err = got_object_idset_add(keep, &qid->id, NULL);
-			if (err)
-				goto done;
-			break;
-		case COLOR_DROP:
-			if (got_object_idset_contains(keep, &qid->id)) {
-				err = paint_commit(qid, COLOR_SKIP);
-				if (err)
-					goto done;
-			} else
-				(*ncolored)++;
-			err = got_object_idset_add(drop, &qid->id, NULL);
-			if (err)
-				goto done;
-			break;
-		case COLOR_SKIP:
-			if (!got_object_idset_contains(skip, &qid->id)) {
-				err = got_object_idset_add(skip, &qid->id,
-				    NULL);
-				if (err)
-					goto done;
-			}
-			break;
-		default:
-			/* should not happen */
-			err = got_error_fmt(GOT_ERR_NOT_IMPL,
-			    "%s invalid commit color %"PRIdPTR, __func__,
-			    color);
-			goto done;
-		}
-
-		err = report_progress(progress_cb, progress_arg, rl,
-		    *ncolored, 0, 0, 0L, 0, 0, 0, 0);
-		if (err)
-			break;
-
-		err = got_object_open_as_commit(&commit, repo, &qid->id);
-		if (err)
-			break;
-
-		parents = got_object_commit_get_parent_ids(commit);
-		if (parents) {
-			struct got_object_qid *pid;
-			color = (intptr_t)qid->data;
-			STAILQ_FOREACH(pid, parents, entry) {
-				err = queue_commit_id(ids, &pid->id,
-				    color, repo);
-				if (err)
-					break;
-				nqueued++;
-				if (color == COLOR_SKIP)
-					nskip++;
-			}
-		}
-
-		if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) {
-			if (packidx == NULL) {
-				err = find_pack_for_commit_painting(&packidx,
-				    ids, nqueued, repo);
-				if (err)
-					goto done;
-			}
-			if (packidx != NULL) {
-				err = cache_pack_for_packidx(&pack, packidx,
-				    repo);
-				if (err)
-					goto done;
-				err = got_privsep_init_commit_painting(
-				    pack->privsep_child->ibuf);
-				if (err)
-					goto done;
-				err = send_idset(pack->privsep_child->ibuf,
-				    keep);
-				if (err)
-					goto done;
-				err = send_idset(pack->privsep_child->ibuf, drop);
-				if (err)
-					goto done;
-				err = send_idset(pack->privsep_child->ibuf, skip);
-				if (err)
-					goto done;
-				err = got_repo_pin_pack(repo, packidx, pack);
-				if (err)
-					goto done;
-			}
-		}
-
-		got_object_commit_close(commit);
-		commit = NULL;
-
-		got_object_qid_free(qid);
-		qid = NULL;
-	}
-done:
-	if (pack) {
-		const struct got_error *pack_err;
-		pack_err = got_privsep_send_painting_commits_done(
-		    pack->privsep_child->ibuf);
-		if (err == NULL)
-			err = pack_err;
-	}
-	if (commit)
-		got_object_commit_close(commit);
-	got_object_qid_free(qid);
-	got_repo_unpin_pack(repo);
-	return err;
-}
-
-static const struct got_error *
 findtwixt(struct got_object_id ***res, int *nres, int *ncolored,
     struct got_object_id **head, int nhead,
     struct got_object_id **tail, int ntail,
@@ -1750,7 +1214,7 @@ findtwixt(struct got_object_id ***res, int *nres, int 
 			goto done;
 	}
 
-	err = paint_commits(ncolored, &ids, nhead + ntail,
+	err = got_pack_paint_commits(ncolored, &ids, nhead + ntail,
 	    keep, drop, skip, repo, progress_cb, progress_arg, rl,
 	    cancel_cb, cancel_arg);
 	if (err)
@@ -1784,159 +1248,7 @@ struct load_packed_obj_arg {
 	return err;
 }
 
-struct load_packed_obj_arg {
-	/* output parameters: */
-	struct got_object_id *id;
-	char *dpath;
-	time_t mtime;
-
-	/* input parameters: */
-	uint32_t seed;
-	int want_meta;
-	struct got_object_idset *idset;
-	struct got_object_idset *idset_exclude;
-	int loose_obj_only;
-	int *ncolored;
-	int *nfound;
-	int *ntrees;
-	got_pack_progress_cb progress_cb;
-	void *progress_arg;
-	struct got_ratelimit *rl;
-	got_cancel_cb cancel_cb;
-	void *cancel_arg;
-};
-
 static const struct got_error *
-load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id,
-    struct got_repository *repo)
-{
-	struct load_packed_obj_arg *a = arg;
-
-	if (got_object_idset_contains(a->idset, id) ||
-	    got_object_idset_contains(a->idset_exclude, id))
-		return NULL;
-
-	return add_object(a->want_meta,
-	    a->want_meta ? a->idset : a->idset_exclude,
-	    id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only,
-	    repo, a->ncolored, a->nfound, a->ntrees,
-	    a->progress_cb, a->progress_arg, a->rl);
-}
-
-static const struct got_error *
-load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime,
-    struct got_object_id *id, const char *dpath, struct got_repository *repo)
-{
-	const struct got_error *err;
-	struct load_packed_obj_arg *a = arg;
-	const char *relpath;
-
-	/*
-	 * When we receive a tree's ID and path but not the tree itself,
-	 * this tree object was not found in the pack file. This is the
-	 * last time we are being called for this optimized traversal.
-	 * Return from here and switch to loading objects the slow way.
-	 */
-	if (tree == NULL) {
-		free(a->id);
-		a->id = got_object_id_dup(id);
-		if (a->id == NULL) {
-			err = got_error_from_errno("got_object_id_dup");
-			free(a->dpath);
-			a->dpath = NULL;
-			return err;
-		}
-
-		free(a->dpath);
-		a->dpath = strdup(dpath);
-		if (a->dpath == NULL) {
-			err = got_error_from_errno("strdup");
-			free(a->id);
-			a->id = NULL;
-			return err;
-		}
-
-		a->mtime = mtime;
-		return NULL;
-	}
-
-	if (got_object_idset_contains(a->idset, id) ||
-	    got_object_idset_contains(a->idset_exclude, id))
-		return NULL;
-
-	relpath = dpath;
-	while (relpath[0] == '/')
-		relpath++;
-
-	err = add_object(a->want_meta,
-	    a->want_meta ? a->idset : a->idset_exclude,
-	    id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed,
-	    a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees,
-	    a->progress_cb, a->progress_arg, a->rl);
-	if (err)
-		return err;
-
-	return load_tree_entries(NULL, a->want_meta, a->idset,
-	    a->idset_exclude, tree, dpath, mtime, a->seed, repo,
-	    a->loose_obj_only, a->ncolored, a->nfound, a->ntrees,
-	    a->progress_cb, a->progress_arg, a->rl,
-	    a->cancel_cb, a->cancel_arg);
-}
-
-static const struct got_error *
-load_packed_object_ids(int *found_all_objects,
-    struct got_object_id **ours, int nours,
-    struct got_object_id **theirs, int ntheirs,
-    int want_meta, uint32_t seed, struct got_object_idset *idset,
-    struct got_object_idset *idset_exclude, int loose_obj_only,
-    struct got_repository *repo, struct got_packidx *packidx,
-    int *ncolored, int *nfound, int *ntrees,
-    got_pack_progress_cb progress_cb, void *progress_arg,
-    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
-{
-	const struct got_error *err = NULL;
-	struct load_packed_obj_arg lpa;
-
-	memset(&lpa, 0, sizeof(lpa));
-	lpa.seed = seed;
-	lpa.want_meta = want_meta;
-	lpa.idset = idset;
-	lpa.idset_exclude = idset_exclude;
-	lpa.loose_obj_only = loose_obj_only;
-	lpa.ncolored = ncolored;
-	lpa.nfound = nfound;
-	lpa.ntrees = ntrees;
-	lpa.progress_cb = progress_cb;
-	lpa.progress_arg = progress_arg;
-	lpa.rl = rl;
-	lpa.cancel_cb = cancel_cb;
-	lpa.cancel_arg = cancel_arg;
-
-	/* Attempt to load objects via got-read-pack, as far as possible. */
-	err = got_object_enumerate(found_all_objects, load_packed_commit_id,
-	   load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs,
-	   packidx, repo);
-	if (err)
-		return err;
-
-	if (lpa.id == NULL)
-		return NULL;
-
-	/*
-	 * An incomplete tree hierarchy was present in the pack file
-	 * and caused loading to be aborted.
-	 * Continue loading trees the slow way.
-	 */
-	err = load_tree(want_meta, idset, idset_exclude,
-	    lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only,
-	    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
-	    cancel_cb, cancel_arg);
-	free(lpa.id);
-	free(lpa.dpath);
-	return err;
-}
-
-static const struct got_error *
 find_pack_for_enumeration(struct got_packidx **best_packidx,
     struct got_object_id **ids, int nids, struct got_repository *repo)
 {
@@ -2016,7 +1328,7 @@ load_object_ids(int *ncolored, int *nfound, int *ntree
 	if (err)
 		goto done;
 	if (packidx) {
-		err = load_packed_object_ids(&found_all_objects,
+		err = got_pack_load_packed_object_ids(&found_all_objects,
 		    theirs, ntheirs, NULL, 0, 0, seed, idset, idset_exclude,
 		    loose_obj_only, repo, packidx, ncolored, nfound, ntrees,
 		    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
@@ -2056,7 +1368,7 @@ load_object_ids(int *ncolored, int *nfound, int *ntree
 	if (err)
 		goto done;
 	if (packidx) {
-		err = load_packed_object_ids(&found_all_objects, ids,
+		err = got_pack_load_packed_object_ids(&found_all_objects, ids,
 		    nobj, theirs, ntheirs, 1, seed, idset, idset_exclude,
 		    loose_obj_only, repo, packidx, ncolored, nfound, ntrees,
 		    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
@@ -2440,7 +1752,7 @@ genpack(uint8_t *pack_sha1, int packfd, FILE *delta_ca
 	qsort(deltify, ndeltify, sizeof(struct got_pack_meta *),
 	    write_order_cmp);
 	for (i = 0; i < ndeltify; i++) {
-		err = report_progress(progress_cb, progress_arg, rl,
+		err = got_pack_report_progress(progress_cb, progress_arg, rl,
 		    ncolored, nfound, ntrees, packfile_size, nours,
 		    ndeltify + nreuse, ndeltify + nreuse, i);
 		if (err)
@@ -2456,7 +1768,7 @@ genpack(uint8_t *pack_sha1, int packfd, FILE *delta_ca
 	qsort(reuse, nreuse, sizeof(struct got_pack_meta *),
 	    reuse_write_order_cmp);
 	for (i = 0; i < nreuse; i++) {
-		err = report_progress(progress_cb, progress_arg, rl,
+		err = got_pack_report_progress(progress_cb, progress_arg, rl,
 		    ncolored, nfound, ntrees, packfile_size, nours,
 		    ndeltify + nreuse, ndeltify + nreuse, ndeltify + i);
 		if (err)
@@ -2505,7 +1817,7 @@ add_meta_idset_cb(struct got_object_id *id, void *data
 	if (m->reused_delta_offset != 0)
 		return NULL;
 
-	return add_meta(m, v);
+	return got_pack_add_meta(m, v);
 }
 
 const struct got_error *
@@ -2568,9 +1880,9 @@ got_pack_create(uint8_t *packsha1, int packfd, FILE *d
 		goto done;
 	}
 
-	err = search_deltas(&reuse, idset, delta_cache_fd, ncolored, nfound,
-	    ntrees, nours, repo, progress_cb, progress_arg, &rl,
-	    cancel_cb, cancel_arg);
+	err = got_pack_search_deltas(&reuse, idset, delta_cache_fd,
+	    ncolored, nfound, ntrees, nours,
+	    repo, progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
 	if (err)
 		goto done;
 
blob - /dev/null
blob + 5180371c65007227a78d76ba0d4b63cd2ee65675 (mode 644)
--- /dev/null
+++ lib/pack_create_privsep.c
@@ -0,0 +1,702 @@
+/*
+ * Copyright (c) 2020 Ori Bernstein
+ * Copyright (c) 2021, 2022 Stefan Sperling <stsp@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <sys/tree.h>
+#include <sys/uio.h>
+
+#include <sha1.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <imsg.h>
+#include <inttypes.h>
+#include <unistd.h>
+
+#include "got_error.h"
+#include "got_cancel.h"
+#include "got_object.h"
+#include "got_reference.h"
+#include "got_repository_admin.h"
+#include "got_path.h"
+
+#include "got_lib_delta.h"
+#include "got_lib_object.h"
+#include "got_lib_object_cache.h"
+#include "got_lib_object_idset.h"
+#include "got_lib_privsep.h"
+#include "got_lib_pack.h"
+#include "got_lib_pack_create.h"
+#include "got_lib_repository.h"
+
+struct send_id_arg {
+	struct imsgbuf *ibuf;
+	struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS];
+	size_t nids;
+};
+
+static const struct got_error *
+send_id(struct got_object_id *id, void *data, void *arg)
+{
+	const struct got_error *err = NULL;
+	struct send_id_arg *a = arg;
+
+	a->ids[a->nids++] = id;
+
+	if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) {
+		err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids);
+		if (err)
+			return err;
+		a->nids = 0;
+	}
+
+	return NULL;
+}
+
+static const struct got_error *
+send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset)
+{
+	const struct got_error *err;
+	struct send_id_arg sia;
+
+	memset(&sia, 0, sizeof(sia));
+	sia.ibuf = ibuf;
+	err = got_object_idset_for_each(idset, send_id, &sia);
+	if (err)
+		return err;
+
+	if (sia.nids > 0) {
+		err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids);
+		if (err)
+			return err;
+	}
+
+	return got_privsep_send_object_idlist_done(ibuf);
+}
+
+static const struct got_error *
+recv_reused_delta(struct got_imsg_reused_delta *delta,
+    struct got_object_idset *idset, struct got_pack_metavec *v)
+{
+	struct got_pack_meta *m, *base;
+
+	if (delta->delta_offset + delta->delta_size < delta->delta_offset ||
+	    delta->delta_offset +
+	    delta->delta_compressed_size < delta->delta_offset)
+		return got_error(GOT_ERR_BAD_PACKFILE);
+
+	m = got_object_idset_get(idset, &delta->id);
+	if (m == NULL)
+		return got_error(GOT_ERR_NO_OBJ);
+
+	base = got_object_idset_get(idset, &delta->base_id);
+	if (base == NULL)
+		return got_error(GOT_ERR_NO_OBJ);
+
+	m->delta_len = delta->delta_size;
+	m->delta_compressed_len = delta->delta_compressed_size;
+	m->delta_offset = delta->delta_out_offset;
+	m->prev = base;
+	m->size = delta->result_size;
+	m->reused_delta_offset = delta->delta_offset;
+	m->base_obj_id = got_object_id_dup(&delta->base_id);
+	if (m->base_obj_id == NULL)
+		return got_error_from_errno("got_object_id_dup");
+
+	return got_pack_add_meta(m, v);
+}
+
+static const struct got_error *
+prepare_delta_reuse(struct got_pack *pack, struct got_packidx *packidx,
+    int delta_outfd, struct got_repository *repo)
+{
+	const struct got_error *err = NULL;
+
+	if (!pack->child_has_delta_outfd) {
+		int outfd_child;
+		outfd_child = dup(delta_outfd);
+		if (outfd_child == -1) {
+			err = got_error_from_errno("dup");
+			goto done;
+		}
+		err = got_privsep_send_raw_delta_outfd(
+		    pack->privsep_child->ibuf, outfd_child);
+		if (err)
+			goto done;
+		pack->child_has_delta_outfd = 1;
+	}
+
+	err = got_privsep_send_delta_reuse_req(pack->privsep_child->ibuf);
+done:
+	return err;
+}
+
+const struct got_error *
+got_pack_search_deltas(struct got_pack_metavec *v,
+    struct got_object_idset *idset, int delta_cache_fd,
+    int ncolored, int nfound, int ntrees, int ncommits,
+    struct got_repository *repo,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
+{
+	const struct got_error *err = NULL;
+	struct got_packidx *packidx;
+	struct got_pack *pack;
+	struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS];
+	size_t ndeltas, i;
+
+	err = got_pack_find_pack_for_reuse(&packidx, repo);
+	if (err)
+		return err;
+
+	if (packidx == NULL)
+		return NULL;
+
+	err = got_pack_cache_pack_for_packidx(&pack, packidx, repo);
+	if (err)
+		return err;
+
+	if (pack->privsep_child == NULL) {
+		err = got_pack_start_privsep_child(pack, packidx);
+		if (err)
+			return err;
+	}
+
+	err = prepare_delta_reuse(pack, packidx, delta_cache_fd, repo);
+	if (err)
+		return err;
+
+	err = send_idset(pack->privsep_child->ibuf, idset);
+	if (err)
+		return err;
+
+	for (;;) {
+		int done = 0;
+
+		if (cancel_cb) {
+			err = (*cancel_cb)(cancel_arg);
+			if (err)
+				break;
+		}
+
+		err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas,
+		    pack->privsep_child->ibuf);
+		if (err || done)
+			break;
+
+		for (i = 0; i < ndeltas; i++) {
+			struct got_imsg_reused_delta *delta = &deltas[i];
+			err = recv_reused_delta(delta, idset, v);
+			if (err)
+				goto done;
+		}
+
+		err = got_pack_report_progress(progress_cb, progress_arg, rl,
+		    ncolored, nfound, ntrees, 0L, ncommits,
+		    got_object_idset_num_elements(idset), v->nmeta, 0);
+		if (err)
+			break;
+	}
+done:
+	return err;
+}
+
+struct recv_painted_commit_arg {
+	int *ncolored;
+	int *nqueued;
+	int *nskip;
+	struct got_object_id_queue *ids;
+	struct got_object_idset *keep;
+	struct got_object_idset *drop;
+	struct got_object_idset *skip;
+	got_pack_progress_cb progress_cb;
+	void *progress_arg;
+	struct got_ratelimit *rl;
+	got_cancel_cb cancel_cb;
+	void *cancel_arg;
+};
+
+static const struct got_error *
+recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color)
+{
+	const struct got_error *err = NULL;
+	struct recv_painted_commit_arg *a = arg;
+	struct got_object_qid *qid, *tmp;
+
+	if (a->cancel_cb) {
+		err = a->cancel_cb(a->cancel_arg);
+		if (err)
+			return err;
+	}
+
+	switch (color) {
+	case COLOR_KEEP:
+		err = got_object_idset_add(a->keep, id, NULL);
+		if (err)
+			return err;
+		(*a->ncolored)++;
+		break;
+	case COLOR_DROP:
+		err = got_object_idset_add(a->drop, id, NULL);
+		if (err)
+			return err;
+		(*a->ncolored)++;
+		break;
+	case COLOR_SKIP:
+		err = got_object_idset_add(a->skip, id, NULL);
+		if (err)
+			return err;
+		break;
+	default:
+		/* should not happen */
+		return got_error_fmt(GOT_ERR_NOT_IMPL,
+		    "%s invalid commit color %"PRIdPTR, __func__, color);
+	}
+
+	STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) {
+		if (got_object_id_cmp(&qid->id, id) != 0)
+			continue;
+		STAILQ_REMOVE(a->ids, qid, got_object_qid, entry);
+		color = (intptr_t)qid->data;
+		got_object_qid_free(qid);
+		(*a->nqueued)--;
+		if (color == COLOR_SKIP)
+			(*a->nskip)--;
+		break;
+	}
+
+	return got_pack_report_progress(a->progress_cb, a->progress_arg, a->rl,
+	    *a->ncolored, 0, 0, 0L, 0, 0, 0, 0);
+}
+
+static const struct got_error *
+paint_packed_commits(struct got_pack *pack, struct got_object_id *id,
+    int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip,
+    struct got_object_id_queue *ids,
+    struct got_object_idset *keep, struct got_object_idset *drop,
+    struct got_object_idset *skip, struct got_repository *repo,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
+{
+	const struct got_error *err = NULL;
+	struct got_object_id_queue next_ids;
+	struct got_object_qid *qid, *tmp;
+	struct recv_painted_commit_arg arg;
+
+	STAILQ_INIT(&next_ids);
+
+	err = got_privsep_send_painting_request(pack->privsep_child->ibuf,
+	    idx, id, color);
+	if (err)
+		return err;
+
+	arg.ncolored = ncolored;
+	arg.nqueued = nqueued;
+	arg.nskip = nskip;
+	arg.ids = ids;
+	arg.keep = keep;
+	arg.drop = drop;
+	arg.skip = skip;
+	arg.progress_cb = progress_cb;
+	arg.progress_arg = progress_arg;
+	arg.rl = rl;
+	arg.cancel_cb = cancel_cb;
+	arg.cancel_arg = cancel_arg;
+	err = got_privsep_recv_painted_commits(&next_ids,
+	    recv_painted_commit, &arg, pack->privsep_child->ibuf);
+	if (err)
+		return err;
+
+	STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) {
+		struct got_object_qid *old_id;
+		intptr_t qcolor, ocolor;
+		STAILQ_FOREACH(old_id, ids, entry) {
+			if (got_object_id_cmp(&qid->id, &old_id->id))
+				continue;
+			qcolor = (intptr_t)qid->data;
+			ocolor = (intptr_t)old_id->data;
+			STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry);
+			got_object_qid_free(qid);
+			qid = NULL;
+			if (qcolor != ocolor) {
+				got_pack_paint_commit(old_id, qcolor);
+				if (ocolor == COLOR_SKIP)
+					(*nskip)--;
+				else if (qcolor == COLOR_SKIP)
+					(*nskip)++;
+			}
+			break;
+		}
+	}
+	while (!STAILQ_EMPTY(&next_ids)) {
+		qid = STAILQ_FIRST(&next_ids);
+		STAILQ_REMOVE_HEAD(&next_ids, entry);
+		got_pack_paint_commit(qid, color);
+		STAILQ_INSERT_TAIL(ids, qid, entry);
+		(*nqueued)++;
+		if (color == COLOR_SKIP)
+			(*nskip)++;
+	}
+
+	return err;
+}
+
+const struct got_error *
+got_pack_paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids,
+    struct got_object_idset *keep, struct got_object_idset *drop,
+    struct got_object_idset *skip, struct got_repository *repo,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
+{
+	const struct got_error *err = NULL;
+	struct got_commit_object *commit = NULL;
+	struct got_packidx *packidx = NULL;
+	struct got_pack *pack = NULL;
+	const struct got_object_id_queue *parents;
+	struct got_object_qid *qid = NULL;
+	int nqueued = nids, nskip = 0;
+	int idx;
+
+	while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
+		intptr_t color;
+
+		if (cancel_cb) {
+			err = cancel_cb(cancel_arg);
+			if (err)
+				break;
+		}
+
+		qid = STAILQ_FIRST(ids);
+		STAILQ_REMOVE_HEAD(ids, entry);
+		nqueued--;
+		color = (intptr_t)qid->data;
+		if (color == COLOR_SKIP)
+			nskip--;
+
+		if (got_object_idset_contains(skip, &qid->id)) {
+			got_object_qid_free(qid);
+			qid = NULL;
+			continue;
+		}
+		if (color == COLOR_KEEP &&
+		    got_object_idset_contains(keep, &qid->id)) {
+			got_object_qid_free(qid);
+			qid = NULL;
+			continue;
+		}
+		if (color == COLOR_DROP &&
+		    got_object_idset_contains(drop, &qid->id)) {
+			got_object_qid_free(qid);
+			qid = NULL;
+			continue;
+		}
+
+		/* Pinned pack may have moved to different cache slot. */
+		pack = got_repo_get_pinned_pack(repo);
+
+		if (packidx && pack) {
+			idx = got_packidx_get_object_idx(packidx, &qid->id);
+			if (idx != -1) {
+				err = paint_packed_commits(pack, &qid->id,
+				    idx, color, ncolored, &nqueued, &nskip,
+				    ids, keep, drop, skip, repo,
+				    progress_cb, progress_arg, rl,
+				    cancel_cb, cancel_arg);
+				if (err)
+					break;
+				got_object_qid_free(qid);
+				qid = NULL;
+				continue;
+			}
+		}
+
+		switch (color) {
+		case COLOR_KEEP:
+			if (got_object_idset_contains(drop, &qid->id)) {
+				err = got_pack_paint_commit(qid, COLOR_SKIP);
+				if (err)
+					goto done;
+			} else
+				(*ncolored)++;
+			err = got_object_idset_add(keep, &qid->id, NULL);
+			if (err)
+				goto done;
+			break;
+		case COLOR_DROP:
+			if (got_object_idset_contains(keep, &qid->id)) {
+				err = got_pack_paint_commit(qid, COLOR_SKIP);
+				if (err)
+					goto done;
+			} else
+				(*ncolored)++;
+			err = got_object_idset_add(drop, &qid->id, NULL);
+			if (err)
+				goto done;
+			break;
+		case COLOR_SKIP:
+			if (!got_object_idset_contains(skip, &qid->id)) {
+				err = got_object_idset_add(skip, &qid->id,
+				    NULL);
+				if (err)
+					goto done;
+			}
+			break;
+		default:
+			/* should not happen */
+			err = got_error_fmt(GOT_ERR_NOT_IMPL,
+			    "%s invalid commit color %"PRIdPTR, __func__,
+			    color);
+			goto done;
+		}
+
+		err = got_pack_report_progress(progress_cb, progress_arg, rl,
+		    *ncolored, 0, 0, 0L, 0, 0, 0, 0);
+		if (err)
+			break;
+
+		err = got_object_open_as_commit(&commit, repo, &qid->id);
+		if (err)
+			break;
+
+		parents = got_object_commit_get_parent_ids(commit);
+		if (parents) {
+			struct got_object_qid *pid;
+			color = (intptr_t)qid->data;
+			STAILQ_FOREACH(pid, parents, entry) {
+				err = got_pack_queue_commit_id(ids, &pid->id,
+				    color, repo);
+				if (err)
+					break;
+				nqueued++;
+				if (color == COLOR_SKIP)
+					nskip++;
+			}
+		}
+
+		if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) {
+			if (packidx == NULL) {
+				err = got_pack_find_pack_for_commit_painting(
+				    &packidx, ids, nqueued, repo);
+				if (err)
+					goto done;
+			}
+			if (packidx != NULL) {
+				err = got_pack_cache_pack_for_packidx(&pack,
+				    packidx, repo);
+				if (err)
+					goto done;
+				if (pack->privsep_child == NULL) {
+					err = got_pack_start_privsep_child(
+					    pack, packidx);
+					if (err)
+						goto done;
+				}
+				err = got_privsep_init_commit_painting(
+				    pack->privsep_child->ibuf);
+				if (err)
+					goto done;
+				err = send_idset(pack->privsep_child->ibuf,
+				    keep);
+				if (err)
+					goto done;
+				err = send_idset(pack->privsep_child->ibuf, drop);
+				if (err)
+					goto done;
+				err = send_idset(pack->privsep_child->ibuf, skip);
+				if (err)
+					goto done;
+				err = got_repo_pin_pack(repo, packidx, pack);
+				if (err)
+					goto done;
+			}
+		}
+
+		got_object_commit_close(commit);
+		commit = NULL;
+
+		got_object_qid_free(qid);
+		qid = NULL;
+	}
+done:
+	if (pack) {
+		const struct got_error *pack_err;
+		pack_err = got_privsep_send_painting_commits_done(
+		    pack->privsep_child->ibuf);
+		if (err == NULL)
+			err = pack_err;
+	}
+	if (commit)
+		got_object_commit_close(commit);
+	got_object_qid_free(qid);
+	got_repo_unpin_pack(repo);
+	return err;
+}
+
+struct load_packed_obj_arg {
+	/* output parameters: */
+	struct got_object_id *id;
+	char *dpath;
+	time_t mtime;
+
+	/* input parameters: */
+	uint32_t seed;
+	int want_meta;
+	struct got_object_idset *idset;
+	struct got_object_idset *idset_exclude;
+	int loose_obj_only;
+	int *ncolored;
+	int *nfound;
+	int *ntrees;
+	got_pack_progress_cb progress_cb;
+	void *progress_arg;
+	struct got_ratelimit *rl;
+	got_cancel_cb cancel_cb;
+	void *cancel_arg;
+};
+
+static const struct got_error *
+load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id,
+    struct got_repository *repo)
+{
+	struct load_packed_obj_arg *a = arg;
+
+	if (got_object_idset_contains(a->idset, id) ||
+	    got_object_idset_contains(a->idset_exclude, id))
+		return NULL;
+
+	return got_pack_add_object(a->want_meta,
+	    a->want_meta ? a->idset : a->idset_exclude,
+	    id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only,
+	    repo, a->ncolored, a->nfound, a->ntrees,
+	    a->progress_cb, a->progress_arg, a->rl);
+}
+
+static const struct got_error *
+load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime,
+    struct got_object_id *id, const char *dpath, struct got_repository *repo)
+{
+	const struct got_error *err;
+	struct load_packed_obj_arg *a = arg;
+	const char *relpath;
+
+	/*
+	 * When we receive a tree's ID and path but not the tree itself,
+	 * this tree object was not found in the pack file. This is the
+	 * last time we are being called for this optimized traversal.
+	 * Return from here and switch to loading objects the slow way.
+	 */
+	if (tree == NULL) {
+		free(a->id);
+		a->id = got_object_id_dup(id);
+		if (a->id == NULL) {
+			err = got_error_from_errno("got_object_id_dup");
+			free(a->dpath);
+			a->dpath = NULL;
+			return err;
+		}
+
+		free(a->dpath);
+		a->dpath = strdup(dpath);
+		if (a->dpath == NULL) {
+			err = got_error_from_errno("strdup");
+			free(a->id);
+			a->id = NULL;
+			return err;
+		}
+
+		a->mtime = mtime;
+		return NULL;
+	}
+
+	if (got_object_idset_contains(a->idset, id) ||
+	    got_object_idset_contains(a->idset_exclude, id))
+		return NULL;
+
+	relpath = dpath;
+	while (relpath[0] == '/')
+		relpath++;
+
+	err = got_pack_add_object(a->want_meta,
+	    a->want_meta ? a->idset : a->idset_exclude,
+	    id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed,
+	    a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees,
+	    a->progress_cb, a->progress_arg, a->rl);
+	if (err)
+		return err;
+
+	return got_pack_load_tree_entries(NULL, a->want_meta, a->idset,
+	    a->idset_exclude, tree, dpath, mtime, a->seed, repo,
+	    a->loose_obj_only, a->ncolored, a->nfound, a->ntrees,
+	    a->progress_cb, a->progress_arg, a->rl,
+	    a->cancel_cb, a->cancel_arg);
+}
+
+const struct got_error *
+got_pack_load_packed_object_ids(int *found_all_objects,
+    struct got_object_id **ours, int nours,
+    struct got_object_id **theirs, int ntheirs,
+    int want_meta, uint32_t seed, struct got_object_idset *idset,
+    struct got_object_idset *idset_exclude, int loose_obj_only,
+    struct got_repository *repo, struct got_packidx *packidx,
+    int *ncolored, int *nfound, int *ntrees,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
+{
+	const struct got_error *err = NULL;
+	struct load_packed_obj_arg lpa;
+
+	memset(&lpa, 0, sizeof(lpa));
+	lpa.seed = seed;
+	lpa.want_meta = want_meta;
+	lpa.idset = idset;
+	lpa.idset_exclude = idset_exclude;
+	lpa.loose_obj_only = loose_obj_only;
+	lpa.ncolored = ncolored;
+	lpa.nfound = nfound;
+	lpa.ntrees = ntrees;
+	lpa.progress_cb = progress_cb;
+	lpa.progress_arg = progress_arg;
+	lpa.rl = rl;
+	lpa.cancel_cb = cancel_cb;
+	lpa.cancel_arg = cancel_arg;
+
+	/* Attempt to load objects via got-read-pack, as far as possible. */
+	err = got_object_enumerate(found_all_objects, load_packed_commit_id,
+	   load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs,
+	   packidx, repo);
+	if (err)
+		return err;
+
+	if (lpa.id == NULL)
+		return NULL;
+
+	/*
+	 * An incomplete tree hierarchy was present in the pack file
+	 * and caused loading to be aborted.
+	 * Continue loading trees the slow way.
+	 */
+	err = got_pack_load_tree(want_meta, idset, idset_exclude,
+	    lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only,
+	    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
+	    cancel_cb, cancel_arg);
+	free(lpa.id);
+	free(lpa.dpath);
+	return err;
+}