From: Tracey Emery Subject: Re: findtwixt in got-read-pack To: gameoftrees@openbsd.org Date: Tue, 28 Jun 2022 05:28:47 -0600 On Tue, Jun 28, 2022 at 01:06:30PM +0200, Stefan Sperling wrote: > On Tue, Jun 28, 2022 at 04:54:31AM -0600, Tracey Emery wrote: > > On Tue, Jun 28, 2022 at 11:14:48AM +0200, Stefan Sperling wrote: > > > static const struct got_error * > > > -prepare_delta_reuse(struct got_pack **pack, struct got_packidx *packidx, > > > - int delta_outfd, struct got_repository *repo) > > > +cache_pack_for_packidx(struct got_pack **pack, struct got_packidx *packidx, > > > + struct got_repository *repo) > > > { > > > - const struct got_error *err = NULL; > > > + const struct got_error *err; > > > char *path_packfile = NULL; > > > > > > > Asking again here, why isn't err being returned? > > Oh right, thanks! I missed this in your initial review. > Fixed version below. ok :) > > diff refs/heads/main refs/heads/findtwixt-pack > commit - c301e4ed7f25d48be4d36d791f90cac50b0be419 > commit + eae38fc320c4356aa44fbdef943306149cdcb2a4 > blob - 8ae16da40c07f4dcb3133f9ff08328a10605722b > blob + 22a9264b9f8d0c0b20b48895dd8ea59708e61d48 > --- include/got_error.h > +++ include/got_error.h > @@ -168,6 +168,7 @@ > #define GOT_ERR_HUNK_FAILED 150 > #define GOT_ERR_PATCH_FAILED 151 > #define GOT_ERR_FILEIDX_DUP_ENTRY 152 > +#define GOT_ERR_PIN_PACK 153 > > struct got_error { > int code; > blob - 29541c0234b7f96a4638157fdd9017888571a76c > blob + 8decec3748ea2448a4255cba6eccf77ced9fab2a > --- lib/error.c > +++ lib/error.c > @@ -216,6 +216,7 @@ static const struct got_error got_errors[] = { > { GOT_ERR_HUNK_FAILED, "hunk failed to apply" }, > { GOT_ERR_PATCH_FAILED, "patch failed to apply" }, > { GOT_ERR_FILEIDX_DUP_ENTRY, "duplicate file index entry" }, > + { GOT_ERR_PIN_PACK, "could not pin pack file" }, > }; > > static struct got_custom_error { > blob - bfee6f4277cb3046739bccf25fc842f391a9b670 > blob + 24c7c68b7574559306b53c587c4f7d5e5374d02d > --- lib/got_lib_pack.h > +++ lib/got_lib_pack.h > @@ -218,5 +218,3 @@ const struct got_error *got_packfile_extract_object_to > const struct got_error *got_packfile_extract_raw_delta(uint8_t **, size_t *, > size_t *, off_t *, off_t *, struct got_object_id *, uint64_t *, uint64_t *, > struct got_pack *, struct got_packidx *, int); > -struct got_pack *got_repo_get_cached_pack(struct got_repository *, > - const char *); > blob - 8e99bbe74f6769ad1550cbcbe10d5ff1368d116d > blob + 6ffe646e98676cf9a0d19fe3ad27f3e63ab04fcc > --- lib/got_lib_privsep.h > +++ lib/got_lib_privsep.h > @@ -194,6 +194,12 @@ enum got_imsg_type { > GOT_IMSG_REUSED_DELTAS, > GOT_IMSG_DELTA_REUSE_DONE, > > + /* Commit coloring in got-read-pack. */ > + GOT_IMSG_COMMIT_PAINTING_INIT, > + GOT_IMSG_COMMIT_PAINTING_REQUEST, > + GOT_IMSG_PAINTED_COMMITS, > + GOT_IMSG_COMMIT_PAINTING_DONE, > + > /* Transfer a list of object IDs. */ > GOT_IMSG_OBJ_ID_LIST, > GOT_IMSG_OBJ_ID_LIST_DONE, > @@ -344,6 +350,27 @@ struct got_imsg_reused_deltas { > / sizeof(struct got_imsg_reused_delta)) > }; > > +/* Structure for GOT_IMSG_COMMIT_PAINTING_REQUEST. */ > +struct got_imsg_commit_painting_request { > + uint8_t id[SHA1_DIGEST_LENGTH]; > + int idx; > + int color; > +} __attribute__((__packed__)); > + > +/* Structure for GOT_IMSG_PAINTED_COMMITS. */ > +struct got_imsg_painted_commit { > + uint8_t id[SHA1_DIGEST_LENGTH]; > + intptr_t color; > +} __attribute__((__packed__)); > + > +struct got_imsg_painted_commits { > + int ncommits; > + int present_in_pack; > + /* > + * Followed by ncommits * struct got_imsg_painted_commit. > + */ > +} __attribute__((__packed__)); > + > /* Structure for GOT_IMSG_TAG data. */ > struct got_imsg_tag_object { > uint8_t id[SHA1_DIGEST_LENGTH]; > @@ -782,4 +809,16 @@ const struct got_error *got_privsep_send_reused_deltas > const struct got_error *got_privsep_recv_reused_deltas(int *, > struct got_imsg_reused_delta *, size_t *, struct imsgbuf *); > > +const struct got_error *got_privsep_init_commit_painting(struct imsgbuf *); > +const struct got_error *got_privsep_send_painting_request(struct imsgbuf *, > + int, struct got_object_id *, intptr_t); > +typedef const struct got_error *(*got_privsep_recv_painted_commit_cb)(void *, > + struct got_object_id *, intptr_t); > +const struct got_error *got_privsep_send_painted_commits(struct imsgbuf *, > + struct got_object_id_queue *, int *, int, int); > +const struct got_error *got_privsep_send_painting_commits_done(struct imsgbuf *); > +const struct got_error *got_privsep_recv_painted_commits( > + struct got_object_id_queue *, got_privsep_recv_painted_commit_cb, void *, > + struct imsgbuf *); > + > void got_privsep_exec_child(int[2], const char *, const char *); > blob - c75567b209bd9f2ce134e3ca05a66884b8dcbc04 > blob + 02b998107e294bc2eca6ffb21a128c5973771755 > --- lib/got_lib_repository.h > +++ lib/got_lib_repository.h > @@ -72,6 +72,16 @@ struct got_repository { > */ > int pack_cache_size; > > + /* > + * Index to cache entries which are pinned to avoid eviction. > + * This may be used to keep one got-index-pack process alive > + * across searches for arbitrary objects which may be stored > + * in other pack files. > + */ > + int pinned_pack; > + pid_t pinned_pid; > + int pinned_packidx; > + > /* Handles to child processes for reading loose objects. */ > struct got_privsep_child privsep_children[5]; > #define GOT_REPO_PRIVSEP_CHILD_OBJECT 0 > @@ -134,3 +144,10 @@ const struct got_error *got_repo_get_packidx(struct go > struct got_repository *); > const struct got_error *got_repo_cache_pack(struct got_pack **, > struct got_repository *, const char *, struct got_packidx *); > +struct got_pack *got_repo_get_cached_pack(struct got_repository *, > + const char *); > +const struct got_error *got_repo_pin_pack(struct got_repository *, > + struct got_packidx *, struct got_pack *); > +struct got_pack *got_repo_get_pinned_pack(struct got_repository *); > +void got_repo_unpin_pack(struct got_repository *); > + > blob - 384b6d4beb4ecb887c5ba512e0dd1f882bb02d4b > blob + d401d4ed9c42375d81a4e41c59a8657a274eb281 > --- lib/pack_create.c > +++ lib/pack_create.c > @@ -563,6 +563,28 @@ send_id(struct got_object_id *id, void *data, void *ar > } > > static const struct got_error * > +send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset) > +{ > + const struct got_error *err; > + struct send_id_arg sia; > + > + memset(&sia, 0, sizeof(sia)); > + sia.ibuf = ibuf; > + err = got_object_idset_for_each(idset, send_id, &sia); > + if (err) > + return err; > + > + if (sia.nids > 0) { > + err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids); > + if (err) > + return err; > + } > + > + return got_privsep_send_object_idlist_done(ibuf); > +} > + > + > +static const struct got_error * > recv_reused_delta(struct got_imsg_reused_delta *delta, > struct got_object_idset *idset, struct got_pack_metavec *v) > { > @@ -595,10 +617,10 @@ recv_reused_delta(struct got_imsg_reused_delta *delta, > } > > static const struct got_error * > -prepare_delta_reuse(struct got_pack **pack, struct got_packidx *packidx, > - int delta_outfd, struct got_repository *repo) > +cache_pack_for_packidx(struct got_pack **pack, struct got_packidx *packidx, > + struct got_repository *repo) > { > - const struct got_error *err = NULL; > + const struct got_error *err; > char *path_packfile = NULL; > > err = got_packidx_get_packfile_path(&path_packfile, > @@ -617,8 +639,18 @@ prepare_delta_reuse(struct got_pack **pack, struct got > if (err) > goto done; > } > +done: > + free(path_packfile); > + return err; > +} > > - if (!(*pack)->child_has_delta_outfd) { > +static const struct got_error * > +prepare_delta_reuse(struct got_pack *pack, struct got_packidx *packidx, > + int delta_outfd, struct got_repository *repo) > +{ > + const struct got_error *err = NULL; > + > + if (!pack->child_has_delta_outfd) { > int outfd_child; > outfd_child = dup(delta_outfd); > if (outfd_child == -1) { > @@ -626,15 +658,14 @@ prepare_delta_reuse(struct got_pack **pack, struct got > goto done; > } > err = got_privsep_send_raw_delta_outfd( > - (*pack)->privsep_child->ibuf, outfd_child); > + pack->privsep_child->ibuf, outfd_child); > if (err) > goto done; > - (*pack)->child_has_delta_outfd = 1; > + pack->child_has_delta_outfd = 1; > } > > - err = got_privsep_send_delta_reuse_req((*pack)->privsep_child->ibuf); > + err = got_privsep_send_delta_reuse_req(pack->privsep_child->ibuf); > done: > - free(path_packfile); > return err; > } > > @@ -649,7 +680,6 @@ search_deltas(struct got_pack_metavec *v, struct got_o > const struct got_error *err = NULL; > struct got_packidx *packidx; > struct got_pack *pack; > - struct send_id_arg sia; > struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS]; > size_t ndeltas, i; > > @@ -660,22 +690,15 @@ search_deltas(struct got_pack_metavec *v, struct got_o > if (packidx == NULL) > return NULL; > > - err = prepare_delta_reuse(&pack, packidx, delta_cache_fd, repo); > + err = cache_pack_for_packidx(&pack, packidx, repo); > if (err) > return err; > > - memset(&sia, 0, sizeof(sia)); > - sia.ibuf = pack->privsep_child->ibuf; > - err = got_object_idset_for_each(idset, send_id, &sia); > + err = prepare_delta_reuse(pack, packidx, delta_cache_fd, repo); > if (err) > return err; > - if (sia.nids > 0) { > - err = got_privsep_send_object_idlist(pack->privsep_child->ibuf, > - sia.ids, sia.nids); > - if (err) > - return err; > - } > - err = got_privsep_send_object_idlist_done(pack->privsep_child->ibuf); > + > + err = send_idset(pack->privsep_child->ibuf, idset); > if (err) > return err; > > @@ -1217,30 +1240,23 @@ done: > enum findtwixt_color { > COLOR_KEEP = 0, > COLOR_DROP, > - COLOR_BLANK, > COLOR_SKIP, > + COLOR_MAX, > }; > > -static const int findtwixt_colors[] = { > - COLOR_KEEP, > - COLOR_DROP, > - COLOR_BLANK, > - COLOR_SKIP, > -}; > - > static const struct got_error * > -paint_commit(struct got_object_qid *qid, int color) > +paint_commit(struct got_object_qid *qid, intptr_t color) > { > - if (color < 0 || color >= nitems(findtwixt_colors)) > + if (color < 0 || color >= COLOR_MAX) > return got_error(GOT_ERR_RANGE); > > - qid->data = (void *)&findtwixt_colors[color]; > + qid->data = (void *)color; > return NULL; > } > > static const struct got_error * > queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id, > - int color, struct got_repository *repo) > + intptr_t color, struct got_repository *repo) > { > const struct got_error *err; > struct got_object_qid *qid; > @@ -1277,7 +1293,7 @@ append_id(struct got_object_id *id, void *data, void * > } > > static const struct got_error * > -queue_commit_or_tag_id(struct got_object_id *id, int color, > +queue_commit_or_tag_id(struct got_object_id *id, intptr_t color, > struct got_object_id_queue *ids, struct got_repository *repo) > { > const struct got_error *err; > @@ -1307,7 +1323,197 @@ done: > return err; > } > > +struct recv_painted_commit_arg { > + int *ncolored; > + int *nqueued; > + int *nskip; > + struct got_object_id_queue *ids; > + struct got_object_idset *keep; > + struct got_object_idset *drop; > + struct got_object_idset *skip; > + got_pack_progress_cb progress_cb; > + void *progress_arg; > + struct got_ratelimit *rl; > + got_cancel_cb cancel_cb; > + void *cancel_arg; > +}; > + > static const struct got_error * > +recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color) > +{ > + const struct got_error *err = NULL; > + struct recv_painted_commit_arg *a = arg; > + struct got_object_qid *qid, *tmp; > + > + if (a->cancel_cb) { > + err = a->cancel_cb(a->cancel_arg); > + if (err) > + return err; > + } > + > + switch (color) { > + case COLOR_KEEP: > + err = got_object_idset_add(a->keep, id, NULL); > + if (err) > + return err; > + (*a->ncolored)++; > + break; > + case COLOR_DROP: > + err = got_object_idset_add(a->drop, id, NULL); > + if (err) > + return err; > + (*a->ncolored)++; > + break; > + case COLOR_SKIP: > + err = got_object_idset_add(a->skip, id, NULL); > + if (err) > + return err; > + break; > + default: > + /* should not happen */ > + return got_error_fmt(GOT_ERR_NOT_IMPL, > + "%s invalid commit color %d", __func__, color); > + } > + > + STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) { > + if (got_object_id_cmp(&qid->id, id) != 0) > + continue; > + STAILQ_REMOVE(a->ids, qid, got_object_qid, entry); > + color = (intptr_t)qid->data; > + got_object_qid_free(qid); > + (*a->nqueued)--; > + if (color == COLOR_SKIP) > + (*a->nskip)--; > + break; > + } > + > + return report_progress(a->progress_cb, a->progress_arg, a->rl, > + *a->ncolored, 0, 0, 0L, 0, 0, 0, 0); > +} > + > +static const struct got_error * > +paint_packed_commits(struct got_pack *pack, struct got_object_id *id, > + int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip, > + struct got_object_id_queue *ids, > + struct got_object_idset *keep, struct got_object_idset *drop, > + struct got_object_idset *skip, struct got_repository *repo, > + got_pack_progress_cb progress_cb, void *progress_arg, > + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) > +{ > + const struct got_error *err = NULL; > + struct got_object_id_queue next_ids; > + struct got_object_qid *qid, *tmp; > + struct recv_painted_commit_arg arg; > + > + STAILQ_INIT(&next_ids); > + > + err = got_privsep_send_painting_request(pack->privsep_child->ibuf, > + idx, id, color); > + if (err) > + return err; > + > + arg.ncolored = ncolored; > + arg.nqueued = nqueued; > + arg.nskip = nskip; > + arg.ids = ids; > + arg.keep = keep; > + arg.drop = drop; > + arg.skip = skip; > + arg.progress_cb = progress_cb; > + arg.progress_arg = progress_arg; > + arg.rl = rl; > + arg.cancel_cb = cancel_cb; > + arg.cancel_arg = cancel_arg; > + err = got_privsep_recv_painted_commits(&next_ids, > + recv_painted_commit, &arg, pack->privsep_child->ibuf); > + if (err) > + return err; > + > + STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) { > + struct got_object_qid *old_id; > + intptr_t qcolor, ocolor; > + STAILQ_FOREACH(old_id, ids, entry) { > + if (got_object_id_cmp(&qid->id, &old_id->id)) > + continue; > + qcolor = (intptr_t)qid->data; > + ocolor = (intptr_t)old_id->data; > + STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry); > + got_object_qid_free(qid); > + qid = NULL; > + if (qcolor != ocolor) { > + paint_commit(old_id, qcolor); > + if (ocolor == COLOR_SKIP) > + (*nskip)--; > + else if (qcolor == COLOR_SKIP) > + (*nskip)++; > + } > + break; > + } > + } > + while (!STAILQ_EMPTY(&next_ids)) { > + qid = STAILQ_FIRST(&next_ids); > + STAILQ_REMOVE_HEAD(&next_ids, entry); > + paint_commit(qid, color); > + STAILQ_INSERT_TAIL(ids, qid, entry); > + (*nqueued)++; > + if (color == COLOR_SKIP) > + (*nskip)++; > + } > + > + return err; > +} > + > +static const struct got_error * > +find_pack_for_commit_painting(struct got_packidx **best_packidx, > + struct got_object_id_queue *ids, int nids, struct got_repository *repo) > +{ > + const struct got_error *err = NULL; > + struct got_pathlist_entry *pe; > + const char *best_packidx_path = NULL; > + int nobj_max = 0; > + int ncommits_max = 0; > + > + *best_packidx = NULL; > + > + /* > + * Find the largest pack which contains at least some of the > + * commits we are interested in. > + */ > + TAILQ_FOREACH(pe, &repo->packidx_paths, entry) { > + const char *path_packidx = pe->path; > + struct got_packidx *packidx; > + int nobj, idx, ncommits = 0; > + struct got_object_qid *qid; > + > + err = got_repo_get_packidx(&packidx, path_packidx, repo); > + if (err) > + break; > + > + nobj = be32toh(packidx->hdr.fanout_table[0xff]); > + if (nobj <= nobj_max) > + continue; > + > + STAILQ_FOREACH(qid, ids, entry) { > + idx = got_packidx_get_object_idx(packidx, &qid->id); > + if (idx != -1) > + ncommits++; > + } > + if (ncommits > ncommits_max) { > + best_packidx_path = path_packidx; > + nobj_max = nobj; > + ncommits_max = ncommits; > + } > + } > + > + if (best_packidx_path && err == NULL) { > + err = got_repo_get_packidx(best_packidx, best_packidx_path, > + repo); > + } > + > + return err; > +} > + > +static const struct got_error * > paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids, > struct got_object_idset *keep, struct got_object_idset *drop, > struct got_object_idset *skip, struct got_repository *repo, > @@ -1316,12 +1522,15 @@ paint_commits(int *ncolored, struct got_object_id_queu > { > const struct got_error *err = NULL; > struct got_commit_object *commit = NULL; > + struct got_packidx *packidx = NULL; > + struct got_pack *pack = NULL; > const struct got_object_id_queue *parents; > - struct got_object_qid *qid; > + struct got_object_qid *qid = NULL; > int nqueued = nids, nskip = 0; > + int idx; > > while (!STAILQ_EMPTY(ids) && nskip != nqueued) { > - int color; > + intptr_t color; > > if (cancel_cb) { > err = cancel_cb(cancel_arg); > @@ -1332,21 +1541,49 @@ paint_commits(int *ncolored, struct got_object_id_queu > qid = STAILQ_FIRST(ids); > STAILQ_REMOVE_HEAD(ids, entry); > nqueued--; > - color = *((int *)qid->data); > + color = (intptr_t)qid->data; > if (color == COLOR_SKIP) > nskip--; > > if (got_object_idset_contains(skip, &qid->id)) { > got_object_qid_free(qid); > + qid = NULL; > continue; > } > + if (color == COLOR_KEEP && > + got_object_idset_contains(keep, &qid->id)) { > + got_object_qid_free(qid); > + qid = NULL; > + continue; > + } > + if (color == COLOR_DROP && > + got_object_idset_contains(drop, &qid->id)) { > + got_object_qid_free(qid); > + qid = NULL; > + continue; > + } > > + /* Pinned pack may have moved to different cache slot. */ > + pack = got_repo_get_pinned_pack(repo); > + > + if (packidx && pack) { > + idx = got_packidx_get_object_idx(packidx, &qid->id); > + if (idx != -1) { > + err = paint_packed_commits(pack, &qid->id, > + idx, color, ncolored, &nqueued, &nskip, > + ids, keep, drop, skip, repo, > + progress_cb, progress_arg, rl, > + cancel_cb, cancel_arg); > + if (err) > + break; > + got_object_qid_free(qid); > + qid = NULL; > + continue; > + } > + } > + > switch (color) { > case COLOR_KEEP: > - if (got_object_idset_contains(keep, &qid->id)) { > - got_object_qid_free(qid); > - continue; > - } > if (got_object_idset_contains(drop, &qid->id)) { > err = paint_commit(qid, COLOR_SKIP); > if (err) > @@ -1358,10 +1595,6 @@ paint_commits(int *ncolored, struct got_object_id_queu > goto done; > break; > case COLOR_DROP: > - if (got_object_idset_contains(drop, &qid->id)) { > - got_object_qid_free(qid); > - continue; > - } > if (got_object_idset_contains(keep, &qid->id)) { > err = paint_commit(qid, COLOR_SKIP); > if (err) > @@ -1392,7 +1625,6 @@ paint_commits(int *ncolored, struct got_object_id_queu > if (err) > break; > > - > err = got_object_open_as_commit(&commit, repo, &qid->id); > if (err) > break; > @@ -1400,10 +1632,10 @@ paint_commits(int *ncolored, struct got_object_id_queu > parents = got_object_commit_get_parent_ids(commit); > if (parents) { > struct got_object_qid *pid; > - color = *((int *)qid->data); > + color = (intptr_t)qid->data; > STAILQ_FOREACH(pid, parents, entry) { > - err = queue_commit_id(ids, &pid->id, color, > - repo); > + err = queue_commit_id(ids, &pid->id, > + color, repo); > if (err) > break; > nqueued++; > @@ -1412,13 +1644,56 @@ paint_commits(int *ncolored, struct got_object_id_queu > } > } > > + if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) { > + if (packidx == NULL) { > + err = find_pack_for_commit_painting(&packidx, > + ids, nqueued, repo); > + if (err) > + goto done; > + } > + if (packidx != NULL) { > + err = cache_pack_for_packidx(&pack, packidx, > + repo); > + if (err) > + goto done; > + err = got_privsep_init_commit_painting( > + pack->privsep_child->ibuf); > + if (err) > + goto done; > + err = send_idset(pack->privsep_child->ibuf, > + keep); > + if (err) > + goto done; > + err = send_idset(pack->privsep_child->ibuf, drop); > + if (err) > + goto done; > + err = send_idset(pack->privsep_child->ibuf, skip); > + if (err) > + goto done; > + err = got_repo_pin_pack(repo, packidx, pack); > + if (err) > + goto done; > + } > + } > + > got_object_commit_close(commit); > commit = NULL; > + > got_object_qid_free(qid); > + qid = NULL; > } > done: > + if (pack) { > + const struct got_error *pack_err; > + pack_err = got_privsep_send_painting_commits_done( > + pack->privsep_child->ibuf); > + if (err == NULL) > + err = pack_err; > + } > if (commit) > got_object_commit_close(commit); > + got_object_qid_free(qid); > + got_repo_unpin_pack(repo); > return err; > } > > blob - 70eb167c5ee71b29c045bdae0de5c7b7859403f7 > blob + c0bdac7221a79c5ec97d1728e862406152d51eb9 > --- lib/privsep.c > +++ lib/privsep.c > @@ -3328,6 +3328,182 @@ got_privsep_recv_reused_deltas(int *done, struct got_i > } > > const struct got_error * > +got_privsep_init_commit_painting(struct imsgbuf *ibuf) > +{ > + if (imsg_compose(ibuf, GOT_IMSG_COMMIT_PAINTING_INIT, > + 0, 0, -1, NULL, 0) > + == -1) > + return got_error_from_errno("imsg_compose " > + "COMMIT_PAINTING_INIT"); > + > + return flush_imsg(ibuf); > +} > + > +const struct got_error * > +got_privsep_send_painting_request(struct imsgbuf *ibuf, int idx, > + struct got_object_id *id, intptr_t color) > +{ > + struct got_imsg_commit_painting_request ireq; > + > + memset(&ireq, 0, sizeof(ireq)); > + memcpy(ireq.id, id->sha1, sizeof(ireq.id)); > + ireq.idx = idx; > + ireq.color = color; > + > + if (imsg_compose(ibuf, GOT_IMSG_COMMIT_PAINTING_REQUEST, 0, 0, -1, > + &ireq, sizeof(ireq)) == -1) > + return got_error_from_errno("imsg_compose " > + "COMMIT_PAINTING_REQUEST"); > + > + return flush_imsg(ibuf); > +} > + > +static const struct got_error * > +send_painted_commits(struct got_object_id_queue *ids, int *nids, > + size_t remain, int present_in_pack, struct imsgbuf *ibuf) > +{ > + const struct got_error *err = NULL; > + struct ibuf *wbuf = NULL; > + struct got_object_qid *qid; > + size_t msglen; > + int ncommits; > + intptr_t color; > + > + msglen = MIN(remain, MAX_IMSGSIZE - IMSG_HEADER_SIZE); > + ncommits = (msglen - sizeof(struct got_imsg_painted_commits)) / > + sizeof(struct got_imsg_painted_commit); > + > + wbuf = imsg_create(ibuf, GOT_IMSG_PAINTED_COMMITS, 0, 0, msglen); > + if (wbuf == NULL) { > + err = got_error_from_errno("imsg_create PAINTED_COMMITS"); > + return err; > + } > + > + /* Keep in sync with struct got_imsg_painted_commits! */ > + if (imsg_add(wbuf, &ncommits, sizeof(ncommits)) == -1) > + return got_error_from_errno("imsg_add PAINTED_COMMITS"); > + if (imsg_add(wbuf, &present_in_pack, sizeof(present_in_pack)) == -1) > + return got_error_from_errno("imsg_add PAINTED_COMMITS"); > + > + while (ncommits > 0) { > + qid = STAILQ_FIRST(ids); > + STAILQ_REMOVE_HEAD(ids, entry); > + ncommits--; > + (*nids)--; > + color = (intptr_t)qid->data; > + > + /* Keep in sync with struct got_imsg_painted_commit! */ > + if (imsg_add(wbuf, qid->id.sha1, SHA1_DIGEST_LENGTH) == -1) > + return got_error_from_errno("imsg_add PAINTED_COMMITS"); > + if (imsg_add(wbuf, &color, sizeof(color)) == -1) > + return got_error_from_errno("imsg_add PAINTED_COMMITS"); > + > + got_object_qid_free(qid); > + } > + > + wbuf->fd = -1; > + imsg_close(ibuf, wbuf); > + > + return flush_imsg(ibuf); > +} > + > +const struct got_error * > +got_privsep_send_painted_commits(struct imsgbuf *ibuf, > + struct got_object_id_queue *ids, int *nids, > + int present_in_pack, int flush) > +{ > + const struct got_error *err; > + size_t remain; > + > + if (*nids <= 0) > + return NULL; > + > + do { > + remain = (sizeof(struct got_imsg_painted_commits)) + > + *nids * sizeof(struct got_imsg_painted_commit); > + if (flush || remain >= MAX_IMSGSIZE - IMSG_HEADER_SIZE) { > + err = send_painted_commits(ids, nids, remain, > + present_in_pack, ibuf); > + if (err) > + return err; > + } > + } while (flush && *nids > 0); > + > + return NULL; > +} > + > +const struct got_error * > +got_privsep_send_painting_commits_done(struct imsgbuf *ibuf) > +{ > + if (imsg_compose(ibuf, GOT_IMSG_COMMIT_PAINTING_DONE, > + 0, 0, -1, NULL, 0) > + == -1) > + return got_error_from_errno("imsg_compose " > + "COMMIT_PAINTING_DONE"); > + > + return flush_imsg(ibuf); > +} > + > +const struct got_error * > +got_privsep_recv_painted_commits(struct got_object_id_queue *new_ids, > + got_privsep_recv_painted_commit_cb cb, void *cb_arg, struct imsgbuf *ibuf) > +{ > + const struct got_error *err = NULL; > + struct imsg imsg; > + struct got_imsg_painted_commits icommits; > + struct got_imsg_painted_commit icommit; > + size_t datalen; > + int i; > + > + for (;;) { > + err = got_privsep_recv_imsg(&imsg, ibuf, 0); > + if (err) > + return err; > + > + datalen = imsg.hdr.len - IMSG_HEADER_SIZE; > + if (imsg.hdr.type == GOT_IMSG_COMMIT_PAINTING_DONE) > + break; > + if (imsg.hdr.type != GOT_IMSG_PAINTED_COMMITS) > + return got_error(GOT_ERR_PRIVSEP_MSG); > + > + if (datalen < sizeof(icommits)) > + return got_error(GOT_ERR_PRIVSEP_LEN); > + memcpy(&icommits, imsg.data, sizeof(icommits)); > + if (icommits.ncommits * sizeof(icommit) < icommits.ncommits || > + datalen < sizeof(icommits) + > + icommits.ncommits * sizeof(icommit)) > + return got_error(GOT_ERR_PRIVSEP_LEN); > + > + for (i = 0; i < icommits.ncommits; i++) { > + memcpy(&icommit, > + (uint8_t *)imsg.data + sizeof(icommits) + i * sizeof(icommit), > + sizeof(icommit)); > + > + if (icommits.present_in_pack) { > + struct got_object_id id; > + memcpy(id.sha1, icommit.id, SHA1_DIGEST_LENGTH); > + err = cb(cb_arg, &id, icommit.color); > + if (err) > + break; > + } else { > + struct got_object_qid *qid; > + err = got_object_qid_alloc_partial(&qid); > + if (err) > + break; > + memcpy(qid->id.sha1, icommit.id, > + SHA1_DIGEST_LENGTH); > + qid->data = (void *)icommit.color; > + STAILQ_INSERT_TAIL(new_ids, qid, entry); > + } > + } > + > + imsg_free(&imsg); > + } > + > + return err; > +} > + > +const struct got_error * > got_privsep_unveil_exec_helpers(void) > { > const char *helpers[] = { > blob - 4aba251d6858a54219d51424919806df0ad2b260 > blob + 4c93c601016c47cab8439703f5925c65095b4b7e > --- lib/repository.c > +++ lib/repository.c > @@ -752,6 +752,9 @@ got_repo_open(struct got_repository **repop, const cha > repo->packs[i].accumfd = -1; > } > } > + repo->pinned_pack = -1; > + repo->pinned_packidx = -1; > + repo->pinned_pid = 0; > > repo_path = realpath(path, NULL); > if (repo_path == NULL) { > @@ -1035,7 +1038,10 @@ cache_packidx(struct got_repository *repo, struct got_ > } > } > if (i == repo->pack_cache_size) { > - i = repo->pack_cache_size - 1; > + do { > + i--; > + } while (i > 0 && repo->pinned_packidx >= 0 && > + i == repo->pinned_packidx); > err = got_packidx_close(repo->packidx_cache[i]); > if (err) > return err; > @@ -1171,6 +1177,11 @@ got_repo_search_packidx(struct got_packidx **packidx, > &repo->packidx_cache[0], > i * sizeof(repo->packidx_cache[0])); > repo->packidx_cache[0] = *packidx; > + if (repo->pinned_packidx >= 0 && > + repo->pinned_packidx < i) > + repo->pinned_packidx++; > + else if (repo->pinned_packidx == i) > + repo->pinned_packidx = 0; > } > return NULL; > } > @@ -1375,17 +1386,25 @@ got_repo_cache_pack(struct got_pack **packp, struct go > > if (i == repo->pack_cache_size) { > struct got_pack tmp; > - err = got_pack_close(&repo->packs[i - 1]); > + do { > + i--; > + } while (i > 0 && repo->pinned_pack >= 0 && > + i == repo->pinned_pack); > + err = got_pack_close(&repo->packs[i]); > if (err) > return err; > - if (ftruncate(repo->packs[i - 1].basefd, 0L) == -1) > + if (ftruncate(repo->packs[i].basefd, 0L) == -1) > return got_error_from_errno("ftruncate"); > - if (ftruncate(repo->packs[i - 1].accumfd, 0L) == -1) > + if (ftruncate(repo->packs[i].accumfd, 0L) == -1) > return got_error_from_errno("ftruncate"); > - memcpy(&tmp, &repo->packs[i - 1], sizeof(tmp)); > - memcpy(&repo->packs[i - 1], &repo->packs[0], > - sizeof(repo->packs[i - 1])); > + memcpy(&tmp, &repo->packs[i], sizeof(tmp)); > + memcpy(&repo->packs[i], &repo->packs[0], > + sizeof(repo->packs[i])); > memcpy(&repo->packs[0], &tmp, sizeof(repo->packs[0])); > + if (repo->pinned_pack == 0) > + repo->pinned_pack = i; > + else if (repo->pinned_pack == i) > + repo->pinned_pack = 0; > i = 0; > } > > @@ -1449,6 +1468,51 @@ got_repo_get_cached_pack(struct got_repository *repo, > } > > const struct got_error * > +got_repo_pin_pack(struct got_repository *repo, struct got_packidx *packidx, > + struct got_pack *pack) > +{ > + size_t i; > + int pinned_pack = -1, pinned_packidx = -1; > + > + for (i = 0; i < repo->pack_cache_size; i++) { > + if (repo->packidx_cache[i] && > + strcmp(repo->packidx_cache[i]->path_packidx, > + packidx->path_packidx) == 0) > + pinned_packidx = i; > + if (repo->packs[i].path_packfile && > + strcmp(repo->packs[i].path_packfile, > + pack->path_packfile) == 0) > + pinned_pack = i; > + } > + > + if (pinned_packidx == -1 || pinned_pack == -1) > + return got_error(GOT_ERR_PIN_PACK); > + > + repo->pinned_pack = pinned_pack; > + repo->pinned_packidx = pinned_packidx; > + repo->pinned_pid = repo->packs[pinned_pack].privsep_child->pid; > + return NULL; > +} > + > +struct got_pack * > +got_repo_get_pinned_pack(struct got_repository *repo) > +{ > + if (repo->pinned_pack >= 0 && > + repo->pinned_pack < repo->pack_cache_size) > + return &repo->packs[repo->pinned_pack]; > + > + return NULL; > +} > + > +void > +got_repo_unpin_pack(struct got_repository *repo) > +{ > + repo->pinned_packidx = -1; > + repo->pinned_pack = -1; > + repo->pinned_pid = 0; > +} > + > +const struct got_error * > got_repo_init(const char *repo_path) > { > const struct got_error *err = NULL; > blob - 71fcef93a4f835b8c00eda8b486c0c1a7aef8fe8 > blob + 30401fee605c3b7937f25d1f3c8f216c945c9525 > --- libexec/got-read-pack/got-read-pack.c > +++ libexec/got-read-pack/got-read-pack.c > @@ -45,6 +45,10 @@ > #include "got_lib_privsep.h" > #include "got_lib_pack.h" > > +#ifndef nitems > +#define nitems(_a) (sizeof((_a)) / sizeof((_a)[0])) > +#endif > + > static volatile sig_atomic_t sigint_received; > > static void > @@ -1540,7 +1544,273 @@ done: > return err; > } > > +enum findtwixt_color { > + COLOR_KEEP = 0, > + COLOR_DROP, > + COLOR_SKIP, > + COLOR_MAX, > +}; > + > static const struct got_error * > +paint_commit(struct got_object_qid *qid, intptr_t color) > +{ > + if (color < 0 || color >= COLOR_MAX) > + return got_error(GOT_ERR_RANGE); > + > + qid->data = (void *)color; > + return NULL; > +} > + > +static const struct got_error * > +queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id, > + intptr_t color) > +{ > + const struct got_error *err; > + struct got_object_qid *qid; > + > + err = got_object_qid_alloc_partial(&qid); > + if (err) > + return err; > + > + memcpy(&qid->id, id, sizeof(qid->id)); > + STAILQ_INSERT_TAIL(ids, qid, entry); > + return paint_commit(qid, color); > +} > + > +static const struct got_error * > +paint_commits(struct got_object_id_queue *ids, int *nids, > + struct got_object_idset *keep, struct got_object_idset *drop, > + struct got_object_idset *skip, struct got_pack *pack, > + struct got_packidx *packidx, struct imsgbuf *ibuf, > + struct got_object_cache *objcache) > +{ > + const struct got_error *err = NULL; > + struct got_commit_object *commit = NULL; > + struct got_object_id_queue painted; > + const struct got_object_id_queue *parents; > + struct got_object_qid *qid = NULL; > + int nqueued = *nids, nskip = 0, npainted = 0; > + > + STAILQ_INIT(&painted); > + > + while (!STAILQ_EMPTY(ids) && nskip != nqueued) { > + int idx; > + intptr_t color; > + > + if (sigint_received) { > + err = got_error(GOT_ERR_CANCELLED); > + goto done; > + } > + > + qid = STAILQ_FIRST(ids); > + idx = got_packidx_get_object_idx(packidx, &qid->id); > + if (idx == -1) { > + qid = NULL; > + break; > + } > + > + STAILQ_REMOVE_HEAD(ids, entry); > + nqueued--; > + color = (intptr_t)qid->data; > + if (color == COLOR_SKIP) > + nskip--; > + > + if (got_object_idset_contains(skip, &qid->id)) { > + got_object_qid_free(qid); > + qid = NULL; > + continue; > + } > + > + switch (color) { > + case COLOR_KEEP: > + if (got_object_idset_contains(keep, &qid->id)) { > + got_object_qid_free(qid); > + qid = NULL; > + continue; > + } > + if (got_object_idset_contains(drop, &qid->id)) { > + err = paint_commit(qid, COLOR_SKIP); > + if (err) > + goto done; > + } > + err = got_object_idset_add(keep, &qid->id, NULL); > + if (err) > + goto done; > + break; > + case COLOR_DROP: > + if (got_object_idset_contains(drop, &qid->id)) { > + got_object_qid_free(qid); > + qid = NULL; > + continue; > + } > + if (got_object_idset_contains(keep, &qid->id)) { > + err = paint_commit(qid, COLOR_SKIP); > + if (err) > + goto done; > + } > + err = got_object_idset_add(drop, &qid->id, NULL); > + if (err) > + goto done; > + break; > + case COLOR_SKIP: > + if (!got_object_idset_contains(skip, &qid->id)) { > + err = got_object_idset_add(skip, &qid->id, > + NULL); > + if (err) > + goto done; > + } > + break; > + default: > + /* should not happen */ > + err = got_error_fmt(GOT_ERR_NOT_IMPL, > + "%s invalid commit color %d", __func__, color); > + goto done; > + } > + > + err = open_commit(&commit, pack, packidx, idx, &qid->id, > + objcache); > + if (err) > + goto done; > + > + parents = got_object_commit_get_parent_ids(commit); > + if (parents) { > + struct got_object_qid *pid; > + color = (intptr_t)qid->data; > + STAILQ_FOREACH(pid, parents, entry) { > + err = queue_commit_id(ids, &pid->id, color); > + if (err) > + goto done; > + nqueued++; > + if (color == COLOR_SKIP) > + nskip++; > + } > + } > + > + got_object_commit_close(commit); > + commit = NULL; > + > + STAILQ_INSERT_TAIL(&painted, qid, entry); > + qid = NULL; > + npainted++; > + > + err = got_privsep_send_painted_commits(ibuf, &painted, > + &npainted, 1, 0); > + if (err) > + goto done; > + } > + > + err = got_privsep_send_painted_commits(ibuf, &painted, &npainted, 1, 1); > + if (err) > + goto done; > + > + *nids = nqueued; > +done: > + if (commit) > + got_object_commit_close(commit); > + got_object_qid_free(qid); > + return err; > +} > + > +static void > +commit_painting_free(struct got_object_idset **keep, > + struct got_object_idset **drop, > + struct got_object_idset **skip) > +{ > + if (*keep) { > + got_object_idset_free(*keep); > + *keep = NULL; > + } > + if (*drop) { > + got_object_idset_free(*drop); > + *drop = NULL; > + } > + if (*skip) { > + got_object_idset_free(*skip); > + *skip = NULL; > + } > +} > + > +static const struct got_error * > +commit_painting_init(struct imsgbuf *ibuf, struct got_object_idset **keep, > + struct got_object_idset **drop, struct got_object_idset **skip) > +{ > + const struct got_error *err = NULL; > + > + *keep = got_object_idset_alloc(); > + if (*keep == NULL) { > + err = got_error_from_errno("got_object_idset_alloc"); > + goto done; > + } > + *drop = got_object_idset_alloc(); > + if (*drop == NULL) { > + err = got_error_from_errno("got_object_idset_alloc"); > + goto done; > + } > + *skip = got_object_idset_alloc(); > + if (*skip == NULL) { > + err = got_error_from_errno("got_object_idset_alloc"); > + goto done; > + } > + > + err = recv_object_ids(*keep, ibuf); > + if (err) > + goto done; > + err = recv_object_ids(*drop, ibuf); > + if (err) > + goto done; > + err = recv_object_ids(*skip, ibuf); > + if (err) > + goto done; > + > +done: > + if (err) > + commit_painting_free(keep, drop, skip); > + > + return err; > +} > + > +static const struct got_error * > +commit_painting_request(struct imsg *imsg, struct imsgbuf *ibuf, > + struct got_pack *pack, struct got_packidx *packidx, > + struct got_object_cache *objcache, struct got_object_idset *keep, > + struct got_object_idset *drop, struct got_object_idset *skip) > +{ > + const struct got_error *err = NULL; > + struct got_imsg_commit_painting_request ireq; > + struct got_object_id id; > + size_t datalen; > + struct got_object_id_queue ids; > + int nids = 0; > + > + STAILQ_INIT(&ids); > + > + datalen = imsg->hdr.len - IMSG_HEADER_SIZE; > + if (datalen != sizeof(ireq)) > + return got_error(GOT_ERR_PRIVSEP_LEN); > + memcpy(&ireq, imsg->data, sizeof(ireq)); > + memcpy(id.sha1, ireq.id, SHA1_DIGEST_LENGTH); > + > + err = queue_commit_id(&ids, &id, ireq.color); > + if (err) > + return err; > + nids = 1; > + > + err = paint_commits(&ids, &nids, keep, drop, skip, > + pack, packidx, ibuf, objcache); > + if (err) > + goto done; > + > + err = got_privsep_send_painted_commits(ibuf, &ids, &nids, 0, 1); > + if (err) > + goto done; > + > + err = got_privsep_send_painting_commits_done(ibuf); > +done: > + got_object_id_queue_free(&ids); > + return err; > +} > + > +static const struct got_error * > receive_pack(struct got_pack **packp, struct imsgbuf *ibuf) > { > const struct got_error *err = NULL; > @@ -1625,6 +1895,7 @@ main(int argc, char *argv[]) > struct got_pack *pack = NULL; > struct got_object_cache objcache; > FILE *basefile = NULL, *accumfile = NULL, *delta_outfile = NULL; > + struct got_object_idset *keep = NULL, *drop = NULL, *skip = NULL; > > //static int attached; > //while (!attached) sleep(1); > @@ -1754,6 +2025,21 @@ main(int argc, char *argv[]) > err = enumeration_request(&imsg, &ibuf, pack, > packidx, &objcache); > break; > + case GOT_IMSG_COMMIT_PAINTING_INIT: > + commit_painting_free(&keep, &drop, &skip); > + err = commit_painting_init(&ibuf, &keep, &drop, &skip); > + break; > + case GOT_IMSG_COMMIT_PAINTING_REQUEST: > + if (keep == NULL || drop == NULL || skip == NULL) { > + err = got_error(GOT_ERR_PRIVSEP_MSG); > + break; > + } > + err = commit_painting_request(&imsg, &ibuf, pack, > + packidx, &objcache, keep, drop, skip); > + break; > + case GOT_IMSG_COMMIT_PAINTING_DONE: > + commit_painting_free(&keep, &drop, &skip); > + break; > default: > err = got_error(GOT_ERR_PRIVSEP_MSG); > break; > @@ -1766,6 +2052,7 @@ main(int argc, char *argv[]) > break; > } > > + commit_painting_free(&keep, &drop, &skip); > if (packidx) > got_packidx_close(packidx); > if (pack) > -- Tracey Emery