split up pack_create.c

Some code in pack_create.c depends on privsep.c, which makes future gotd(8) error out during the build like this: ld: error: undefined symbol: got_privsep_recv_reused_deltas ld: error: undefined symbol: got_privsep_init_commit_painting ld: error: undefined symbol: got_privsep_send_painting_commits_done ld: error: undefined symbol: got_privsep_send_painting_request ld: error: undefined symbol: got_privsep_recv_painted_commits ld: error: undefined symbol: got_privsep_send_object_idlist ld: error: undefined symbol: got_privsep_send_object_idlist_done ld: error: undefined symbol: got_privsep_send_raw_delta_outfd ld: error: undefined symbol: got_privsep_send_delta_reuse_req To allow me to fix this, move relevant code to a new file, called pack_create_privsep.c, and link to this file where neeed. gotd will then have to implement its own versions of got_pack_search_deltas(), got_pack_paint_commits(), and got_pack_load_packed_object_ids(), avoiding the privsep dance (instead, gotd reads objects directly inside a chroot). ok? diff a6072ec90f252faf1f1256591fb0d4623f4bc0c2 a4df42eded0178f40d762a4cf5239dcb95a02d42 commit - a6072ec90f252faf1f1256591fb0d4623f4bc0c2 commit + a4df42eded0178f40d762a4cf5239dcb95a02d42 blob - 2491213e9e4a591c7ea3ebd247528bf036f39e53 blob + 652bddb959b1bd1a528daf84da62d9a2f52d5d51 --- got/Makefile +++ got/Makefile @@ -15,7 +15,7 @@ SRCS= got.c blame.c commit_graph.c delta.c diff.c \ diff_patience.c send.c deltify.c pack_create.c dial.c \ bloom.c murmurhash2.c ratelimit.c patch.c sigs.c date.c \ object_open_privsep.c read_gitconfig_privsep.c \ - read_gotconfig_privsep.c + read_gotconfig_privsep.c pack_create_privsep.c MAN = ${PROG}.1 got-worktree.5 git-repository.5 got.conf.5 blob - 6ffb2c64697c223811c72f8cf386b645cb1bc757 blob + 00e5e77e780621968aa2a18606882840fb9f4d2c --- gotadmin/Makefile +++ gotadmin/Makefile @@ -10,7 +10,8 @@ SRCS= gotadmin.c \ path.c privsep.c reference.c repository.c repository_admin.c \ worktree_open.c sha1.c bloom.c murmurhash2.c ratelimit.c \ sigs.c buf.c date.c object_open_privsep.c \ - read_gitconfig_privsep.c read_gotconfig_privsep.c + read_gitconfig_privsep.c read_gotconfig_privsep.c \ + pack_create_privsep.c MAN = ${PROG}.1 CPPFLAGS = -I${.CURDIR}/../include -I${.CURDIR}/../lib blob - bc54ae58a9ac550958c259f684367576576d5209 blob + eccf16c525cb2cec8d54d8ed5220540f001ddb78 --- lib/got_lib_pack_create.h +++ lib/got_lib_pack_create.h @@ -27,3 +27,120 @@ const struct got_error *got_pack_create(uint8_t *pack_ struct got_repository *repo, int loose_obj_only, int allow_empty, got_pack_progress_cb progress_cb, void *progress_arg, got_cancel_cb cancel_cb, void *cancel_arg); + +const struct got_error * +got_pack_cache_pack_for_packidx(struct got_pack **pack, + struct got_packidx *packidx, struct got_repository *repo); + +const struct got_error * +got_pack_find_pack_for_commit_painting(struct got_packidx **best_packidx, + struct got_object_id_queue *ids, int nids, struct got_repository *repo); +const struct got_error *got_pack_find_pack_for_reuse( + struct got_packidx **best_packidx, struct got_repository *repo); + +struct got_ratelimit; +const struct got_error *got_pack_paint_commits(int *ncolored, + struct got_object_id_queue *ids, int nids, + struct got_object_idset *keep, struct got_object_idset *drop, + struct got_object_idset *skip, struct got_repository *repo, + got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg); + +enum got_pack_findtwixt_color { + COLOR_KEEP = 0, + COLOR_DROP, + COLOR_SKIP, + COLOR_MAX, +}; + +const struct got_error *got_pack_paint_commit(struct got_object_qid *qid, + intptr_t color); +const struct got_error *got_pack_queue_commit_id( + struct got_object_id_queue *ids, struct got_object_id *id, intptr_t color, + struct got_repository *repo); + +struct got_pack_metavec { + struct got_pack_meta **meta; + int nmeta; + int metasz; +}; + +struct got_pack_meta { + struct got_object_id id; + uint32_t path_hash; + int obj_type; + off_t size; + time_t mtime; + + /* The best delta we picked */ + struct got_pack_meta *head; + struct got_pack_meta *prev; + unsigned char *delta_buf; /* if encoded in memory (compressed) */ + off_t delta_offset; /* offset in delta cache file (compressed) */ + off_t delta_len; /* encoded delta length */ + off_t delta_compressed_len; /* encoded+compressed delta length */ + int nchain; + + off_t reused_delta_offset; /* offset of delta in reused pack file */ + struct got_object_id *base_obj_id; + + /* Only used for delta window */ + struct got_delta_table *dtab; + + /* Only used for writing offset deltas */ + off_t off; +}; + +const struct got_error *got_pack_add_meta(struct got_pack_meta *m, + struct got_pack_metavec *v); + +const struct got_error * +got_pack_search_deltas(struct got_pack_metavec *v, + struct got_object_idset *idset, int delta_cache_fd, + int ncolored, int nfound, int ntrees, int ncommits, + struct got_repository *repo, + got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg); + +const struct got_error * +got_pack_report_progress(got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, int ncolored, int nfound, int ntrees, + off_t packfile_size, int ncommits, int nobj_total, int obj_deltify, + int nobj_written); + +const struct got_error * +got_pack_load_packed_object_ids(int *found_all_objects, + struct got_object_id **ours, int nours, + struct got_object_id **theirs, int ntheirs, + int want_meta, uint32_t seed, struct got_object_idset *idset, + struct got_object_idset *idset_exclude, int loose_obj_only, + struct got_repository *repo, struct got_packidx *packidx, + int *ncolored, int *nfound, int *ntrees, + got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg); + +const struct got_error * +got_pack_load_tree_entries(struct got_object_id_queue *ids, int want_meta, + struct got_object_idset *idset, struct got_object_idset *idset_exclude, + struct got_tree_object *tree, + const char *dpath, time_t mtime, uint32_t seed, struct got_repository *repo, + int loose_obj_only, int *ncolored, int *nfound, int *ntrees, + got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg); + +const struct got_error * +got_pack_load_tree(int want_meta, struct got_object_idset *idset, + struct got_object_idset *idset_exclude, + struct got_object_id *tree_id, const char *dpath, time_t mtime, + uint32_t seed, struct got_repository *repo, int loose_obj_only, + int *ncolored, int *nfound, int *ntrees, + got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg); + +const struct got_error * +got_pack_add_object(int want_meta, struct got_object_idset *idset, + struct got_object_id *id, const char *path, int obj_type, + time_t mtime, uint32_t seed, int loose_obj_only, + struct got_repository *repo, int *ncolored, int *nfound, int *ntrees, + got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl); blob - a3679a51361ead3fefdf763047f2b5756aaa45ee blob + 5c304b81b40e2f353d56b5acb1e76b45f3448678 --- lib/pack_create.c +++ lib/pack_create.c @@ -52,7 +52,6 @@ #include "got_lib_deflate.h" #include "got_lib_pack.h" #include "got_lib_pack_create.h" -#include "got_lib_privsep.h" #include "got_lib_repository.h" #include "got_lib_ratelimit.h" #include "got_lib_inflate.h" @@ -71,38 +70,6 @@ struct got_pack_meta { #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0])) #endif -struct got_pack_meta { - struct got_object_id id; - uint32_t path_hash; - int obj_type; - off_t size; - time_t mtime; - - /* The best delta we picked */ - struct got_pack_meta *head; - struct got_pack_meta *prev; - unsigned char *delta_buf; /* if encoded in memory (compressed) */ - off_t delta_offset; /* offset in delta cache file (compressed) */ - off_t delta_len; /* encoded delta length */ - off_t delta_compressed_len; /* encoded+compressed delta length */ - int nchain; - - off_t reused_delta_offset; /* offset of delta in reused pack file */ - struct got_object_id *base_obj_id; - - /* Only used for delta window */ - struct got_delta_table *dtab; - - /* Only used for writing offset deltas */ - off_t off; -}; - -struct got_pack_metavec { - struct got_pack_meta **meta; - int nmeta; - int metasz; -}; - static const struct got_error * alloc_meta(struct got_pack_meta **new, struct got_object_id *id, const char *path, int obj_type, time_t mtime, uint32_t seed) @@ -466,8 +433,8 @@ static const struct got_error * return err; } -static const struct got_error * -report_progress(got_pack_progress_cb progress_cb, void *progress_arg, +const struct got_error * +got_pack_report_progress(got_pack_progress_cb progress_cb, void *progress_arg, struct got_ratelimit *rl, int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits, int nobj_total, int obj_deltify, int nobj_written) @@ -486,8 +453,8 @@ static const struct got_error * packfile_size, ncommits, nobj_total, obj_deltify, nobj_written); } -static const struct got_error * -add_meta(struct got_pack_meta *m, struct got_pack_metavec *v) +const struct got_error * +got_pack_add_meta(struct got_pack_meta *m, struct got_pack_metavec *v) { if (v->nmeta == v->metasz){ size_t newsize = 2 * v->metasz; @@ -503,8 +470,8 @@ static const struct got_error * return NULL; } -static const struct got_error * -find_pack_for_reuse(struct got_packidx **best_packidx, +const struct got_error * +got_pack_find_pack_for_reuse(struct got_packidx **best_packidx, struct got_repository *repo) { const struct got_error *err = NULL; @@ -538,89 +505,11 @@ struct send_id_arg { return err; } -struct send_id_arg { - struct imsgbuf *ibuf; - struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS]; - size_t nids; -}; - -static const struct got_error * -send_id(struct got_object_id *id, void *data, void *arg) +const struct got_error * +got_pack_cache_pack_for_packidx(struct got_pack **pack, + struct got_packidx *packidx, struct got_repository *repo) { - const struct got_error *err = NULL; - struct send_id_arg *a = arg; - - a->ids[a->nids++] = id; - - if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) { - err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids); - if (err) - return err; - a->nids = 0; - } - - return NULL; -} - -static const struct got_error * -send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset) -{ const struct got_error *err; - struct send_id_arg sia; - - memset(&sia, 0, sizeof(sia)); - sia.ibuf = ibuf; - err = got_object_idset_for_each(idset, send_id, &sia); - if (err) - return err; - - if (sia.nids > 0) { - err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids); - if (err) - return err; - } - - return got_privsep_send_object_idlist_done(ibuf); -} - - -static const struct got_error * -recv_reused_delta(struct got_imsg_reused_delta *delta, - struct got_object_idset *idset, struct got_pack_metavec *v) -{ - struct got_pack_meta *m, *base; - - if (delta->delta_offset + delta->delta_size < delta->delta_offset || - delta->delta_offset + - delta->delta_compressed_size < delta->delta_offset) - return got_error(GOT_ERR_BAD_PACKFILE); - - m = got_object_idset_get(idset, &delta->id); - if (m == NULL) - return got_error(GOT_ERR_NO_OBJ); - - base = got_object_idset_get(idset, &delta->base_id); - if (base == NULL) - return got_error(GOT_ERR_NO_OBJ); - - m->delta_len = delta->delta_size; - m->delta_compressed_len = delta->delta_compressed_size; - m->delta_offset = delta->delta_out_offset; - m->prev = base; - m->size = delta->result_size; - m->reused_delta_offset = delta->delta_offset; - m->base_obj_id = got_object_id_dup(&delta->base_id); - if (m->base_obj_id == NULL) - return got_error_from_errno("got_object_id_dup"); - - return add_meta(m, v); -} - -static const struct got_error * -cache_pack_for_packidx(struct got_pack **pack, struct got_packidx *packidx, - struct got_repository *repo) -{ - const struct got_error *err; char *path_packfile = NULL; err = got_packidx_get_packfile_path(&path_packfile, @@ -634,106 +523,12 @@ cache_pack_for_packidx(struct got_pack **pack, struct if (err) goto done; } - if ((*pack)->privsep_child == NULL) { - err = got_pack_start_privsep_child(*pack, packidx); - if (err) - goto done; - } done: free(path_packfile); return err; } static const struct got_error * -prepare_delta_reuse(struct got_pack *pack, struct got_packidx *packidx, - int delta_outfd, struct got_repository *repo) -{ - const struct got_error *err = NULL; - - if (!pack->child_has_delta_outfd) { - int outfd_child; - outfd_child = dup(delta_outfd); - if (outfd_child == -1) { - err = got_error_from_errno("dup"); - goto done; - } - err = got_privsep_send_raw_delta_outfd( - pack->privsep_child->ibuf, outfd_child); - if (err) - goto done; - pack->child_has_delta_outfd = 1; - } - - err = got_privsep_send_delta_reuse_req(pack->privsep_child->ibuf); -done: - return err; -} - - -static const struct got_error * -search_deltas(struct got_pack_metavec *v, struct got_object_idset *idset, - int delta_cache_fd, int ncolored, int nfound, int ntrees, int ncommits, - struct got_repository *repo, - got_pack_progress_cb progress_cb, void *progress_arg, - struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) -{ - const struct got_error *err = NULL; - struct got_packidx *packidx; - struct got_pack *pack; - struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS]; - size_t ndeltas, i; - - err = find_pack_for_reuse(&packidx, repo); - if (err) - return err; - - if (packidx == NULL) - return NULL; - - err = cache_pack_for_packidx(&pack, packidx, repo); - if (err) - return err; - - err = prepare_delta_reuse(pack, packidx, delta_cache_fd, repo); - if (err) - return err; - - err = send_idset(pack->privsep_child->ibuf, idset); - if (err) - return err; - - for (;;) { - int done = 0; - - if (cancel_cb) { - err = (*cancel_cb)(cancel_arg); - if (err) - break; - } - - err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas, - pack->privsep_child->ibuf); - if (err || done) - break; - - for (i = 0; i < ndeltas; i++) { - struct got_imsg_reused_delta *delta = &deltas[i]; - err = recv_reused_delta(delta, idset, v); - if (err) - goto done; - } - - err = report_progress(progress_cb, progress_arg, rl, - ncolored, nfound, ntrees, 0L, ncommits, - got_object_idset_num_elements(idset), v->nmeta, 0); - if (err) - break; - } -done: - return err; -} - -static const struct got_error * pick_deltas(struct got_pack_meta **meta, int nmeta, int ncolored, int nfound, int ntrees, int ncommits, int nreused, FILE *delta_cache, struct got_repository *repo, @@ -761,7 +556,7 @@ pick_deltas(struct got_pack_meta **meta, int nmeta, in if (err) break; } - err = report_progress(progress_cb, progress_arg, rl, + err = got_pack_report_progress(progress_cb, progress_arg, rl, ncolored, nfound, ntrees, 0L, ncommits, nreused + nmeta, nreused + i, 0); if (err) @@ -924,8 +719,8 @@ static const struct got_error * return err; } -static const struct got_error * -add_object(int want_meta, struct got_object_idset *idset, +const struct got_error * +got_pack_add_object(int want_meta, struct got_object_idset *idset, struct got_object_id *id, const char *path, int obj_type, time_t mtime, uint32_t seed, int loose_obj_only, struct got_repository *repo, int *ncolored, int *nfound, int *ntrees, @@ -950,7 +745,7 @@ add_object(int want_meta, struct got_object_idset *ids return err; (*nfound)++; - err = report_progress(progress_cb, progress_arg, rl, + err = got_pack_report_progress(progress_cb, progress_arg, rl, *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0); if (err) { clear_meta(m); @@ -967,8 +762,8 @@ static const struct got_error * return err; } -static const struct got_error * -load_tree_entries(struct got_object_id_queue *ids, int want_meta, +const struct got_error * +got_pack_load_tree_entries(struct got_object_id_queue *ids, int want_meta, struct got_object_idset *idset, struct got_object_idset *idset_exclude, struct got_tree_object *tree, const char *dpath, time_t mtime, uint32_t seed, struct got_repository *repo, @@ -981,7 +776,7 @@ load_tree_entries(struct got_object_id_queue *ids, int int i; (*ntrees)++; - err = report_progress(progress_cb, progress_arg, rl, + err = got_pack_report_progress(progress_cb, progress_arg, rl, *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0); if (err) return err; @@ -1025,7 +820,7 @@ load_tree_entries(struct got_object_id_queue *ids, int p = NULL; STAILQ_INSERT_TAIL(ids, qid, entry); } else if (S_ISREG(mode) || S_ISLNK(mode)) { - err = add_object(want_meta, + err = got_pack_add_object(want_meta, want_meta ? idset : idset_exclude, id, p, GOT_OBJ_TYPE_BLOB, mtime, seed, loose_obj_only, repo, ncolored, nfound, ntrees, @@ -1044,8 +839,8 @@ static const struct got_error * return err; } -static const struct got_error * -load_tree(int want_meta, struct got_object_idset *idset, +const struct got_error * +got_pack_load_tree(int want_meta, struct got_object_idset *idset, struct got_object_idset *idset_exclude, struct got_object_id *tree_id, const char *dpath, time_t mtime, uint32_t seed, struct got_repository *repo, int loose_obj_only, @@ -1094,7 +889,8 @@ load_tree(int want_meta, struct got_object_idset *idse continue; } - err = add_object(want_meta, want_meta ? idset : idset_exclude, + err = got_pack_add_object(want_meta, + want_meta ? idset : idset_exclude, &qid->id, path, GOT_OBJ_TYPE_TREE, mtime, seed, loose_obj_only, repo, ncolored, nfound, ntrees, progress_cb, progress_arg, rl); @@ -1111,7 +907,7 @@ load_tree(int want_meta, struct got_object_idset *idse break; } - err = load_tree_entries(&tree_ids, want_meta, idset, + err = got_pack_load_tree_entries(&tree_ids, want_meta, idset, idset_exclude, tree, path, mtime, seed, repo, loose_obj_only, ncolored, nfound, ntrees, progress_cb, progress_arg, rl, @@ -1161,15 +957,15 @@ load_commit(int want_meta, struct got_object_idset *id if (err) return err; - err = add_object(want_meta, want_meta ? idset : idset_exclude, - id, "", GOT_OBJ_TYPE_COMMIT, + err = got_pack_add_object(want_meta, + want_meta ? idset : idset_exclude, id, "", GOT_OBJ_TYPE_COMMIT, got_object_commit_get_committer_time(commit), seed, loose_obj_only, repo, ncolored, nfound, ntrees, progress_cb, progress_arg, rl); if (err) goto done; - err = load_tree(want_meta, idset, idset_exclude, + err = got_pack_load_tree(want_meta, idset, idset_exclude, got_object_commit_get_tree_id(commit), "", got_object_commit_get_committer_time(commit), seed, repo, loose_obj_only, ncolored, nfound, ntrees, @@ -1207,8 +1003,8 @@ load_tag(int want_meta, struct got_object_idset *idset if (err) return err; - err = add_object(want_meta, want_meta ? idset : idset_exclude, - id, "", GOT_OBJ_TYPE_TAG, + err = got_pack_add_object(want_meta, + want_meta ? idset : idset_exclude, id, "", GOT_OBJ_TYPE_TAG, got_object_tag_get_tagger_time(tag), seed, loose_obj_only, repo, ncolored, nfound, ntrees, progress_cb, progress_arg, rl); if (err) @@ -1222,7 +1018,7 @@ load_tag(int want_meta, struct got_object_idset *idset progress_cb, progress_arg, rl, cancel_cb, cancel_arg); break; case GOT_OBJ_TYPE_TREE: - err = load_tree(want_meta, idset, idset_exclude, + err = got_pack_load_tree(want_meta, idset, idset_exclude, got_object_tag_get_object_id(tag), "", got_object_tag_get_tagger_time(tag), seed, repo, loose_obj_only, ncolored, nfound, ntrees, @@ -1237,15 +1033,8 @@ enum findtwixt_color { return err; } -enum findtwixt_color { - COLOR_KEEP = 0, - COLOR_DROP, - COLOR_SKIP, - COLOR_MAX, -}; - -static const struct got_error * -paint_commit(struct got_object_qid *qid, intptr_t color) +const struct got_error * +got_pack_paint_commit(struct got_object_qid *qid, intptr_t color) { if (color < 0 || color >= COLOR_MAX) return got_error(GOT_ERR_RANGE); @@ -1254,9 +1043,9 @@ static const struct got_error * return NULL; } -static const struct got_error * -queue_commit_id(struct got_object_id_queue *ids, struct got_object_id *id, - intptr_t color, struct got_repository *repo) +const struct got_error * +got_pack_queue_commit_id(struct got_object_id_queue *ids, + struct got_object_id *id, intptr_t color, struct got_repository *repo) { const struct got_error *err; struct got_object_qid *qid; @@ -1266,7 +1055,7 @@ queue_commit_id(struct got_object_id_queue *ids, struc return err; STAILQ_INSERT_TAIL(ids, qid, entry); - return paint_commit(qid, color); + return got_pack_paint_commit(qid, color); } struct append_id_arg { @@ -1313,7 +1102,7 @@ queue_commit_or_tag_id(struct got_object_id *id, intpt } if (obj_type == GOT_OBJ_TYPE_COMMIT) { - err = queue_commit_id(ids, id, color, repo); + err = got_pack_queue_commit_id(ids, id, color, repo); if (err) goto done; } @@ -1323,148 +1112,8 @@ struct recv_painted_commit_arg { return err; } -struct recv_painted_commit_arg { - int *ncolored; - int *nqueued; - int *nskip; - struct got_object_id_queue *ids; - struct got_object_idset *keep; - struct got_object_idset *drop; - struct got_object_idset *skip; - got_pack_progress_cb progress_cb; - void *progress_arg; - struct got_ratelimit *rl; - got_cancel_cb cancel_cb; - void *cancel_arg; -}; - -static const struct got_error * -recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color) -{ - const struct got_error *err = NULL; - struct recv_painted_commit_arg *a = arg; - struct got_object_qid *qid, *tmp; - - if (a->cancel_cb) { - err = a->cancel_cb(a->cancel_arg); - if (err) - return err; - } - - switch (color) { - case COLOR_KEEP: - err = got_object_idset_add(a->keep, id, NULL); - if (err) - return err; - (*a->ncolored)++; - break; - case COLOR_DROP: - err = got_object_idset_add(a->drop, id, NULL); - if (err) - return err; - (*a->ncolored)++; - break; - case COLOR_SKIP: - err = got_object_idset_add(a->skip, id, NULL); - if (err) - return err; - break; - default: - /* should not happen */ - return got_error_fmt(GOT_ERR_NOT_IMPL, - "%s invalid commit color %"PRIdPTR, __func__, color); - } - - STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) { - if (got_object_id_cmp(&qid->id, id) != 0) - continue; - STAILQ_REMOVE(a->ids, qid, got_object_qid, entry); - color = (intptr_t)qid->data; - got_object_qid_free(qid); - (*a->nqueued)--; - if (color == COLOR_SKIP) - (*a->nskip)--; - break; - } - - return report_progress(a->progress_cb, a->progress_arg, a->rl, - *a->ncolored, 0, 0, 0L, 0, 0, 0, 0); -} - -static const struct got_error * -paint_packed_commits(struct got_pack *pack, struct got_object_id *id, - int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip, - struct got_object_id_queue *ids, - struct got_object_idset *keep, struct got_object_idset *drop, - struct got_object_idset *skip, struct got_repository *repo, - got_pack_progress_cb progress_cb, void *progress_arg, - struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) -{ - const struct got_error *err = NULL; - struct got_object_id_queue next_ids; - struct got_object_qid *qid, *tmp; - struct recv_painted_commit_arg arg; - - STAILQ_INIT(&next_ids); - - err = got_privsep_send_painting_request(pack->privsep_child->ibuf, - idx, id, color); - if (err) - return err; - - arg.ncolored = ncolored; - arg.nqueued = nqueued; - arg.nskip = nskip; - arg.ids = ids; - arg.keep = keep; - arg.drop = drop; - arg.skip = skip; - arg.progress_cb = progress_cb; - arg.progress_arg = progress_arg; - arg.rl = rl; - arg.cancel_cb = cancel_cb; - arg.cancel_arg = cancel_arg; - err = got_privsep_recv_painted_commits(&next_ids, - recv_painted_commit, &arg, pack->privsep_child->ibuf); - if (err) - return err; - - STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) { - struct got_object_qid *old_id; - intptr_t qcolor, ocolor; - STAILQ_FOREACH(old_id, ids, entry) { - if (got_object_id_cmp(&qid->id, &old_id->id)) - continue; - qcolor = (intptr_t)qid->data; - ocolor = (intptr_t)old_id->data; - STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry); - got_object_qid_free(qid); - qid = NULL; - if (qcolor != ocolor) { - paint_commit(old_id, qcolor); - if (ocolor == COLOR_SKIP) - (*nskip)--; - else if (qcolor == COLOR_SKIP) - (*nskip)++; - } - break; - } - } - while (!STAILQ_EMPTY(&next_ids)) { - qid = STAILQ_FIRST(&next_ids); - STAILQ_REMOVE_HEAD(&next_ids, entry); - paint_commit(qid, color); - STAILQ_INSERT_TAIL(ids, qid, entry); - (*nqueued)++; - if (color == COLOR_SKIP) - (*nskip)++; - } - - return err; -} - -static const struct got_error * -find_pack_for_commit_painting(struct got_packidx **best_packidx, +const struct got_error * +got_pack_find_pack_for_commit_painting(struct got_packidx **best_packidx, struct got_object_id_queue *ids, int nids, struct got_repository *repo) { const struct got_error *err = NULL; @@ -1514,191 +1163,6 @@ paint_commits(int *ncolored, struct got_object_id_queu } static const struct got_error * -paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids, - struct got_object_idset *keep, struct got_object_idset *drop, - struct got_object_idset *skip, struct got_repository *repo, - got_pack_progress_cb progress_cb, void *progress_arg, - struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) -{ - const struct got_error *err = NULL; - struct got_commit_object *commit = NULL; - struct got_packidx *packidx = NULL; - struct got_pack *pack = NULL; - const struct got_object_id_queue *parents; - struct got_object_qid *qid = NULL; - int nqueued = nids, nskip = 0; - int idx; - - while (!STAILQ_EMPTY(ids) && nskip != nqueued) { - intptr_t color; - - if (cancel_cb) { - err = cancel_cb(cancel_arg); - if (err) - break; - } - - qid = STAILQ_FIRST(ids); - STAILQ_REMOVE_HEAD(ids, entry); - nqueued--; - color = (intptr_t)qid->data; - if (color == COLOR_SKIP) - nskip--; - - if (got_object_idset_contains(skip, &qid->id)) { - got_object_qid_free(qid); - qid = NULL; - continue; - } - if (color == COLOR_KEEP && - got_object_idset_contains(keep, &qid->id)) { - got_object_qid_free(qid); - qid = NULL; - continue; - } - if (color == COLOR_DROP && - got_object_idset_contains(drop, &qid->id)) { - got_object_qid_free(qid); - qid = NULL; - continue; - } - - /* Pinned pack may have moved to different cache slot. */ - pack = got_repo_get_pinned_pack(repo); - - if (packidx && pack) { - idx = got_packidx_get_object_idx(packidx, &qid->id); - if (idx != -1) { - err = paint_packed_commits(pack, &qid->id, - idx, color, ncolored, &nqueued, &nskip, - ids, keep, drop, skip, repo, - progress_cb, progress_arg, rl, - cancel_cb, cancel_arg); - if (err) - break; - got_object_qid_free(qid); - qid = NULL; - continue; - } - } - - switch (color) { - case COLOR_KEEP: - if (got_object_idset_contains(drop, &qid->id)) { - err = paint_commit(qid, COLOR_SKIP); - if (err) - goto done; - } else - (*ncolored)++; - err = got_object_idset_add(keep, &qid->id, NULL); - if (err) - goto done; - break; - case COLOR_DROP: - if (got_object_idset_contains(keep, &qid->id)) { - err = paint_commit(qid, COLOR_SKIP); - if (err) - goto done; - } else - (*ncolored)++; - err = got_object_idset_add(drop, &qid->id, NULL); - if (err) - goto done; - break; - case COLOR_SKIP: - if (!got_object_idset_contains(skip, &qid->id)) { - err = got_object_idset_add(skip, &qid->id, - NULL); - if (err) - goto done; - } - break; - default: - /* should not happen */ - err = got_error_fmt(GOT_ERR_NOT_IMPL, - "%s invalid commit color %"PRIdPTR, __func__, - color); - goto done; - } - - err = report_progress(progress_cb, progress_arg, rl, - *ncolored, 0, 0, 0L, 0, 0, 0, 0); - if (err) - break; - - err = got_object_open_as_commit(&commit, repo, &qid->id); - if (err) - break; - - parents = got_object_commit_get_parent_ids(commit); - if (parents) { - struct got_object_qid *pid; - color = (intptr_t)qid->data; - STAILQ_FOREACH(pid, parents, entry) { - err = queue_commit_id(ids, &pid->id, - color, repo); - if (err) - break; - nqueued++; - if (color == COLOR_SKIP) - nskip++; - } - } - - if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) { - if (packidx == NULL) { - err = find_pack_for_commit_painting(&packidx, - ids, nqueued, repo); - if (err) - goto done; - } - if (packidx != NULL) { - err = cache_pack_for_packidx(&pack, packidx, - repo); - if (err) - goto done; - err = got_privsep_init_commit_painting( - pack->privsep_child->ibuf); - if (err) - goto done; - err = send_idset(pack->privsep_child->ibuf, - keep); - if (err) - goto done; - err = send_idset(pack->privsep_child->ibuf, drop); - if (err) - goto done; - err = send_idset(pack->privsep_child->ibuf, skip); - if (err) - goto done; - err = got_repo_pin_pack(repo, packidx, pack); - if (err) - goto done; - } - } - - got_object_commit_close(commit); - commit = NULL; - - got_object_qid_free(qid); - qid = NULL; - } -done: - if (pack) { - const struct got_error *pack_err; - pack_err = got_privsep_send_painting_commits_done( - pack->privsep_child->ibuf); - if (err == NULL) - err = pack_err; - } - if (commit) - got_object_commit_close(commit); - got_object_qid_free(qid); - got_repo_unpin_pack(repo); - return err; -} - -static const struct got_error * findtwixt(struct got_object_id ***res, int *nres, int *ncolored, struct got_object_id **head, int nhead, struct got_object_id **tail, int ntail, @@ -1750,7 +1214,7 @@ findtwixt(struct got_object_id ***res, int *nres, int goto done; } - err = paint_commits(ncolored, &ids, nhead + ntail, + err = got_pack_paint_commits(ncolored, &ids, nhead + ntail, keep, drop, skip, repo, progress_cb, progress_arg, rl, cancel_cb, cancel_arg); if (err) @@ -1784,159 +1248,7 @@ struct load_packed_obj_arg { return err; } -struct load_packed_obj_arg { - /* output parameters: */ - struct got_object_id *id; - char *dpath; - time_t mtime; - - /* input parameters: */ - uint32_t seed; - int want_meta; - struct got_object_idset *idset; - struct got_object_idset *idset_exclude; - int loose_obj_only; - int *ncolored; - int *nfound; - int *ntrees; - got_pack_progress_cb progress_cb; - void *progress_arg; - struct got_ratelimit *rl; - got_cancel_cb cancel_cb; - void *cancel_arg; -}; - static const struct got_error * -load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id, - struct got_repository *repo) -{ - struct load_packed_obj_arg *a = arg; - - if (got_object_idset_contains(a->idset, id) || - got_object_idset_contains(a->idset_exclude, id)) - return NULL; - - return add_object(a->want_meta, - a->want_meta ? a->idset : a->idset_exclude, - id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only, - repo, a->ncolored, a->nfound, a->ntrees, - a->progress_cb, a->progress_arg, a->rl); -} - -static const struct got_error * -load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime, - struct got_object_id *id, const char *dpath, struct got_repository *repo) -{ - const struct got_error *err; - struct load_packed_obj_arg *a = arg; - const char *relpath; - - /* - * When we receive a tree's ID and path but not the tree itself, - * this tree object was not found in the pack file. This is the - * last time we are being called for this optimized traversal. - * Return from here and switch to loading objects the slow way. - */ - if (tree == NULL) { - free(a->id); - a->id = got_object_id_dup(id); - if (a->id == NULL) { - err = got_error_from_errno("got_object_id_dup"); - free(a->dpath); - a->dpath = NULL; - return err; - } - - free(a->dpath); - a->dpath = strdup(dpath); - if (a->dpath == NULL) { - err = got_error_from_errno("strdup"); - free(a->id); - a->id = NULL; - return err; - } - - a->mtime = mtime; - return NULL; - } - - if (got_object_idset_contains(a->idset, id) || - got_object_idset_contains(a->idset_exclude, id)) - return NULL; - - relpath = dpath; - while (relpath[0] == '/') - relpath++; - - err = add_object(a->want_meta, - a->want_meta ? a->idset : a->idset_exclude, - id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed, - a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees, - a->progress_cb, a->progress_arg, a->rl); - if (err) - return err; - - return load_tree_entries(NULL, a->want_meta, a->idset, - a->idset_exclude, tree, dpath, mtime, a->seed, repo, - a->loose_obj_only, a->ncolored, a->nfound, a->ntrees, - a->progress_cb, a->progress_arg, a->rl, - a->cancel_cb, a->cancel_arg); -} - -static const struct got_error * -load_packed_object_ids(int *found_all_objects, - struct got_object_id **ours, int nours, - struct got_object_id **theirs, int ntheirs, - int want_meta, uint32_t seed, struct got_object_idset *idset, - struct got_object_idset *idset_exclude, int loose_obj_only, - struct got_repository *repo, struct got_packidx *packidx, - int *ncolored, int *nfound, int *ntrees, - got_pack_progress_cb progress_cb, void *progress_arg, - struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) -{ - const struct got_error *err = NULL; - struct load_packed_obj_arg lpa; - - memset(&lpa, 0, sizeof(lpa)); - lpa.seed = seed; - lpa.want_meta = want_meta; - lpa.idset = idset; - lpa.idset_exclude = idset_exclude; - lpa.loose_obj_only = loose_obj_only; - lpa.ncolored = ncolored; - lpa.nfound = nfound; - lpa.ntrees = ntrees; - lpa.progress_cb = progress_cb; - lpa.progress_arg = progress_arg; - lpa.rl = rl; - lpa.cancel_cb = cancel_cb; - lpa.cancel_arg = cancel_arg; - - /* Attempt to load objects via got-read-pack, as far as possible. */ - err = got_object_enumerate(found_all_objects, load_packed_commit_id, - load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs, - packidx, repo); - if (err) - return err; - - if (lpa.id == NULL) - return NULL; - - /* - * An incomplete tree hierarchy was present in the pack file - * and caused loading to be aborted. - * Continue loading trees the slow way. - */ - err = load_tree(want_meta, idset, idset_exclude, - lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only, - ncolored, nfound, ntrees, progress_cb, progress_arg, rl, - cancel_cb, cancel_arg); - free(lpa.id); - free(lpa.dpath); - return err; -} - -static const struct got_error * find_pack_for_enumeration(struct got_packidx **best_packidx, struct got_object_id **ids, int nids, struct got_repository *repo) { @@ -2016,7 +1328,7 @@ load_object_ids(int *ncolored, int *nfound, int *ntree if (err) goto done; if (packidx) { - err = load_packed_object_ids(&found_all_objects, + err = got_pack_load_packed_object_ids(&found_all_objects, theirs, ntheirs, NULL, 0, 0, seed, idset, idset_exclude, loose_obj_only, repo, packidx, ncolored, nfound, ntrees, progress_cb, progress_arg, rl, cancel_cb, cancel_arg); @@ -2056,7 +1368,7 @@ load_object_ids(int *ncolored, int *nfound, int *ntree if (err) goto done; if (packidx) { - err = load_packed_object_ids(&found_all_objects, ids, + err = got_pack_load_packed_object_ids(&found_all_objects, ids, nobj, theirs, ntheirs, 1, seed, idset, idset_exclude, loose_obj_only, repo, packidx, ncolored, nfound, ntrees, progress_cb, progress_arg, rl, cancel_cb, cancel_arg); @@ -2440,7 +1752,7 @@ genpack(uint8_t *pack_sha1, int packfd, FILE *delta_ca qsort(deltify, ndeltify, sizeof(struct got_pack_meta *), write_order_cmp); for (i = 0; i < ndeltify; i++) { - err = report_progress(progress_cb, progress_arg, rl, + err = got_pack_report_progress(progress_cb, progress_arg, rl, ncolored, nfound, ntrees, packfile_size, nours, ndeltify + nreuse, ndeltify + nreuse, i); if (err) @@ -2456,7 +1768,7 @@ genpack(uint8_t *pack_sha1, int packfd, FILE *delta_ca qsort(reuse, nreuse, sizeof(struct got_pack_meta *), reuse_write_order_cmp); for (i = 0; i < nreuse; i++) { - err = report_progress(progress_cb, progress_arg, rl, + err = got_pack_report_progress(progress_cb, progress_arg, rl, ncolored, nfound, ntrees, packfile_size, nours, ndeltify + nreuse, ndeltify + nreuse, ndeltify + i); if (err) @@ -2505,7 +1817,7 @@ add_meta_idset_cb(struct got_object_id *id, void *data if (m->reused_delta_offset != 0) return NULL; - return add_meta(m, v); + return got_pack_add_meta(m, v); } const struct got_error * @@ -2568,9 +1880,9 @@ got_pack_create(uint8_t *packsha1, int packfd, FILE *d goto done; } - err = search_deltas(&reuse, idset, delta_cache_fd, ncolored, nfound, - ntrees, nours, repo, progress_cb, progress_arg, &rl, - cancel_cb, cancel_arg); + err = got_pack_search_deltas(&reuse, idset, delta_cache_fd, + ncolored, nfound, ntrees, nours, + repo, progress_cb, progress_arg, &rl, cancel_cb, cancel_arg); if (err) goto done; blob - /dev/null blob + 5180371c65007227a78d76ba0d4b63cd2ee65675 (mode 644) --- /dev/null +++ lib/pack_create_privsep.c @@ -0,0 +1,702 @@ +/* + * Copyright (c) 2020 Ori Bernstein + * Copyright (c) 2021, 2022 Stefan Sperling <stsp@openbsd.org> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <sys/types.h> +#include <sys/queue.h> +#include <sys/tree.h> +#include <sys/uio.h> + +#include <sha1.h> +#include <limits.h> +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <imsg.h> +#include <inttypes.h> +#include <unistd.h> + +#include "got_error.h" +#include "got_cancel.h" +#include "got_object.h" +#include "got_reference.h" +#include "got_repository_admin.h" +#include "got_path.h" + +#include "got_lib_delta.h" +#include "got_lib_object.h" +#include "got_lib_object_cache.h" +#include "got_lib_object_idset.h" +#include "got_lib_privsep.h" +#include "got_lib_pack.h" +#include "got_lib_pack_create.h" +#include "got_lib_repository.h" + +struct send_id_arg { + struct imsgbuf *ibuf; + struct got_object_id *ids[GOT_IMSG_OBJ_ID_LIST_MAX_NIDS]; + size_t nids; +}; + +static const struct got_error * +send_id(struct got_object_id *id, void *data, void *arg) +{ + const struct got_error *err = NULL; + struct send_id_arg *a = arg; + + a->ids[a->nids++] = id; + + if (a->nids >= GOT_IMSG_OBJ_ID_LIST_MAX_NIDS) { + err = got_privsep_send_object_idlist(a->ibuf, a->ids, a->nids); + if (err) + return err; + a->nids = 0; + } + + return NULL; +} + +static const struct got_error * +send_idset(struct imsgbuf *ibuf, struct got_object_idset *idset) +{ + const struct got_error *err; + struct send_id_arg sia; + + memset(&sia, 0, sizeof(sia)); + sia.ibuf = ibuf; + err = got_object_idset_for_each(idset, send_id, &sia); + if (err) + return err; + + if (sia.nids > 0) { + err = got_privsep_send_object_idlist(ibuf, sia.ids, sia.nids); + if (err) + return err; + } + + return got_privsep_send_object_idlist_done(ibuf); +} + +static const struct got_error * +recv_reused_delta(struct got_imsg_reused_delta *delta, + struct got_object_idset *idset, struct got_pack_metavec *v) +{ + struct got_pack_meta *m, *base; + + if (delta->delta_offset + delta->delta_size < delta->delta_offset || + delta->delta_offset + + delta->delta_compressed_size < delta->delta_offset) + return got_error(GOT_ERR_BAD_PACKFILE); + + m = got_object_idset_get(idset, &delta->id); + if (m == NULL) + return got_error(GOT_ERR_NO_OBJ); + + base = got_object_idset_get(idset, &delta->base_id); + if (base == NULL) + return got_error(GOT_ERR_NO_OBJ); + + m->delta_len = delta->delta_size; + m->delta_compressed_len = delta->delta_compressed_size; + m->delta_offset = delta->delta_out_offset; + m->prev = base; + m->size = delta->result_size; + m->reused_delta_offset = delta->delta_offset; + m->base_obj_id = got_object_id_dup(&delta->base_id); + if (m->base_obj_id == NULL) + return got_error_from_errno("got_object_id_dup"); + + return got_pack_add_meta(m, v); +} + +static const struct got_error * +prepare_delta_reuse(struct got_pack *pack, struct got_packidx *packidx, + int delta_outfd, struct got_repository *repo) +{ + const struct got_error *err = NULL; + + if (!pack->child_has_delta_outfd) { + int outfd_child; + outfd_child = dup(delta_outfd); + if (outfd_child == -1) { + err = got_error_from_errno("dup"); + goto done; + } + err = got_privsep_send_raw_delta_outfd( + pack->privsep_child->ibuf, outfd_child); + if (err) + goto done; + pack->child_has_delta_outfd = 1; + } + + err = got_privsep_send_delta_reuse_req(pack->privsep_child->ibuf); +done: + return err; +} + +const struct got_error * +got_pack_search_deltas(struct got_pack_metavec *v, + struct got_object_idset *idset, int delta_cache_fd, + int ncolored, int nfound, int ntrees, int ncommits, + struct got_repository *repo, + got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) +{ + const struct got_error *err = NULL; + struct got_packidx *packidx; + struct got_pack *pack; + struct got_imsg_reused_delta deltas[GOT_IMSG_REUSED_DELTAS_MAX_NDELTAS]; + size_t ndeltas, i; + + err = got_pack_find_pack_for_reuse(&packidx, repo); + if (err) + return err; + + if (packidx == NULL) + return NULL; + + err = got_pack_cache_pack_for_packidx(&pack, packidx, repo); + if (err) + return err; + + if (pack->privsep_child == NULL) { + err = got_pack_start_privsep_child(pack, packidx); + if (err) + return err; + } + + err = prepare_delta_reuse(pack, packidx, delta_cache_fd, repo); + if (err) + return err; + + err = send_idset(pack->privsep_child->ibuf, idset); + if (err) + return err; + + for (;;) { + int done = 0; + + if (cancel_cb) { + err = (*cancel_cb)(cancel_arg); + if (err) + break; + } + + err = got_privsep_recv_reused_deltas(&done, deltas, &ndeltas, + pack->privsep_child->ibuf); + if (err || done) + break; + + for (i = 0; i < ndeltas; i++) { + struct got_imsg_reused_delta *delta = &deltas[i]; + err = recv_reused_delta(delta, idset, v); + if (err) + goto done; + } + + err = got_pack_report_progress(progress_cb, progress_arg, rl, + ncolored, nfound, ntrees, 0L, ncommits, + got_object_idset_num_elements(idset), v->nmeta, 0); + if (err) + break; + } +done: + return err; +} + +struct recv_painted_commit_arg { + int *ncolored; + int *nqueued; + int *nskip; + struct got_object_id_queue *ids; + struct got_object_idset *keep; + struct got_object_idset *drop; + struct got_object_idset *skip; + got_pack_progress_cb progress_cb; + void *progress_arg; + struct got_ratelimit *rl; + got_cancel_cb cancel_cb; + void *cancel_arg; +}; + +static const struct got_error * +recv_painted_commit(void *arg, struct got_object_id *id, intptr_t color) +{ + const struct got_error *err = NULL; + struct recv_painted_commit_arg *a = arg; + struct got_object_qid *qid, *tmp; + + if (a->cancel_cb) { + err = a->cancel_cb(a->cancel_arg); + if (err) + return err; + } + + switch (color) { + case COLOR_KEEP: + err = got_object_idset_add(a->keep, id, NULL); + if (err) + return err; + (*a->ncolored)++; + break; + case COLOR_DROP: + err = got_object_idset_add(a->drop, id, NULL); + if (err) + return err; + (*a->ncolored)++; + break; + case COLOR_SKIP: + err = got_object_idset_add(a->skip, id, NULL); + if (err) + return err; + break; + default: + /* should not happen */ + return got_error_fmt(GOT_ERR_NOT_IMPL, + "%s invalid commit color %"PRIdPTR, __func__, color); + } + + STAILQ_FOREACH_SAFE(qid, a->ids, entry, tmp) { + if (got_object_id_cmp(&qid->id, id) != 0) + continue; + STAILQ_REMOVE(a->ids, qid, got_object_qid, entry); + color = (intptr_t)qid->data; + got_object_qid_free(qid); + (*a->nqueued)--; + if (color == COLOR_SKIP) + (*a->nskip)--; + break; + } + + return got_pack_report_progress(a->progress_cb, a->progress_arg, a->rl, + *a->ncolored, 0, 0, 0L, 0, 0, 0, 0); +} + +static const struct got_error * +paint_packed_commits(struct got_pack *pack, struct got_object_id *id, + int idx, intptr_t color, int *ncolored, int *nqueued, int *nskip, + struct got_object_id_queue *ids, + struct got_object_idset *keep, struct got_object_idset *drop, + struct got_object_idset *skip, struct got_repository *repo, + got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) +{ + const struct got_error *err = NULL; + struct got_object_id_queue next_ids; + struct got_object_qid *qid, *tmp; + struct recv_painted_commit_arg arg; + + STAILQ_INIT(&next_ids); + + err = got_privsep_send_painting_request(pack->privsep_child->ibuf, + idx, id, color); + if (err) + return err; + + arg.ncolored = ncolored; + arg.nqueued = nqueued; + arg.nskip = nskip; + arg.ids = ids; + arg.keep = keep; + arg.drop = drop; + arg.skip = skip; + arg.progress_cb = progress_cb; + arg.progress_arg = progress_arg; + arg.rl = rl; + arg.cancel_cb = cancel_cb; + arg.cancel_arg = cancel_arg; + err = got_privsep_recv_painted_commits(&next_ids, + recv_painted_commit, &arg, pack->privsep_child->ibuf); + if (err) + return err; + + STAILQ_FOREACH_SAFE(qid, &next_ids, entry, tmp) { + struct got_object_qid *old_id; + intptr_t qcolor, ocolor; + STAILQ_FOREACH(old_id, ids, entry) { + if (got_object_id_cmp(&qid->id, &old_id->id)) + continue; + qcolor = (intptr_t)qid->data; + ocolor = (intptr_t)old_id->data; + STAILQ_REMOVE(&next_ids, qid, got_object_qid, entry); + got_object_qid_free(qid); + qid = NULL; + if (qcolor != ocolor) { + got_pack_paint_commit(old_id, qcolor); + if (ocolor == COLOR_SKIP) + (*nskip)--; + else if (qcolor == COLOR_SKIP) + (*nskip)++; + } + break; + } + } + while (!STAILQ_EMPTY(&next_ids)) { + qid = STAILQ_FIRST(&next_ids); + STAILQ_REMOVE_HEAD(&next_ids, entry); + got_pack_paint_commit(qid, color); + STAILQ_INSERT_TAIL(ids, qid, entry); + (*nqueued)++; + if (color == COLOR_SKIP) + (*nskip)++; + } + + return err; +} + +const struct got_error * +got_pack_paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids, + struct got_object_idset *keep, struct got_object_idset *drop, + struct got_object_idset *skip, struct got_repository *repo, + got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) +{ + const struct got_error *err = NULL; + struct got_commit_object *commit = NULL; + struct got_packidx *packidx = NULL; + struct got_pack *pack = NULL; + const struct got_object_id_queue *parents; + struct got_object_qid *qid = NULL; + int nqueued = nids, nskip = 0; + int idx; + + while (!STAILQ_EMPTY(ids) && nskip != nqueued) { + intptr_t color; + + if (cancel_cb) { + err = cancel_cb(cancel_arg); + if (err) + break; + } + + qid = STAILQ_FIRST(ids); + STAILQ_REMOVE_HEAD(ids, entry); + nqueued--; + color = (intptr_t)qid->data; + if (color == COLOR_SKIP) + nskip--; + + if (got_object_idset_contains(skip, &qid->id)) { + got_object_qid_free(qid); + qid = NULL; + continue; + } + if (color == COLOR_KEEP && + got_object_idset_contains(keep, &qid->id)) { + got_object_qid_free(qid); + qid = NULL; + continue; + } + if (color == COLOR_DROP && + got_object_idset_contains(drop, &qid->id)) { + got_object_qid_free(qid); + qid = NULL; + continue; + } + + /* Pinned pack may have moved to different cache slot. */ + pack = got_repo_get_pinned_pack(repo); + + if (packidx && pack) { + idx = got_packidx_get_object_idx(packidx, &qid->id); + if (idx != -1) { + err = paint_packed_commits(pack, &qid->id, + idx, color, ncolored, &nqueued, &nskip, + ids, keep, drop, skip, repo, + progress_cb, progress_arg, rl, + cancel_cb, cancel_arg); + if (err) + break; + got_object_qid_free(qid); + qid = NULL; + continue; + } + } + + switch (color) { + case COLOR_KEEP: + if (got_object_idset_contains(drop, &qid->id)) { + err = got_pack_paint_commit(qid, COLOR_SKIP); + if (err) + goto done; + } else + (*ncolored)++; + err = got_object_idset_add(keep, &qid->id, NULL); + if (err) + goto done; + break; + case COLOR_DROP: + if (got_object_idset_contains(keep, &qid->id)) { + err = got_pack_paint_commit(qid, COLOR_SKIP); + if (err) + goto done; + } else + (*ncolored)++; + err = got_object_idset_add(drop, &qid->id, NULL); + if (err) + goto done; + break; + case COLOR_SKIP: + if (!got_object_idset_contains(skip, &qid->id)) { + err = got_object_idset_add(skip, &qid->id, + NULL); + if (err) + goto done; + } + break; + default: + /* should not happen */ + err = got_error_fmt(GOT_ERR_NOT_IMPL, + "%s invalid commit color %"PRIdPTR, __func__, + color); + goto done; + } + + err = got_pack_report_progress(progress_cb, progress_arg, rl, + *ncolored, 0, 0, 0L, 0, 0, 0, 0); + if (err) + break; + + err = got_object_open_as_commit(&commit, repo, &qid->id); + if (err) + break; + + parents = got_object_commit_get_parent_ids(commit); + if (parents) { + struct got_object_qid *pid; + color = (intptr_t)qid->data; + STAILQ_FOREACH(pid, parents, entry) { + err = got_pack_queue_commit_id(ids, &pid->id, + color, repo); + if (err) + break; + nqueued++; + if (color == COLOR_SKIP) + nskip++; + } + } + + if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) { + if (packidx == NULL) { + err = got_pack_find_pack_for_commit_painting( + &packidx, ids, nqueued, repo); + if (err) + goto done; + } + if (packidx != NULL) { + err = got_pack_cache_pack_for_packidx(&pack, + packidx, repo); + if (err) + goto done; + if (pack->privsep_child == NULL) { + err = got_pack_start_privsep_child( + pack, packidx); + if (err) + goto done; + } + err = got_privsep_init_commit_painting( + pack->privsep_child->ibuf); + if (err) + goto done; + err = send_idset(pack->privsep_child->ibuf, + keep); + if (err) + goto done; + err = send_idset(pack->privsep_child->ibuf, drop); + if (err) + goto done; + err = send_idset(pack->privsep_child->ibuf, skip); + if (err) + goto done; + err = got_repo_pin_pack(repo, packidx, pack); + if (err) + goto done; + } + } + + got_object_commit_close(commit); + commit = NULL; + + got_object_qid_free(qid); + qid = NULL; + } +done: + if (pack) { + const struct got_error *pack_err; + pack_err = got_privsep_send_painting_commits_done( + pack->privsep_child->ibuf); + if (err == NULL) + err = pack_err; + } + if (commit) + got_object_commit_close(commit); + got_object_qid_free(qid); + got_repo_unpin_pack(repo); + return err; +} + +struct load_packed_obj_arg { + /* output parameters: */ + struct got_object_id *id; + char *dpath; + time_t mtime; + + /* input parameters: */ + uint32_t seed; + int want_meta; + struct got_object_idset *idset; + struct got_object_idset *idset_exclude; + int loose_obj_only; + int *ncolored; + int *nfound; + int *ntrees; + got_pack_progress_cb progress_cb; + void *progress_arg; + struct got_ratelimit *rl; + got_cancel_cb cancel_cb; + void *cancel_arg; +}; + +static const struct got_error * +load_packed_commit_id(void *arg, time_t mtime, struct got_object_id *id, + struct got_repository *repo) +{ + struct load_packed_obj_arg *a = arg; + + if (got_object_idset_contains(a->idset, id) || + got_object_idset_contains(a->idset_exclude, id)) + return NULL; + + return got_pack_add_object(a->want_meta, + a->want_meta ? a->idset : a->idset_exclude, + id, "", GOT_OBJ_TYPE_COMMIT, mtime, a->seed, a->loose_obj_only, + repo, a->ncolored, a->nfound, a->ntrees, + a->progress_cb, a->progress_arg, a->rl); +} + +static const struct got_error * +load_packed_tree_ids(void *arg, struct got_tree_object *tree, time_t mtime, + struct got_object_id *id, const char *dpath, struct got_repository *repo) +{ + const struct got_error *err; + struct load_packed_obj_arg *a = arg; + const char *relpath; + + /* + * When we receive a tree's ID and path but not the tree itself, + * this tree object was not found in the pack file. This is the + * last time we are being called for this optimized traversal. + * Return from here and switch to loading objects the slow way. + */ + if (tree == NULL) { + free(a->id); + a->id = got_object_id_dup(id); + if (a->id == NULL) { + err = got_error_from_errno("got_object_id_dup"); + free(a->dpath); + a->dpath = NULL; + return err; + } + + free(a->dpath); + a->dpath = strdup(dpath); + if (a->dpath == NULL) { + err = got_error_from_errno("strdup"); + free(a->id); + a->id = NULL; + return err; + } + + a->mtime = mtime; + return NULL; + } + + if (got_object_idset_contains(a->idset, id) || + got_object_idset_contains(a->idset_exclude, id)) + return NULL; + + relpath = dpath; + while (relpath[0] == '/') + relpath++; + + err = got_pack_add_object(a->want_meta, + a->want_meta ? a->idset : a->idset_exclude, + id, relpath, GOT_OBJ_TYPE_TREE, mtime, a->seed, + a->loose_obj_only, repo, a->ncolored, a->nfound, a->ntrees, + a->progress_cb, a->progress_arg, a->rl); + if (err) + return err; + + return got_pack_load_tree_entries(NULL, a->want_meta, a->idset, + a->idset_exclude, tree, dpath, mtime, a->seed, repo, + a->loose_obj_only, a->ncolored, a->nfound, a->ntrees, + a->progress_cb, a->progress_arg, a->rl, + a->cancel_cb, a->cancel_arg); +} + +const struct got_error * +got_pack_load_packed_object_ids(int *found_all_objects, + struct got_object_id **ours, int nours, + struct got_object_id **theirs, int ntheirs, + int want_meta, uint32_t seed, struct got_object_idset *idset, + struct got_object_idset *idset_exclude, int loose_obj_only, + struct got_repository *repo, struct got_packidx *packidx, + int *ncolored, int *nfound, int *ntrees, + got_pack_progress_cb progress_cb, void *progress_arg, + struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg) +{ + const struct got_error *err = NULL; + struct load_packed_obj_arg lpa; + + memset(&lpa, 0, sizeof(lpa)); + lpa.seed = seed; + lpa.want_meta = want_meta; + lpa.idset = idset; + lpa.idset_exclude = idset_exclude; + lpa.loose_obj_only = loose_obj_only; + lpa.ncolored = ncolored; + lpa.nfound = nfound; + lpa.ntrees = ntrees; + lpa.progress_cb = progress_cb; + lpa.progress_arg = progress_arg; + lpa.rl = rl; + lpa.cancel_cb = cancel_cb; + lpa.cancel_arg = cancel_arg; + + /* Attempt to load objects via got-read-pack, as far as possible. */ + err = got_object_enumerate(found_all_objects, load_packed_commit_id, + load_packed_tree_ids, &lpa, ours, nours, theirs, ntheirs, + packidx, repo); + if (err) + return err; + + if (lpa.id == NULL) + return NULL; + + /* + * An incomplete tree hierarchy was present in the pack file + * and caused loading to be aborted. + * Continue loading trees the slow way. + */ + err = got_pack_load_tree(want_meta, idset, idset_exclude, + lpa.id, lpa.dpath, lpa.mtime, seed, repo, loose_obj_only, + ncolored, nfound, ntrees, progress_cb, progress_arg, rl, + cancel_cb, cancel_arg); + free(lpa.id); + free(lpa.dpath); + return err; +}

2022-10-16 10:31 Stefan Sperling:
split up pack_create.c
- 2022-10-16 10:43 Omar Polo:
  split up pack_create.c
- - 2022-10-16 11:48 Stefan Sperling:
    split up pack_create.c