Download raw body.
Move got_opentempfd out of got_repo_open
On Thu, Jun 02, 2022 at 12:51:46PM +0200, Stefan Sperling wrote: > On Wed, Jun 01, 2022 at 10:22:49AM -0600, Tracey Emery wrote: > > Hello, > > > > Here is a first attempt to move got_opentempfd out of got_repo_open. Is > > this direction ok? I am not asking for a commit ok yet, just starting a > > conversation about the direction. > > > > This creates an array of fds twice the size of GOT_PACK_CACHE_SIZE to > > pass to got_repo_open by the caller. > > > > Thoughts? This move will remove the need for gotwebd to have > > /var/www/got/tmp finally. > > Not yet really, because at least lib/diffreg.c and lib/blame.c do > still have internal calls to got_opentemp(). Well, I can dream. One step closer then. :) > > > diff 2497f032fa6bc06264d8990fdd57a9ffbaf1429b /home/tracey/src/got > > blob - cf96009500ac33bb3b660dd7e99902801b0cb975 > > file + got/got.c > > --- got/got.c > > +++ got/got.c > > @@ -715,6 +715,8 @@ cmd_import(int argc, char *argv[]) > > struct got_pathlist_head ignores; > > struct got_pathlist_entry *pe; > > int preserve_logmsg = 0; > > + int cs = got_repo_get_pack_cache_size(); > > + int pack_fds[cs]; > > I think we should get rid of got_repo_get_pack_cache_size(). > > Instead, we could declare GOT_PACK_CACHE_SIZE in a public header, > and perhaps declare GOT_PACK_CACHE_SIZE * 2 as another macro > with a different name, which could then be used to set the > size of the pack_fds array at compile-time. > > Alternatively, allocate the array dynamically in got_repo_pack_fds_init() > and return the number of elements in addition to returning the array, so > the caller could do something like this: > > int *fds = NULL; > size_t nfds; > > err = got_repo_pack_fds_init(&fds, &nfds); > [...] > done: > got_repo_pack_fds_close(fds); /* closes files and then does a free(fds) */ > > > And got_repo_pack_fds_open() might be a better name than _init(). > > > @@ -703,12 +742,12 @@ got_repo_open(struct got_repository **repop, const cha > > repo->pack_cache_size = rl.rlim_cur / 8; > > for (i = 0; i < nitems(repo->packs); i++) { > > if (i < repo->pack_cache_size) { > > - repo->packs[i].basefd = got_opentempfd(); > > + repo->packs[i].basefd = dup(pack_fds[j++]); > > Is use of dup() really necessary? > This doubles the amount of file descriptors required, doesn't it? > Is it not enough to require the caller to create one set of file > descriptors per repository instance they want to use? > For got and tog, no. I had gotwebd on the brain. gotwebd can certainly dup its own, since it needs to keep them open. Removing this also removes got_repo_pack_fds_close(), since those are closed in repository.c, unless we should take the close operation out of there and leave it up to the caller to close what they've opened with got_repo_pack_fds_open(). So this is, got_repo_pack_fds_open(), got_repo_open(), free(pack_fds). Getting closer. Thoughts? -- Tracey Emery diff 6a881297759a4b3c4f62093aef5fdbaad0dae8df /home/tracey/src/got blob - d88f23ce5b63a9d80862b5fac05c212d054a0b74 file + got/got.c --- got/got.c +++ got/got.c @@ -58,6 +58,7 @@ #include "got_gotconfig.h" #include "got_dial.h" #include "got_patch.h" +#include "got_pack_cache_sizes.h" #ifndef nitems #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0])) @@ -715,6 +716,7 @@ cmd_import(int argc, char *argv[]) struct got_pathlist_head ignores; struct got_pathlist_entry *pe; int preserve_logmsg = 0; + int *pack_fds = NULL; TAILQ_INIT(&ignores); @@ -773,9 +775,13 @@ cmd_import(int argc, char *argv[]) error = get_gitconfig_path(&gitconfig_path); if (error) goto done; - error = got_repo_open(&repo, repo_path, gitconfig_path); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, repo_path, gitconfig_path, pack_fds); if (error) goto done; + free(pack_fds); error = get_author(&author, repo, NULL); if (error) @@ -1484,6 +1490,7 @@ cmd_clone(int argc, char *argv[]) char *git_url = NULL; int verbosity = 0, fetch_all_branches = 0, mirror_references = 0; int list_refs_only = 0; + int *pack_fds = NULL; TAILQ_INIT(&refs); TAILQ_INIT(&symrefs); @@ -1628,9 +1635,13 @@ cmd_clone(int argc, char *argv[]) error = got_repo_init(repo_path); if (error) goto done; - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); if (error) goto done; + free(pack_fds); } fpa.last_scaled_size[0] = '\0'; @@ -2212,6 +2223,7 @@ cmd_fetch(int argc, char *argv[]) struct got_fetch_progress_arg fpa; int verbosity = 0, fetch_all_branches = 0, list_refs_only = 0; int delete_refs = 0, replace_tags = 0, delete_remote = 0; + int *pack_fds = NULL; TAILQ_INIT(&refs); TAILQ_INIT(&symrefs); @@ -2333,10 +2345,16 @@ cmd_fetch(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + + error = got_repo_open(&repo, repo_path, NULL, pack_fds); if (error) goto done; + free(pack_fds); + if (delete_remote) { error = delete_refs_for_remote(repo, remote_name); goto done; /* nothing else to do */ @@ -2848,6 +2866,7 @@ cmd_checkout(int argc, char *argv[]) int ch, same_path_prefix, allow_nonempty = 0, verbosity = 0; struct got_pathlist_head paths; struct got_checkout_progress_arg cpa; + int *pack_fds = NULL; TAILQ_INIT(&paths); @@ -2936,10 +2955,16 @@ cmd_checkout(int argc, char *argv[]) got_path_strip_trailing_slashes(repo_path); got_path_strip_trailing_slashes(worktree_path); - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + + free(pack_fds); + /* Pre-create work tree path for unveil(2) */ error = got_path_mkdir(worktree_path); if (error) { @@ -3281,11 +3306,18 @@ wrap_not_worktree_error(const struct got_error *orig_e const struct got_error *err; struct got_repository *repo; static char msg[512]; + int *pack_fds = NULL; - err = got_repo_open(&repo, path, NULL); + err = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (err) + return err; + + err = got_repo_open(&repo, path, NULL, pack_fds); + if (err) return orig_err; + free(pack_fds); + snprintf(msg, sizeof(msg), "'got %s' needs a work tree in addition to a git repository\n" "Work trees can be checked out from this Git repository with " @@ -3311,6 +3343,7 @@ cmd_update(int argc, char *argv[]) struct got_pathlist_entry *pe; int ch, verbosity = 0; struct got_update_progress_arg upa; + int *pack_fds = NULL; TAILQ_INIT(&paths); @@ -3358,11 +3391,17 @@ cmd_update(int argc, char *argv[]) if (error) goto done; + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 0, got_worktree_get_root_path(worktree)); if (error) @@ -4120,6 +4159,7 @@ cmd_log(int argc, char *argv[]) const char *errstr; struct got_reflist_head refs; struct got_reflist_object_id_map *refs_idmap = NULL; + int *pack_fds = NULL; TAILQ_INIT(&refs); @@ -4227,10 +4267,16 @@ cmd_log(int argc, char *argv[]) goto done; } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 1, worktree ? got_worktree_get_root_path(worktree) : NULL); if (error) @@ -4579,6 +4625,7 @@ cmd_diff(int argc, char *argv[]) struct got_pathlist_head paths; struct got_pathlist_entry *pe; FILE *f1 = NULL, *f2 = NULL; + int *pack_fds = NULL; TAILQ_INIT(&refs); TAILQ_INIT(&paths); @@ -4660,11 +4707,17 @@ cmd_diff(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + + error = got_repo_open(&repo, repo_path, NULL, pack_fds); free(repo_path); if (error != NULL) goto done; + free(pack_fds); + if (rflag || worktree == NULL || ncommit_args > 0) { if (force_path) { error = got_error_msg(GOT_ERR_NOT_IMPL, @@ -5061,6 +5114,7 @@ cmd_blame(int argc, char *argv[]) struct blame_cb_args bca; int ch, obj_type, i; off_t filesize; + int *pack_fds = NULL; memset(&bca, 0, sizeof(bca)); @@ -5124,10 +5178,16 @@ cmd_blame(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + + free(pack_fds); + if (worktree) { const char *prefix = got_worktree_get_path_prefix(worktree); char *p; @@ -5402,7 +5462,9 @@ cmd_tree(int argc, char *argv[]) char *commit_id_str = NULL; int show_ids = 0, recurse = 0; int ch; + int *pack_fds = NULL; + #ifndef PROFILE if (pledge("stdio rpath wpath cpath flock proc exec sendfd unveil", NULL) == -1) @@ -5470,10 +5532,16 @@ cmd_tree(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + + free(pack_fds); + if (worktree) { const char *prefix = got_worktree_get_path_prefix(worktree); char *p; @@ -5624,7 +5692,9 @@ cmd_status(int argc, char *argv[]) struct got_pathlist_head paths; struct got_pathlist_entry *pe; int ch, i, no_ignores = 0; + int *pack_fds = NULL; + TAILQ_INIT(&paths); memset(&st, 0, sizeof(st)); @@ -5690,11 +5760,19 @@ cmd_status(int argc, char *argv[]) goto done; } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error != NULL) goto done; + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = apply_unveil(got_repo_get_path(repo), 1, got_worktree_get_root_path(worktree)); if (error) @@ -5848,6 +5926,7 @@ cmd_ref(int argc, char *argv[]) int ch, do_list = 0, do_delete = 0, sort_by_time = 0; const char *obj_arg = NULL, *symref_target= NULL; char *refname = NULL; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "c:dr:ls:t")) != -1) { switch (ch) { @@ -5953,10 +6032,18 @@ cmd_ref(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + #ifndef PROFILE if (do_list) { /* Remove "cpath" promise. */ @@ -6238,6 +6325,7 @@ cmd_branch(int argc, char *argv[]) struct got_pathlist_entry *pe; struct got_object_id *commit_id = NULL; char *commit_id_str = NULL; + int *pack_fds = NULL; TAILQ_INIT(&paths); @@ -6324,10 +6412,16 @@ cmd_branch(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + + free(pack_fds); + #ifndef PROFILE if (do_list || do_show) { /* Remove "cpath" promise. */ @@ -6796,6 +6890,7 @@ cmd_tag(int argc, char *argv[]) char *gitconfig_path = NULL, *tagger = NULL; const char *tag_name, *commit_id_arg = NULL, *tagmsg = NULL; int ch, do_list = 0; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "c:m:r:l")) != -1) { switch (ch) { @@ -6870,15 +6965,21 @@ cmd_tag(int argc, char *argv[]) } } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + if (do_list) { if (worktree) { /* Release work tree lock. */ got_worktree_close(worktree); worktree = NULL; } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_open(&repo, repo_path, NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + #ifndef PROFILE /* Remove "cpath" promise. */ if (pledge("stdio rpath wpath flock proc exec sendfd unveil", @@ -6893,10 +6994,13 @@ cmd_tag(int argc, char *argv[]) error = get_gitconfig_path(&gitconfig_path); if (error) goto done; - error = got_repo_open(&repo, repo_path, gitconfig_path); + error = got_repo_open(&repo, repo_path, gitconfig_path, + pack_fds); if (error != NULL) goto done; + free(pack_fds); + error = get_author(&tagger, repo, worktree); if (error) goto done; @@ -6976,6 +7080,7 @@ cmd_add(int argc, char *argv[]) struct got_pathlist_head paths; struct got_pathlist_entry *pe; int ch, can_recurse = 0, no_ignores = 0; + int *pack_fds = NULL; TAILQ_INIT(&paths); @@ -7017,11 +7122,17 @@ cmd_add(int argc, char *argv[]) goto done; } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 1, got_worktree_get_root_path(worktree)); if (error) @@ -7111,6 +7222,7 @@ cmd_remove(int argc, char *argv[]) struct got_pathlist_entry *pe; int ch, delete_local_mods = 0, can_recurse = 0, keep_on_disk = 0, i; int ignore_missing_paths = 0; + int *pack_fds = NULL; TAILQ_INIT(&paths); @@ -7171,11 +7283,17 @@ cmd_remove(int argc, char *argv[]) goto done; } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error) goto done; + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 1, got_worktree_get_root_path(worktree)); if (error) @@ -7327,6 +7445,7 @@ cmd_patch(int argc, char *argv[]) char *cwd = NULL; int ch, nop = 0, strip = -1, reverse = 0; int patchfd; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "np:R")) != -1) { switch (ch) { @@ -7373,11 +7492,17 @@ cmd_patch(int argc, char *argv[]) if (error != NULL) goto done; + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + const char *repo_path = got_worktree_get_repo_path(worktree); - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_open(&repo, repo_path, NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 0, got_worktree_get_root_path(worktree)); if (error != NULL) @@ -7552,6 +7677,7 @@ cmd_revert(int argc, char *argv[]) FILE *patch_script_file = NULL; const char *patch_script_path = NULL; struct choose_patch_arg cpa; + int *pack_fds = NULL; TAILQ_INIT(&paths); @@ -7597,11 +7723,17 @@ cmd_revert(int argc, char *argv[]) goto done; } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + if (patch_script_path) { patch_script_file = fopen(patch_script_path, "re"); if (patch_script_file == NULL) { @@ -7836,6 +7968,7 @@ cmd_commit(int argc, char *argv[]) int ch, rebase_in_progress, histedit_in_progress, preserve_logmsg = 0; int allow_bad_symlinks = 0, non_interactive = 0, merge_in_progress = 0; struct got_pathlist_head paths; + int *pack_fds = NULL; TAILQ_INIT(&paths); cl_arg.logmsg_path = NULL; @@ -7903,10 +8036,14 @@ cmd_commit(int argc, char *argv[]) error = get_gitconfig_path(&gitconfig_path); if (error) goto done; + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - gitconfig_path); + gitconfig_path, pack_fds); if (error != NULL) goto done; + free(pack_fds); error = got_worktree_merge_in_progress(&merge_in_progress, worktree, repo); if (error) @@ -8215,6 +8352,7 @@ cmd_send(int argc, char *argv[]) int verbosity = 0, overwrite_refs = 0; int send_all_branches = 0, send_all_tags = 0; struct got_reference *ref = NULL; + int *pack_fds = NULL; TAILQ_INIT(&branches); TAILQ_INIT(&tags); @@ -8316,10 +8454,16 @@ cmd_send(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + + error = got_repo_open(&repo, repo_path, NULL, pack_fds); if (error) goto done; + free(pack_fds); + if (worktree) { worktree_conf = got_worktree_get_gotconfig(worktree); if (worktree_conf) { @@ -8561,6 +8705,7 @@ cmd_cherrypick(int argc, char *argv[]) struct got_object_qid *pid; int ch; struct got_update_progress_arg upa; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "")) != -1) { switch (ch) { @@ -8594,11 +8739,17 @@ cmd_cherrypick(int argc, char *argv[]) goto done; } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 0, got_worktree_get_root_path(worktree)); if (error) @@ -8659,6 +8810,7 @@ cmd_backout(int argc, char *argv[]) struct got_object_qid *pid; int ch; struct got_update_progress_arg upa; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "")) != -1) { switch (ch) { @@ -8691,11 +8843,17 @@ cmd_backout(int argc, char *argv[]) goto done; } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 0, got_worktree_get_root_path(worktree)); if (error) @@ -9322,6 +9480,7 @@ cmd_rebase(int argc, char *argv[]) const struct got_object_id_queue *parent_ids; struct got_object_qid *qid, *pid; struct got_update_progress_arg upa; + int *pack_fds = NULL; STAILQ_INIT(&commits); TAILQ_INIT(&merged_paths); @@ -9401,11 +9560,18 @@ cmd_rebase(int argc, char *argv[]) } } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, - worktree ? got_worktree_get_repo_path(worktree) : cwd, NULL); + worktree ? got_worktree_get_repo_path(worktree) : cwd, NULL, + pack_fds); if (error != NULL) goto done; + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 0, worktree ? got_worktree_get_root_path(worktree) : NULL); if (error) @@ -10541,6 +10707,7 @@ cmd_histedit(int argc, char *argv[]) struct got_object_qid *pid; struct got_histedit_list histedit_cmds; struct got_histedit_list_entry *hle; + int *pack_fds = NULL; STAILQ_INIT(&commits); TAILQ_INIT(&histedit_cmds); @@ -10675,12 +10842,17 @@ cmd_histedit(int argc, char *argv[]) } } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + if (list_backups || delete_backups) { error = got_repo_open(&repo, worktree ? got_worktree_get_repo_path(worktree) : cwd, - NULL); + NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); error = apply_unveil(got_repo_get_path(repo), 0, worktree ? got_worktree_get_root_path(worktree) : NULL); if (error) @@ -10691,11 +10863,17 @@ cmd_histedit(int argc, char *argv[]) goto done; /* nothing else to do */ } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + error = got_worktree_rebase_in_progress(&rebase_in_progress, worktree); if (error) goto done; @@ -11085,6 +11263,7 @@ cmd_integrate(int argc, char *argv[]) struct got_object_id *commit_id = NULL, *base_commit_id = NULL; int ch; struct got_update_progress_arg upa; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "")) != -1) { switch (ch) { @@ -11123,11 +11302,17 @@ cmd_integrate(int argc, char *argv[]) if (error) goto done; + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 0, got_worktree_get_root_path(worktree)); if (error) @@ -11236,6 +11421,7 @@ cmd_merge(int argc, char *argv[]) struct got_update_progress_arg upa; struct got_object_id *merge_commit_id = NULL; char *branch_name = NULL; + int *pack_fds = NULL; memset(&upa, 0, sizeof(upa)); @@ -11287,11 +11473,18 @@ cmd_merge(int argc, char *argv[]) goto done; } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, - worktree ? got_worktree_get_repo_path(worktree) : cwd, NULL); + worktree ? got_worktree_get_repo_path(worktree) : cwd, NULL, + pack_fds); if (error != NULL) goto done; + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 0, worktree ? got_worktree_get_root_path(worktree) : NULL); if (error) @@ -11526,6 +11719,7 @@ cmd_stage(int argc, char *argv[]) FILE *patch_script_file = NULL; const char *patch_script_path = NULL; struct choose_patch_arg cpa; + int *pack_fds = NULL; TAILQ_INIT(&paths); @@ -11575,11 +11769,17 @@ cmd_stage(int argc, char *argv[]) goto done; } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + if (patch_script_path) { patch_script_file = fopen(patch_script_path, "re"); if (patch_script_file == NULL) { @@ -11654,6 +11854,7 @@ cmd_unstage(int argc, char *argv[]) FILE *patch_script_file = NULL; const char *patch_script_path = NULL; struct choose_patch_arg cpa; + int *pack_fds = NULL; TAILQ_INIT(&paths); @@ -11695,11 +11896,17 @@ cmd_unstage(int argc, char *argv[]) goto done; } + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, got_worktree_get_repo_path(worktree), - NULL); + NULL, pack_fds); if (error != NULL) goto done; + free(pack_fds); + if (patch_script_path) { patch_script_file = fopen(patch_script_path, "re"); if (patch_script_file == NULL) { @@ -11938,6 +12145,7 @@ cmd_cat(int argc, char *argv[]) struct got_commit_object *commit = NULL; int ch, obj_type, i, force_path = 0; struct got_reflist_head refs; + int *pack_fds = NULL; TAILQ_INIT(&refs); @@ -12001,11 +12209,17 @@ cmd_cat(int argc, char *argv[]) return got_error_from_errno("strdup"); } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + + error = got_repo_open(&repo, repo_path, NULL, pack_fds); free(repo_path); if (error != NULL) goto done; + free(pack_fds); + error = apply_unveil(got_repo_get_path(repo), 1, NULL); if (error) goto done; blob - 59c11d1ab01684d8a2f16d934fb1d6870d7ac871 file + gotadmin/gotadmin.c --- gotadmin/gotadmin.c +++ gotadmin/gotadmin.c @@ -43,6 +43,7 @@ #include "got_privsep.h" #include "got_opentemp.h" #include "got_worktree.h" +#include "got_pack_cache_sizes.h" #ifndef nitems #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0])) @@ -275,6 +276,7 @@ cmd_info(int argc, char *argv[]) int ch, npackfiles, npackedobj, nobj; off_t packsize, loose_size; char scaled[FMT_SCALED_STRSIZE]; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "r:")) != -1) { switch (ch) { @@ -304,9 +306,13 @@ cmd_info(int argc, char *argv[]) if (error) goto done; } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); if (error) goto done; + free(pack_fds); #ifndef PROFILE /* Remove "cpath" promise. */ if (pledge("stdio rpath wpath flock proc exec sendfd unveil", @@ -629,6 +635,7 @@ cmd_pack(int argc, char *argv[]) struct got_reflist_head exclude_refs; struct got_reflist_head include_refs; struct got_reflist_entry *re, *new; + int *pack_fds = NULL; TAILQ_INIT(&exclude_args); TAILQ_INIT(&exclude_refs); @@ -675,9 +682,13 @@ cmd_pack(int argc, char *argv[]) if (error) goto done; } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); if (error) goto done; + free(pack_fds); error = apply_unveil(got_repo_get_path_git_dir(repo), 0); if (error) @@ -775,6 +786,7 @@ cmd_indexpack(int argc, char *argv[]) char *id_str = NULL; struct got_pack_progress_arg ppa; FILE *packfile = NULL; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "")) != -1) { switch (ch) { @@ -800,9 +812,13 @@ cmd_indexpack(int argc, char *argv[]) err(1, "pledge"); #endif - error = got_repo_open(&repo, packfile_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, packfile_path, NULL, pack_fds); if (error) goto done; + free(pack_fds); error = apply_unveil(got_repo_get_path_git_dir(repo), 0); if (error) @@ -943,6 +959,7 @@ cmd_listpack(int argc, char *argv[]) struct gotadmin_list_pack_cb_args lpa; FILE *packfile = NULL; int show_stats = 0, human_readable = 0; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "hs")) != -1) { switch (ch) { @@ -972,9 +989,13 @@ cmd_listpack(int argc, char *argv[]) NULL) == -1) err(1, "pledge"); #endif - error = got_repo_open(&repo, packfile_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, packfile_path, NULL, pack_fds); if (error) goto done; + free(pack_fds); #ifndef PROFILE /* Remove "cpath" promise. */ if (pledge("stdio rpath wpath flock proc exec sendfd unveil", @@ -1120,6 +1141,7 @@ cmd_cleanup(int argc, char *argv[]) char scaled_diff[FMT_SCALED_STRSIZE]; char **extensions; int nextensions, i; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "apr:nq")) != -1) { switch (ch) { @@ -1161,9 +1183,13 @@ cmd_cleanup(int argc, char *argv[]) if (error) goto done; } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (error != NULL) + goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); if (error) goto done; + free(pack_fds); error = apply_unveil(got_repo_get_path_git_dir(repo), 0); if (error) blob - /dev/null file + include/got_pack_cache_sizes.h --- /dev/null +++ include/got_pack_cache_sizes.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2022 Tracey Emery <tracey@openbsd.org> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#define GOT_PACK_CACHE_SIZE 32 +#define GOT_PACK_CACHE_SIZE_DOUBLE GOT_PACK_CACHE_SIZE * 2 blob - b6e44e8b40b8476a471c53fd10cffd4a7ff3b32d file + include/got_repository.h --- include/got_repository.h +++ include/got_repository.h @@ -20,7 +20,7 @@ struct got_tag_object; /* Open and close repositories. */ const struct got_error *got_repo_open(struct got_repository**, const char *, - const char *); + const char *, int []); const struct got_error *got_repo_close(struct got_repository*); /* Obtain the on-disk path to the repository. */ @@ -177,3 +177,6 @@ const struct got_error *got_repo_get_loose_object_info /* Obtain the number and size of packed objects in the repository. */ const struct got_error *got_repo_get_packfile_info(int *npackfiles, int *nobjects, off_t *total_packsize, struct got_repository *); + +/* Create an array of file descriptors to hand over to got_repo_open for pack */ +const struct got_error *got_repo_pack_fds_open(int **, int); blob - c75567b209bd9f2ce134e3ca05a66884b8dcbc04 file + lib/got_lib_repository.h --- lib/got_lib_repository.h +++ lib/got_lib_repository.h @@ -14,6 +14,8 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include "got_pack_cache_sizes.h" + #define GOT_GIT_DIR ".git" /* Mandatory files and directories inside the git directory. */ @@ -28,8 +30,6 @@ #define GOT_OBJECTS_PACK_DIR "objects/pack" #define GOT_PACKED_REFS_FILE "packed-refs" -#define GOT_PACK_CACHE_SIZE 32 - struct got_packidx_bloom_filter { RB_ENTRY(got_packidx_bloom_filter) entry; char path[PATH_MAX]; /* on-disk path */ blob - 5be66db732b6a51d032ae835a233ef3cbcd3733b file + lib/repository.c --- lib/repository.c +++ lib/repository.c @@ -247,6 +247,25 @@ done: } const struct got_error * +got_repo_pack_fds_open(int **pack_fds, int nfds) +{ + const struct got_error *err = NULL; + int i, pack_fds_tmp[nfds]; + + *pack_fds = calloc(nfds, sizeof(**pack_fds)); + if (*pack_fds == NULL) + return got_error_from_errno("calloc"); + + for (i = 0; i < nfds; i++) { + pack_fds_tmp[i] = got_opentempfd(); + if (pack_fds_tmp[i] == -1) + return got_error_from_errno("got_opentempfd"); + } + memcpy(*pack_fds, pack_fds_tmp, sizeof(**pack_fds)); + return err; +} + +const struct got_error * got_repo_cache_object(struct got_repository *repo, struct got_object_id *id, struct got_object *obj) { @@ -651,12 +670,12 @@ static const char *const repo_extensions[] = { const struct got_error * got_repo_open(struct got_repository **repop, const char *path, - const char *global_gitconfig_path) + const char *global_gitconfig_path, int pack_fds[]) { struct got_repository *repo = NULL; const struct got_error *err = NULL; char *repo_path = NULL; - size_t i; + size_t i, j = 0; struct rlimit rl; *repop = NULL; @@ -703,12 +722,8 @@ got_repo_open(struct got_repository **repop, const cha repo->pack_cache_size = rl.rlim_cur / 8; for (i = 0; i < nitems(repo->packs); i++) { if (i < repo->pack_cache_size) { - repo->packs[i].basefd = got_opentempfd(); - if (repo->packs[i].basefd == -1) - return got_error_from_errno("got_opentempfd"); - repo->packs[i].accumfd = got_opentempfd(); - if (repo->packs[i].accumfd == -1) - return got_error_from_errno("got_opentempfd"); + repo->packs[i].basefd = pack_fds[j++]; + repo->packs[i].accumfd = pack_fds[j++]; } else { repo->packs[i].basefd = -1; repo->packs[i].accumfd = -1; @@ -796,21 +811,6 @@ got_repo_close(struct got_repository *repo) free(bf); } - for (i = 0; i < repo->pack_cache_size; i++) { - if (repo->packs[i].path_packfile) - got_pack_close(&repo->packs[i]); - if (repo->packs[i].basefd != -1) { - if (close(repo->packs[i].basefd) == -1 && err == NULL) - err = got_error_from_errno("close"); - repo->packs[i].basefd = -1; - } - if (repo->packs[i].accumfd != -1) { - if (close(repo->packs[i].accumfd) == -1 && err == NULL) - err = got_error_from_errno("close"); - repo->packs[i].accumfd = -1; - } - } - free(repo->path); free(repo->path_git_dir); blob - 673cbeb89c705904097774881d38ef9d51751939 file + lib/worktree_open.c --- lib/worktree_open.c +++ lib/worktree_open.c @@ -39,6 +39,7 @@ #include "got_lib_worktree.h" #include "got_lib_gotconfig.h" +#include "got_pack_cache_sizes.h" static const struct got_error * read_meta_file(char **content, const char *path_got, const char *name) @@ -117,6 +118,7 @@ open_worktree(struct got_worktree **worktree, const ch const char *errstr; struct got_repository *repo = NULL; uint32_t uuid_status; + int *pack_fds = NULL; *worktree = NULL; @@ -190,10 +192,16 @@ open_worktree(struct got_worktree **worktree, const ch goto done; } - err = got_repo_open(&repo, (*worktree)->repo_path, NULL); + err = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (err != NULL) + goto done; + + err = got_repo_open(&repo, (*worktree)->repo_path, NULL, pack_fds); if (err) goto done; + free(pack_fds); + err = got_object_resolve_id_str(&(*worktree)->base_commit_id, repo, base_commit_id_str); if (err) blob - b4efe82ad2000b743da8b7312623bf0fe3708969 file + tog/tog.c --- tog/tog.c +++ tog/tog.c @@ -55,6 +55,7 @@ #include "got_privsep.h" #include "got_path.h" #include "got_worktree.h" +#include "got_pack_cache_sizes.h" #ifndef MIN #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b)) @@ -2317,6 +2318,7 @@ open_log_view(struct tog_view *view, struct got_object struct got_repository *thread_repo = NULL; struct got_commit_graph *thread_graph = NULL; int errcode; + int *pack_fds = NULL; if (in_repo_path != s->in_repo_path) { free(s->in_repo_path); @@ -2370,9 +2372,14 @@ open_log_view(struct tog_view *view, struct got_object view->search_start = search_start_log_view; view->search_next = search_next_log_view; - err = got_repo_open(&thread_repo, got_repo_get_path(repo), NULL); + err = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (err) goto done; + err = got_repo_open(&thread_repo, got_repo_get_path(repo), NULL, + pack_fds); + if (err) + goto done; + free(pack_fds); err = got_commit_graph_open(&thread_graph, s->in_repo_path, !s->log_branches); if (err) @@ -2442,6 +2449,7 @@ input_log_view(struct tog_view **new_view, struct tog_ struct tog_view *ref_view = NULL; struct commit_queue_entry *entry; int begin_x = 0, n; + int *pack_fds = NULL; if (s->thread_args.load_all) { if (ch == KEY_BACKSPACE) @@ -2634,10 +2642,14 @@ input_log_view(struct tog_view **new_view, struct tog_ } else /* 'B' */ s->log_branches = !s->log_branches; + err = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); + if (err) + return err; err = got_repo_open(&s->thread_args.repo, - got_repo_get_path(s->repo), NULL); + got_repo_get_path(s->repo), NULL, pack_fds); if (err) return err; + free(pack_fds); tog_free_refs(); err = tog_load_refs(s->repo, 0); if (err) @@ -2791,6 +2803,7 @@ cmd_log(int argc, char *argv[]) const char *head_ref_name = NULL; int ch, log_branches = 0; struct tog_view *view; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "bc:r:")) != -1) { switch (ch) { @@ -2836,9 +2849,13 @@ cmd_log(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + free(pack_fds); error = get_in_repo_path_from_argv0(&in_repo_path, argc, argv, repo, worktree); @@ -3911,6 +3928,7 @@ cmd_diff(int argc, char *argv[]) int ch, force_text_diff = 0; const char *errstr; struct tog_view *view; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "aC:r:w")) != -1) { switch (ch) { @@ -3969,9 +3987,13 @@ cmd_diff(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error) + goto done; + free(pack_fds); init_curses(); @@ -4162,7 +4184,7 @@ draw_blame(struct tog_view *view) width = 9; wline = wcsdup(L""); if (wline == NULL) { - err = got_error_from_errno("wcsdup"); + err = got_error_from_errno("wcsup"); free(line); return err; } @@ -4372,6 +4394,7 @@ run_blame(struct tog_view *view) struct got_repository *thread_repo = NULL; struct got_object_id *obj_id = NULL; int obj_type; + int *pack_fds = NULL; err = got_object_open_as_commit(&commit, s->repo, &s->blamed_commit->id); @@ -4418,9 +4441,14 @@ run_blame(struct tog_view *view) goto done; } - err = got_repo_open(&thread_repo, got_repo_get_path(s->repo), NULL); + err = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (err) goto done; + err = got_repo_open(&thread_repo, got_repo_get_path(s->repo), NULL, + pack_fds); + if (err) + goto done; + free(pack_fds); blame->cb_args.view = view; blame->cb_args.lines = blame->lines; @@ -4861,6 +4889,7 @@ cmd_blame(int argc, char *argv[]) char *commit_id_str = NULL; int ch; struct tog_view *view; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "c:r:")) != -1) { switch (ch) { @@ -4903,9 +4932,13 @@ cmd_blame(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + free(pack_fds); error = get_in_repo_path_from_argv0(&in_repo_path, argc, argv, repo, worktree); @@ -5723,6 +5756,7 @@ cmd_tree(int argc, char *argv[]) const char *head_ref_name = NULL; int ch; struct tog_view *view; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "c:r:")) != -1) { switch (ch) { @@ -5765,9 +5799,13 @@ cmd_tree(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + free(pack_fds); error = get_in_repo_path_from_argv0(&in_repo_path, argc, argv, repo, worktree); @@ -6476,6 +6514,7 @@ cmd_ref(int argc, char *argv[]) char *cwd = NULL, *repo_path = NULL; int ch; struct tog_view *view; + int *pack_fds = NULL; while ((ch = getopt(argc, argv, "r:")) != -1) { switch (ch) { @@ -6515,9 +6554,13 @@ cmd_ref(int argc, char *argv[]) } } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + free(pack_fds); init_curses(); @@ -6624,6 +6667,7 @@ tog_log_with_path(int argc, char *argv[]) struct got_commit_object *commit = NULL; char *cwd = NULL, *repo_path = NULL, *in_repo_path = NULL; char *commit_id_str = NULL, **cmd_argv = NULL; + int *pack_fds = NULL; cwd = getcwd(NULL, 0); if (cwd == NULL) @@ -6642,9 +6686,13 @@ tog_log_with_path(int argc, char *argv[]) goto done; } - error = got_repo_open(&repo, repo_path, NULL); + error = got_repo_pack_fds_open(&pack_fds, GOT_PACK_CACHE_SIZE_DOUBLE); if (error != NULL) goto done; + error = got_repo_open(&repo, repo_path, NULL, pack_fds); + if (error != NULL) + goto done; + free(pack_fds); error = get_in_repo_path_from_argv0(&in_repo_path, argc, argv, repo, worktree); @@ -6806,4 +6854,5 @@ main(int argc, char *argv[]) if (error && error->code != GOT_ERR_CANCELLED) fprintf(stderr, "%s: %s\n", getprogname(), error->msg); return 0; + }
Move got_opentempfd out of got_repo_open