Download raw body.
open tempfiles in got_repo_open()
Programs like gotwebd(8) and perhaps a future gotd(8) server need to manage resources such as open files upfront, rather than having code in lib/ open files as needed. The patch below is a first step towards fixing one such issue for gotwebd. The issue here is that gotwebd runs in a chroot environment that may lack a /tmp directory, which breaks got_opentemp() at run-time. ok? ----------------------------------------------- commit bd9b38c590a5a0c4e6b47264b667fc5e3e6c4b93 (pack-opentemp) from: Stefan Sperling <stsp@stsp.name> date: Sat Mar 12 12:43:24 2022 UTC open temporary files needed for delta application in got_repo_open() This prepares for callers of got_repo_open() that cannot afford to open files in /tmp, such as gotwebd. In a follow-up change, we could ask such callers to pass in the required amount of open temporary files. One consequence is that got_repo_open() now requires the "cpath" pledge promise. Add the "cpath" promise to affected callers and remove it once the repository has been opened. diff f95923a581dde77a840de5e9f56060e84118b413 9eff67c819a8cc0b0c4db3579999877cde09b335 blob - 17d637cabb59cb76c220d33c371c8c2a25da94cf blob + fbda8cac06e23e0d902b849b50b650436a9dcc11 --- got/got.c +++ got/got.c @@ -5810,15 +5810,9 @@ cmd_ref(int argc, char *argv[]) got_path_strip_trailing_slashes(refname); #ifndef PROFILE - if (do_list) { - if (pledge("stdio rpath wpath flock proc exec sendfd unveil", - NULL) == -1) - err(1, "pledge"); - } else { - if (pledge("stdio rpath wpath cpath fattr flock proc exec " - "sendfd unveil", NULL) == -1) - err(1, "pledge"); - } + if (pledge("stdio rpath wpath cpath fattr flock proc exec " + "sendfd unveil", NULL) == -1) + err(1, "pledge"); #endif cwd = getcwd(NULL, 0); if (cwd == NULL) { @@ -5852,6 +5846,15 @@ cmd_ref(int argc, char *argv[]) if (error != NULL) goto done; +#ifndef PROFILE + if (do_list) { + /* Remove "cpath" promise. */ + if (pledge("stdio rpath wpath flock proc exec sendfd unveil", + NULL) == -1) + err(1, "pledge"); + } +#endif + error = apply_unveil(got_repo_get_path(repo), do_list, worktree ? got_worktree_get_root_path(worktree) : NULL); if (error) @@ -6178,48 +6181,51 @@ cmd_branch(int argc, char *argv[]) usage_branch(); #ifndef PROFILE + if (pledge("stdio rpath wpath cpath fattr flock proc exec " + "sendfd unveil", NULL) == -1) + err(1, "pledge"); +#endif + cwd = getcwd(NULL, 0); + if (cwd == NULL) { + error = got_error_from_errno("getcwd"); + goto done; + } + + if (repo_path == NULL) { + error = got_worktree_open(&worktree, cwd); + if (error && error->code != GOT_ERR_NOT_WORKTREE) + goto done; + else + error = NULL; + if (worktree) { + repo_path = + strdup(got_worktree_get_repo_path(worktree)); + if (repo_path == NULL) + error = got_error_from_errno("strdup"); + if (error) + goto done; + } else { + repo_path = strdup(cwd); + if (repo_path == NULL) { + error = got_error_from_errno("strdup"); + goto done; + } + } + } + + error = got_repo_open(&repo, repo_path, NULL); + if (error != NULL) + goto done; + +#ifndef PROFILE if (do_list || do_show) { + /* Remove "cpath" promise. */ if (pledge("stdio rpath wpath flock proc exec sendfd unveil", NULL) == -1) err(1, "pledge"); - } else { - if (pledge("stdio rpath wpath cpath fattr flock proc exec " - "sendfd unveil", NULL) == -1) - err(1, "pledge"); } #endif - cwd = getcwd(NULL, 0); - if (cwd == NULL) { - error = got_error_from_errno("getcwd"); - goto done; - } - if (repo_path == NULL) { - error = got_worktree_open(&worktree, cwd); - if (error && error->code != GOT_ERR_NOT_WORKTREE) - goto done; - else - error = NULL; - if (worktree) { - repo_path = - strdup(got_worktree_get_repo_path(worktree)); - if (repo_path == NULL) - error = got_error_from_errno("strdup"); - if (error) - goto done; - } else { - repo_path = strdup(cwd); - if (repo_path == NULL) { - error = got_error_from_errno("strdup"); - goto done; - } - } - } - - error = got_repo_open(&repo, repo_path, NULL); - if (error != NULL) - goto done; - error = apply_unveil(got_repo_get_path(repo), do_list, worktree ? got_worktree_get_root_path(worktree) : NULL); if (error) @@ -6726,15 +6732,9 @@ cmd_tag(int argc, char *argv[]) tag_name = argv[0]; #ifndef PROFILE - if (do_list) { - if (pledge("stdio rpath wpath flock proc exec sendfd unveil", - NULL) == -1) - err(1, "pledge"); - } else { - if (pledge("stdio rpath wpath cpath fattr flock proc exec " - "sendfd unveil", NULL) == -1) - err(1, "pledge"); - } + if (pledge("stdio rpath wpath cpath fattr flock proc exec " + "sendfd unveil", NULL) == -1) + err(1, "pledge"); #endif cwd = getcwd(NULL, 0); if (cwd == NULL) { @@ -6768,6 +6768,12 @@ cmd_tag(int argc, char *argv[]) error = got_repo_open(&repo, repo_path, NULL); if (error != NULL) goto done; +#ifndef PROFILE + /* Remove "cpath" promise. */ + if (pledge("stdio rpath wpath flock proc exec sendfd unveil", + NULL) == -1) + err(1, "pledge"); +#endif error = apply_unveil(got_repo_get_path(repo), 1, NULL); if (error) goto done; @@ -11960,7 +11966,7 @@ cmd_info(int argc, char *argv[]) argv += optind; #ifndef PROFILE - if (pledge("stdio rpath wpath flock proc exec sendfd unveil", + if (pledge("stdio rpath wpath cpath flock proc exec sendfd unveil", NULL) == -1) err(1, "pledge"); #endif @@ -11977,6 +11983,12 @@ cmd_info(int argc, char *argv[]) goto done; } +#ifndef PROFILE + /* Remove "cpath" promise. */ + if (pledge("stdio rpath wpath flock proc exec sendfd unveil", + NULL) == -1) + err(1, "pledge"); +#endif error = apply_unveil(NULL, 0, got_worktree_get_root_path(worktree)); if (error) goto done; blob - bd7343dcca754f398aaba30df74ecd27b90d2ee7 blob + 0f9c2137d554258de7f66a0bcdb8c4a945cea69a --- gotadmin/gotadmin.c +++ gotadmin/gotadmin.c @@ -295,7 +295,7 @@ cmd_info(int argc, char *argv[]) argv += optind; #ifndef PROFILE - if (pledge("stdio rpath wpath flock proc exec sendfd unveil", + if (pledge("stdio rpath wpath cpath flock proc exec sendfd unveil", NULL) == -1) err(1, "pledge"); #endif @@ -307,7 +307,12 @@ cmd_info(int argc, char *argv[]) error = got_repo_open(&repo, repo_path, NULL); if (error) goto done; - +#ifndef PROFILE + /* Remove "cpath" promise. */ + if (pledge("stdio rpath wpath flock proc exec sendfd unveil", + NULL) == -1) + err(1, "pledge"); +#endif error = apply_unveil(got_repo_get_path_git_dir(repo), 1); if (error) goto done; @@ -899,14 +904,19 @@ cmd_listpack(int argc, char *argv[]) return got_error_from_errno2("realpath", argv[0]); #ifndef PROFILE - if (pledge("stdio rpath wpath flock proc exec sendfd unveil", + if (pledge("stdio rpath wpath cpath flock proc exec sendfd unveil", NULL) == -1) err(1, "pledge"); #endif error = got_repo_open(&repo, packfile_path, NULL); if (error) goto done; - +#ifndef PROFILE + /* Remove "cpath" promise. */ + if (pledge("stdio rpath wpath flock proc exec sendfd unveil", + NULL) == -1) + err(1, "pledge"); +#endif error = apply_unveil(got_repo_get_path_git_dir(repo), 1); if (error) goto done; blob - e8fb373e287ee80486d50ed07964d9d39924308d blob + 61f0c6f6be1ff738e75c196b532adeec0d286f6d --- lib/got_lib_pack.h +++ lib/got_lib_pack.h @@ -21,6 +21,8 @@ struct got_pack { uint8_t *map; size_t filesize; struct got_privsep_child *privsep_child; + int basefd; + int accumfd; int child_has_tempfiles; int child_has_delta_outfd; struct got_delta_cache *delta_cache; blob - 8218b2c4c146e832c63f84d4a9f6a43f8b3283f8 blob + aed88977b89e2cbc403071fd2d8fd81478ad6570 --- lib/object.c +++ lib/object.c @@ -172,7 +172,7 @@ static const struct got_error * pack_child_send_tempfiles(struct imsgbuf *ibuf, struct got_pack *pack) { const struct got_error *err; - int basefd, accumfd; + int basefd = -1, accumfd = -1; /* * For performance reasons, the child will keep reusing the @@ -183,23 +183,29 @@ pack_child_send_tempfiles(struct imsgbuf *ibuf, struct if (pack->child_has_tempfiles) return NULL; - basefd = got_opentempfd(); + basefd = dup(pack->basefd); if (basefd == -1) - return got_error_from_errno("got_opentempfd"); + return got_error_from_errno("dup"); + accumfd = dup(pack->accumfd); + if (accumfd == -1) { + err = got_error_from_errno("dup"); + goto done; + } + err = got_privsep_send_tmpfd(ibuf, basefd); if (err) - return err; + goto done; - accumfd = got_opentempfd(); - if (accumfd == -1) - return got_error_from_errno("got_opentempfd"); - err = got_privsep_send_tmpfd(ibuf, accumfd); - if (err) - return err; - - pack->child_has_tempfiles = 1; +done: + if (err) { + if (basefd != -1) + close(basefd); + if (accumfd != -1) + close(accumfd); + } else + pack->child_has_tempfiles = 1; return NULL; } blob - 4aa9467c86b6e0a5f571584951e24abda6d8f5f5 blob + efe03f932898f14a489edd1540f83f72b472413f --- lib/pack.c +++ lib/pack.c @@ -751,6 +751,11 @@ got_pack_close(struct got_pack *pack) pack->delta_cache = NULL; } + /* + * Leave accumfd and basefd alone. They are managed by the + * repository layer and can be reused. + */ + return err; } blob - d4175545865a20ba9a3cff43736298aaae5e8c37 blob + 487889ce6084b085c5bd704c6a005d435c1992de --- lib/repository.c +++ lib/repository.c @@ -50,6 +50,7 @@ #include "got_path.h" #include "got_cancel.h" #include "got_object.h" +#include "got_opentemp.h" #include "got_lib_delta.h" #include "got_lib_inflate.h" @@ -701,6 +702,19 @@ got_repo_open(struct got_repository **repop, const cha repo->pack_cache_size = GOT_PACK_CACHE_SIZE; if (repo->pack_cache_size > rl.rlim_cur / 8) repo->pack_cache_size = rl.rlim_cur / 8; + for (i = 0; i < nitems(repo->packs); i++) { + if (i < repo->pack_cache_size) { + repo->packs[i].basefd = got_opentempfd(); + if (repo->packs[i].basefd == -1) + return got_error_from_errno("got_opentempfd"); + repo->packs[i].accumfd = got_opentempfd(); + if (repo->packs[i].accumfd == -1) + return got_error_from_errno("got_opentempfd"); + } else { + repo->packs[i].basefd = -1; + repo->packs[i].accumfd = -1; + } + } repo_path = realpath(path, NULL); if (repo_path == NULL) { @@ -784,6 +798,16 @@ got_repo_close(struct got_repository *repo) if (repo->packs[i].path_packfile == NULL) break; got_pack_close(&repo->packs[i]); + if (repo->packs[i].basefd != -1) { + if (close(repo->packs[i].basefd) == -1 && err == NULL) + err = got_error_from_errno("close"); + repo->packs[i].basefd = -1; + } + if (repo->packs[i].accumfd != -1) { + if (close(repo->packs[i].accumfd) == -1 && err == NULL) + err = got_error_from_errno("close"); + repo->packs[i].accumfd = -1; + } } free(repo->path); @@ -1360,6 +1384,10 @@ got_repo_cache_pack(struct got_pack **packp, struct go err = got_pack_close(&repo->packs[i - 1]); if (err) return err; + if (ftruncate(repo->packs[i - 1].basefd, 0L) == -1) + return got_error_from_errno("ftruncate"); + if (ftruncate(repo->packs[i - 1].accumfd, 0L) == -1) + return got_error_from_errno("ftruncate"); memmove(&repo->packs[1], &repo->packs[0], sizeof(repo->packs) - sizeof(repo->packs[0])); i = 0;
open tempfiles in got_repo_open()