- From:
- Stefan Sperling <stsp@stsp.name>
- Subject:
- run gotadmin without libexec helpers
- To:
- gameoftrees@openbsd.org
- Date:
- Mon, 4 Sep 2023 02:02:39 +0200
Download raw body.
Thread
run gotadmin without libexec helpers
Download raw body.
run gotadmin without libexec helpers
This patch makes gotadmin run without libexec helpers.
I am not sure if I really want to go through with this patch but
wanted to share it in case it is interesting.
The idea came during discussion with bluhm@ and is an easy way to avoid
the excessive forking issue during gotadmin pack with many pack files,
without having to change got-read-pack and related caching which would
require a lot more effort to implement.
The upside is that packing throughput rougly doubles in my testing.
And we can remove all "proc exec sendfd" pledges.
The downside is that gotadmin now runs with pledges "rpath wpath flock"
always active.
Opinions?
Anecdotal testing on freshly copied copies of got.git:
$ gotadmin info
repository: /git/got.git
pack files: 91
packed objects: 32619
packed total size: 16.9M
loose objects: 3269
loose total size: 12.3M
Withtout patch:
$ time /usr/local/bin/gotadmin pack -a
5588 commits colored; 31725 objects found; 14699 trees scanned
packing 109 references; 31725 objects; deltify: 100%; writing pack: 10.6M 100%
Wrote 8ee337396c79a990d63cc31017175d0706ea5000.pack
10.6M packed; indexing 100%; resolving deltas 100%
Indexed 8ee337396c79a990d63cc31017175d0706ea5000.pack
2m10.69s real 1m00.78s user 1m05.93s system
$
With patch:
$ time gotadmin pack -a
5588 commits colored; 31725 objects found; 14699 trees scanned
packing 109 references; 31725 objects; deltify: 100%; writing pack: 10.5M 100%
Wrote 801d80cd6fed9df5796725d7d89d2195443c8564.pack
10.5M packed; indexing 100%; resolving deltas 100%
Indexed 801d80cd6fed9df5796725d7d89d2195443c8564.pack
1m23.78s real 1m04.50s user 0m13.76s system
-----------------------------------------------
run gotadmin without libexec helpers
diff d71015b33dbfbef534be92e584006caddbf16aa6 6aafb0b19ad09cce0addac5b594419acb7168ce7
commit - d71015b33dbfbef534be92e584006caddbf16aa6
commit + 6aafb0b19ad09cce0addac5b594419acb7168ce7
blob - 4485f4876de9c6581701463287d7bb44c2dec812
blob + 9acdc69ed3951000e7f95a14946d9ebff610c83a
--- gotadmin/Makefile
+++ gotadmin/Makefile
@@ -7,11 +7,11 @@ SRCS= gotadmin.c \
deflate.c delta.c delta_cache.c deltify.c error.c gotconfig.c \
inflate.c lockfile.c object.c object_cache.c object_create.c \
object_idset.c object_parse.c opentemp.c pack.c pack_create.c \
- path.c privsep.c reference.c repository.c repository_admin.c \
+ path.c privsep_stub.c reference.c repository.c repository_admin.c \
worktree_open.c hash.c bloom.c murmurhash2.c ratelimit.c \
- sigs.c buf.c date.c object_open_privsep.c \
- read_gitconfig_privsep.c read_gotconfig_privsep.c \
- pack_create_privsep.c pollfd.c reference_parse.c object_qid.c \
+ sigs.c buf.c date.c object_open_io.c pack_index.c \
+ gitconfig.c read_gitconfig.c read_gotconfig.c \
+ pack_create_io.c pollfd.c reference_parse.c object_qid.c \
dump.c load.c
MAN = ${PROG}.1
blob - ac13af4d643c13b2ba5b2cdc4285db4bb2b2789c
blob + f42075bac68d14b1f37b71e1c77921a7c9f5d111
--- gotadmin/gotadmin.c
+++ gotadmin/gotadmin.c
@@ -214,8 +214,6 @@ usage(int hflag, int status)
static const struct got_error *
apply_unveil(const char *repo_path, int repo_read_only)
{
- const struct got_error *err;
-
#ifdef PROFILE
if (unveil("gmon.out", "rwc") != 0)
return got_error_from_errno2("unveil", "gmon.out");
@@ -226,10 +224,6 @@ apply_unveil(const char *repo_path, int repo_read_only
if (unveil(GOT_TMPDIR_STR, "rwc") != 0)
return got_error_from_errno2("unveil", GOT_TMPDIR_STR);
- err = got_privsep_unveil_exec_helpers();
- if (err != NULL)
- return err;
-
if (unveil(NULL, NULL) != 0)
return got_error_from_errno("unveil");
@@ -346,11 +340,10 @@ cmd_info(int argc, char *argv[])
int ch, npackfiles, npackedobj, nobj;
off_t packsize, loose_size;
char scaled[FMT_SCALED_STRSIZE];
- int *pack_fds = NULL;
+ int *pack_fds = NULL, *temp_fds = NULL;
#ifndef PROFILE
- if (pledge("stdio rpath wpath cpath flock proc exec sendfd unveil",
- NULL) == -1)
+ if (pledge("stdio rpath wpath cpath flock unveil", NULL) == -1)
err(1, "pledge");
#endif
@@ -380,13 +373,16 @@ cmd_info(int argc, char *argv[])
error = got_repo_pack_fds_open(&pack_fds);
if (error != NULL)
goto done;
+ error = got_repo_temp_fds_open(&temp_fds);
+ if (error)
+ goto done;
error = got_repo_open(&repo, repo_path, NULL, pack_fds);
if (error)
goto done;
+ got_repo_temp_fds_set(repo, temp_fds);
#ifndef PROFILE
/* Remove "cpath" promise. */
- if (pledge("stdio rpath wpath flock proc exec sendfd unveil",
- NULL) == -1)
+ if (pledge("stdio rpath wpath flock unveil", NULL) == -1)
err(1, "pledge");
#endif
error = apply_unveil(got_repo_get_path_git_dir(repo), 1);
@@ -447,6 +443,8 @@ cmd_info(int argc, char *argv[])
done:
if (repo)
got_repo_close(repo);
+ if (temp_fds)
+ got_repo_temp_fds_close(temp_fds);
if (pack_fds) {
const struct got_error *pack_err =
got_repo_pack_fds_close(pack_fds);
@@ -712,15 +710,14 @@ cmd_pack(int argc, char *argv[])
struct got_reflist_head exclude_refs;
struct got_reflist_head include_refs;
struct got_reflist_entry *re, *new;
- int *pack_fds = NULL;
+ int *pack_fds = NULL, *temp_fds = NULL;
TAILQ_INIT(&exclude_args);
TAILQ_INIT(&exclude_refs);
TAILQ_INIT(&include_refs);
#ifndef PROFILE
- if (pledge("stdio rpath wpath cpath fattr flock proc exec sendfd unveil",
- NULL) == -1)
+ if (pledge("stdio rpath wpath cpath fattr flock unveil", NULL) == -1)
err(1, "pledge");
#endif
@@ -766,9 +763,13 @@ cmd_pack(int argc, char *argv[])
error = got_repo_pack_fds_open(&pack_fds);
if (error != NULL)
goto done;
+ error = got_repo_temp_fds_open(&temp_fds);
+ if (error)
+ goto done;
error = got_repo_open(&repo, repo_path, NULL, pack_fds);
if (error)
goto done;
+ got_repo_temp_fds_set(repo, temp_fds);
error = apply_unveil(got_repo_get_path_git_dir(repo), 0);
if (error)
@@ -838,6 +839,8 @@ cmd_pack(int argc, char *argv[])
done:
if (repo)
got_repo_close(repo);
+ if (temp_fds)
+ got_repo_temp_fds_close(temp_fds);
if (pack_fds) {
const struct got_error *pack_err =
got_repo_pack_fds_close(pack_fds);
@@ -872,7 +875,7 @@ cmd_indexpack(int argc, char *argv[])
char *id_str = NULL;
struct got_pack_progress_arg ppa;
FILE *packfile = NULL;
- int *pack_fds = NULL;
+ int *pack_fds = NULL, *temp_fds = NULL;
while ((ch = getopt(argc, argv, "")) != -1) {
switch (ch) {
@@ -893,17 +896,20 @@ cmd_indexpack(int argc, char *argv[])
return got_error_from_errno2("realpath", argv[0]);
#ifndef PROFILE
- if (pledge("stdio rpath wpath cpath fattr flock proc exec sendfd unveil",
- NULL) == -1)
+ if (pledge("stdio rpath wpath cpath fattr flock unveil", NULL) == -1)
err(1, "pledge");
#endif
error = got_repo_pack_fds_open(&pack_fds);
if (error != NULL)
goto done;
+ error = got_repo_temp_fds_open(&temp_fds);
+ if (error)
+ goto done;
error = got_repo_open(&repo, packfile_path, NULL, pack_fds);
if (error)
goto done;
+ got_repo_temp_fds_set(repo, temp_fds);
error = apply_unveil(got_repo_get_path_git_dir(repo), 0);
if (error)
@@ -932,6 +938,8 @@ cmd_indexpack(int argc, char *argv[])
done:
if (repo)
got_repo_close(repo);
+ if (temp_fds)
+ got_repo_temp_fds_close(temp_fds);
if (pack_fds) {
const struct got_error *pack_err =
got_repo_pack_fds_close(pack_fds);
@@ -1051,11 +1059,10 @@ cmd_listpack(int argc, char *argv[])
struct gotadmin_list_pack_cb_args lpa;
FILE *packfile = NULL;
int show_stats = 0, human_readable = 0;
- int *pack_fds = NULL;
+ int *pack_fds = NULL, *temp_fds = NULL;
#ifndef PROFILE
- if (pledge("stdio rpath wpath cpath flock proc exec sendfd unveil",
- NULL) == -1)
+ if (pledge("stdio rpath wpath cpath flock unveil", NULL) == -1)
err(1, "pledge");
#endif
@@ -1085,13 +1092,17 @@ cmd_listpack(int argc, char *argv[])
error = got_repo_pack_fds_open(&pack_fds);
if (error != NULL)
goto done;
+ error = got_repo_temp_fds_open(&temp_fds);
+ if (error)
+ goto done;
error = got_repo_open(&repo, packfile_path, NULL, pack_fds);
if (error)
goto done;
+ got_repo_temp_fds_set(repo, temp_fds);
+
#ifndef PROFILE
/* Remove "cpath" promise. */
- if (pledge("stdio rpath wpath flock proc exec sendfd unveil",
- NULL) == -1)
+ if (pledge("stdio rpath wpath flock unveil", NULL) == -1)
err(1, "pledge");
#endif
error = apply_unveil(got_repo_get_path_git_dir(repo), 1);
@@ -1123,6 +1134,8 @@ cmd_listpack(int argc, char *argv[])
done:
if (repo)
got_repo_close(repo);
+ if (temp_fds)
+ got_repo_temp_fds_close(temp_fds);
if (pack_fds) {
const struct got_error *pack_err =
got_repo_pack_fds_close(pack_fds);
@@ -1261,11 +1274,10 @@ cmd_cleanup(int argc, char *argv[])
char pack_before_scaled[FMT_SCALED_STRSIZE];
char pack_after_scaled[FMT_SCALED_STRSIZE];
char total_size_scaled[FMT_SCALED_STRSIZE];
- int *pack_fds = NULL;
+ int *pack_fds = NULL, *temp_fds = NULL;
#ifndef PROFILE
- if (pledge("stdio rpath wpath cpath flock proc exec sendfd unveil",
- NULL) == -1)
+ if (pledge("stdio rpath wpath cpath flock unveil", NULL) == -1)
err(1, "pledge");
#endif
@@ -1307,9 +1319,13 @@ cmd_cleanup(int argc, char *argv[])
error = got_repo_pack_fds_open(&pack_fds);
if (error != NULL)
goto done;
+ error = got_repo_temp_fds_open(&temp_fds);
+ if (error)
+ goto done;
error = got_repo_open(&repo, repo_path, NULL, pack_fds);
if (error)
goto done;
+ got_repo_temp_fds_set(repo, temp_fds);
error = apply_unveil(got_repo_get_path_git_dir(repo), 0);
if (error)
@@ -1386,6 +1402,8 @@ cmd_cleanup(int argc, char *argv[])
done:
if (repo)
got_repo_close(repo);
+ if (temp_fds)
+ got_repo_temp_fds_close(temp_fds);
if (pack_fds) {
const struct got_error *pack_err =
got_repo_pack_fds_close(pack_fds);
@@ -1417,7 +1435,7 @@ cmd_dump(int argc, char *argv[])
struct got_reflist_entry *re, *new;
const char *refname;
char *repo_path = NULL;
- int *pack_fds = NULL;
+ int *pack_fds = NULL, *temp_fds = NULL;
int verbosity = 0;
int i, ch;
@@ -1426,8 +1444,7 @@ cmd_dump(int argc, char *argv[])
TAILQ_INIT(&include_refs);
#ifndef PROFILE
- if (pledge("stdio rpath wpath cpath flock proc exec sendfd unveil",
- NULL) == -1)
+ if (pledge("stdio rpath wpath cpath flock unveil", NULL) == -1)
err(1, "pledge");
#endif
@@ -1465,9 +1482,13 @@ cmd_dump(int argc, char *argv[])
error = got_repo_pack_fds_open(&pack_fds);
if (error != NULL)
goto done;
+ error = got_repo_temp_fds_open(&temp_fds);
+ if (error)
+ goto done;
error = got_repo_open(&repo, repo_path, NULL, pack_fds);
if (error)
goto done;
+ got_repo_temp_fds_set(repo, temp_fds);
error = apply_unveil(got_repo_get_path_git_dir(repo), 1);
if (error)
@@ -1516,7 +1537,8 @@ cmd_dump(int argc, char *argv[])
done:
if (repo)
got_repo_close(repo);
-
+ if (temp_fds)
+ got_repo_temp_fds_close(temp_fds);
if (pack_fds) {
const struct got_error *pack_err;
@@ -1664,7 +1686,7 @@ cmd_load(int argc, char *argv[])
struct got_pathlist_entry *pe;
struct got_pack_progress_arg ppa;
FILE *in = stdin;
- int *pack_fds = NULL;
+ int *pack_fds = NULL, *temp_fds = NULL;
char *repo_path = NULL;
int list_refs_only = 0;
int noop = 0;
@@ -1675,8 +1697,7 @@ cmd_load(int argc, char *argv[])
TAILQ_INIT(&available_refs);
#ifndef PROFILE
- if (pledge("stdio rpath wpath cpath fattr flock proc exec "
- "sendfd unveil", NULL) == -1)
+ if (pledge("stdio rpath wpath cpath fattr flock unveil", NULL) == -1)
err(1, "pledge");
#endif
@@ -1732,9 +1753,13 @@ cmd_load(int argc, char *argv[])
error = got_repo_pack_fds_open(&pack_fds);
if (error != NULL)
goto done;
+ error = got_repo_temp_fds_open(&temp_fds);
+ if (error)
+ goto done;
error = got_repo_open(&repo, repo_path, NULL, pack_fds);
if (error)
goto done;
+ got_repo_temp_fds_set(repo, temp_fds);
error = apply_unveil(got_repo_get_path_git_dir(repo), 0);
if (error)
@@ -1805,7 +1830,8 @@ cmd_load(int argc, char *argv[])
if (repo)
got_repo_close(repo);
-
+ if (temp_fds)
+ got_repo_temp_fds_close(temp_fds);
if (pack_fds) {
const struct got_error *pack_err;
blob - 7175f86e3c1f00b9b4d870af1638dc3f1068d951
blob + a104bbf0b7040b724e51aa8ef10470b01598568f
--- include/got_repository_admin.h
+++ include/got_repository_admin.h
@@ -47,7 +47,7 @@ got_repo_find_pack(FILE **packfile, struct got_object_
struct got_repository *repo, const char *packfile_path);
/* A callback function which gets invoked with progress information to print. */
-typedef const struct got_error *(*got_pack_index_progress_cb)(void *arg,
+typedef const struct got_error *(*got_repo_pack_index_progress_cb)(void *arg,
off_t packfile_size, int nobj_total, int nobj_indexed,
int nobj_loose, int nobj_resolved);
@@ -55,7 +55,7 @@ typedef const struct got_error *(*got_pack_index_progr
const struct got_error *
got_repo_index_pack(FILE *packfile, struct got_object_id *pack_hash,
struct got_repository *repo,
- got_pack_index_progress_cb progress_cb, void *progress_arg,
+ got_repo_pack_index_progress_cb progress_cb, void *progress_arg,
got_cancel_cb cancel_cb, void *cancel_arg);
typedef const struct got_error *(*got_pack_list_cb)(void *arg,
blob - 8fcb52f1333aada44e211d3b89366979118e617d
blob + 38f24a35b625be0bff6f7c820ada870335f97270
--- lib/got_lib_pack_index.h
+++ lib/got_lib_pack_index.h
@@ -14,7 +14,7 @@
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
-typedef const struct got_error *(got_pack_index_progress_cb)(void *,
+typedef const struct got_error *(*got_pack_index_progress_cb)(void *,
uint32_t nobj_total, uint32_t nobj_indexed, uint32_t nobj_loose,
uint32_t nobj_resolved);
blob - 544f59bad76e1479b2badc43c1532f06d70db9d9
blob + 11251a1e6b4f07ee1f5d6caa281cdd038493e023
--- lib/load.c
+++ lib/load.c
@@ -44,11 +44,13 @@
#include "got_repository_load.h"
#include "got_lib_delta.h"
+#include "got_lib_delta_cache.h"
#include "got_lib_hash.h"
#include "got_lib_object.h"
#include "got_lib_object_cache.h"
#include "got_lib_pack.h"
#include "got_lib_ratelimit.h"
+#include "got_lib_pack_index.h"
#include "got_lib_repository.h"
#include "got_lib_privsep.h"
@@ -203,6 +205,39 @@ copypack(FILE *in, int outfd, off_t *tot,
return NULL;
}
+struct pack_index_progress_arg {
+ off_t packfile_size;
+ got_load_progress_cb progress_cb;
+ void *progress_arg;
+ got_cancel_cb cancel_cb;
+ void *cancel_arg;
+ struct got_ratelimit *rl;
+};
+
+static const struct got_error *
+pack_index_progress(void *arg, uint32_t nobj_total, uint32_t nobj_indexed,
+ uint32_t nobj_loose, uint32_t nobj_resolved)
+{
+ const struct got_error *err = NULL;
+ struct pack_index_progress_arg *a = arg;
+
+ if (a->cancel_cb) {
+ err = a->cancel_cb(a->cancel_arg);
+ if (err)
+ return err;
+ }
+
+ if (nobj_indexed != 0) {
+ err = load_report_progress(a->progress_cb, a->progress_arg,
+ a->rl, a->packfile_size, nobj_total, nobj_indexed,
+ nobj_loose, nobj_resolved);
+ if (err)
+ return err;
+ }
+
+ return err;
+}
+
const struct got_error *
got_repo_load(FILE *in, struct got_pathlist_head *refs_found,
struct got_repository *repo, int list_refs_only, int noop,
@@ -214,7 +249,6 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs
struct got_object *obj;
struct got_packfile_hdr pack_hdr;
struct got_ratelimit rl;
- struct imsgbuf idxibuf;
const char *repo_path;
char *packpath = NULL, *idxpath = NULL;
char *tmppackpath = NULL, *tmpidxpath = NULL;
@@ -226,11 +260,15 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs
size_t i;
ssize_t n;
off_t packsiz;
- int tmpfds[3] = {-1, -1, -1};
+ FILE *tempfiles[3] = { NULL, NULL, NULL };
int imsg_idxfds[2] = {-1, -1};
- int ch, done, nobj, idxstatus;
- pid_t idxpid;
+ int ch, nobj;
+ struct pack_index_progress_arg arg;
+ struct got_pack pack;
+ uint8_t pack_sha1_expected[SHA1_DIGEST_LENGTH];
+ memset(&pack, 0, sizeof(pack));
+
got_ratelimit_init(&rl, 0, 500);
repo_path = got_repo_get_path_git_dir(repo);
@@ -394,80 +432,54 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs
if (nobj == 0)
goto done;
- for (i = 0; i < nitems(tmpfds); i++) {
- tmpfds[i] = got_opentempfd();
- if (tmpfds[i] == -1) {
- err = got_error_from_errno("got_opentempfd");
+ for (i = 0; i < nitems(tempfiles); i++) {
+ tempfiles[i] = got_opentemp();
+ if (tempfiles[i] == NULL) {
+ err = got_error_from_errno("got_opentemp");
goto done;
}
}
+
+ if (lseek(packfd, packsiz - SHA1_DIGEST_LENGTH, SEEK_SET) == -1) {
+ err = got_error_from_errno("lseek");
+ goto done;
+ }
+
+ n = read(packfd, pack_sha1_expected, SHA1_DIGEST_LENGTH);
+ if (n == -1) {
+ err = got_error_from_errno("read");
+ goto done;
+ }
+ if (n != SHA1_DIGEST_LENGTH) {
+ err = got_error(GOT_ERR_IO);
+ goto done;
+ }
+
if (lseek(packfd, 0, SEEK_SET) == -1) {
err = got_error_from_errno("lseek");
goto done;
}
- if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
- err = got_error_from_errno("socketpair");
- goto done;
- }
- idxpid = fork();
- if (idxpid == -1) {
- err= got_error_from_errno("fork");
- goto done;
- } else if (idxpid == 0)
- got_privsep_exec_child(imsg_idxfds,
- GOT_PATH_PROG_INDEX_PACK, tmppackpath);
- if (close(imsg_idxfds[1]) == -1) {
- err = got_error_from_errno("close");
- goto done;
- }
- imsg_idxfds[1] = -1;
- imsg_init(&idxibuf, imsg_idxfds[0]);
+ arg.packfile_size = packsiz;
+ arg.progress_cb = progress_cb;
+ arg.progress_arg = progress_arg;
+ arg.cancel_cb = cancel_cb;
+ arg.cancel_arg = cancel_arg;
+ arg.rl = &rl;
- err = got_privsep_send_index_pack_req(&idxibuf, id.sha1, packfd);
+ pack.fd = packfd;
+ pack.filesize = packsiz;
+ err = got_delta_cache_alloc(&pack.delta_cache);
if (err)
goto done;
- packfd = -1;
- err = got_privsep_send_index_pack_outfd(&idxibuf, idxfd);
+ err = got_pack_index(&pack, idxfd,
+ tempfiles[0], tempfiles[1], tempfiles[2], pack_sha1_expected,
+ pack_index_progress, &arg, &rl);
if (err)
goto done;
- idxfd = -1;
- for (i = 0; i < nitems(tmpfds); i++) {
- err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
- if (err != NULL)
- goto done;
- tmpfds[i] = -1;
- }
-
- done = 0;
- while (!done) {
- int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
-
- err = got_privsep_recv_index_progress(&done, &nobj_total,
- &nobj_indexed, &nobj_loose, &nobj_resolved, &idxibuf);
- if (err)
- goto done;
- if (nobj_indexed != 0) {
- err = load_report_progress(progress_cb, progress_arg,
- &rl, packsiz, nobj_total, nobj_indexed,
- nobj_loose, nobj_resolved);
- if (err)
- goto done;
- }
- }
- if (close(imsg_idxfds[0]) == -1) {
- err = got_error_from_errno("close");
- goto done;
- }
- imsg_idxfds[0] = -1;
- if (waitpid(idxpid, &idxstatus, 0) == -1) {
- err = got_error_from_errno("waitpid");
- goto done;
- }
-
if (noop)
goto done;
@@ -524,9 +536,11 @@ got_repo_load(FILE *in, struct got_pathlist_head *refs
if (imsg_idxfds[1] != -1 && close(imsg_idxfds[1]) == -1 && err == NULL)
err = got_error_from_errno("close");
- for (i = 0; i < nitems(tmpfds); ++i)
- if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
- err = got_error_from_errno("close");
+ for (i = 0; i < nitems(tempfiles); ++i) {
+ if (tempfiles[i] != NULL && fclose(tempfiles[i]) == EOF &&
+ err == NULL)
+ err = got_error_from_errno("fclose");
+ }
return err;
}
blob - ba7276ad6779e4b6a924ef8284b01c4bbdb79216
blob + cce18fb6b576d23d2081a21f54b3637c7e910aee
--- lib/repository_admin.c
+++ lib/repository_admin.c
@@ -46,15 +46,18 @@
#include "got_path.h"
#include "got_lib_delta.h"
+#include "got_lib_delta_cache.h"
#include "got_lib_object.h"
#include "got_lib_object_idset.h"
#include "got_lib_object_cache.h"
+#include "got_lib_object_parse.h"
#include "got_lib_pack.h"
#include "got_lib_privsep.h"
#include "got_lib_repository.h"
#include "got_lib_ratelimit.h"
-#include "got_lib_pack_create.h"
#include "got_lib_hash.h"
+#include "got_lib_pack_index.h"
+#include "got_lib_pack_create.h"
#include "got_lib_lockfile.h"
#ifndef nitems
@@ -271,29 +274,58 @@ done:
return err;
}
+struct pack_index_progress_arg {
+ off_t packfile_size;
+ got_repo_pack_index_progress_cb progress_cb;
+ void *progress_arg;
+ got_cancel_cb cancel_cb;
+ void *cancel_arg;
+};
+
+static const struct got_error *
+pack_index_progress(void *arg, uint32_t nobj_total, uint32_t nobj_indexed,
+ uint32_t nobj_loose, uint32_t nobj_resolved)
+{
+ const struct got_error *err = NULL;
+ struct pack_index_progress_arg *a = arg;
+
+ if (a->cancel_cb) {
+ err = a->cancel_cb(a->cancel_arg);
+ if (err)
+ return err;
+ }
+
+ if (a->progress_cb) {
+ err = a->progress_cb(a->progress_arg, a->packfile_size,
+ nobj_total, nobj_indexed, nobj_loose, nobj_resolved);
+ }
+
+ return err;
+}
+
const struct got_error *
got_repo_index_pack(FILE *packfile, struct got_object_id *pack_hash,
struct got_repository *repo,
- got_pack_index_progress_cb progress_cb, void *progress_arg,
+ got_repo_pack_index_progress_cb progress_cb, void *progress_arg,
got_cancel_cb cancel_cb, void *cancel_arg)
{
size_t i;
char *path;
- int imsg_idxfds[2];
- int npackfd = -1, idxfd = -1, nidxfd = -1;
- int tmpfds[3];
- int idxstatus, done = 0;
- const struct got_error *err;
- struct imsgbuf idxibuf;
- pid_t idxpid;
+ int idxfd = -1;
+ FILE *tempfiles[3] = { NULL, NULL, NULL };
+ const struct got_error *err, *close_err;
char *tmpidxpath = NULL;
char *packfile_path = NULL, *idxpath = NULL, *id_str = NULL;
const char *repo_path = got_repo_get_path_git_dir(repo);
struct stat sb;
+ struct pack_index_progress_arg arg;
+ struct got_pack pack;
+ struct got_ratelimit rl;
- for (i = 0; i < nitems(tmpfds); i++)
- tmpfds[i] = -1;
+ memset(&pack, 0, sizeof(pack));
+ got_ratelimit_init(&rl, 0, 500);
+
if (asprintf(&path, "%s/%s/indexing.idx",
repo_path, GOT_OBJECTS_PACK_DIR) == -1) {
err = got_error_from_errno("asprintf");
@@ -308,16 +340,10 @@ got_repo_index_pack(FILE *packfile, struct got_object_
goto done;
}
- nidxfd = dup(idxfd);
- if (nidxfd == -1) {
- err = got_error_from_errno("dup");
- goto done;
- }
-
- for (i = 0; i < nitems(tmpfds); i++) {
- tmpfds[i] = got_opentempfd();
- if (tmpfds[i] == -1) {
- err = got_error_from_errno("got_opentempfd");
+ for (i = 0; i < nitems(tempfiles); i++) {
+ tempfiles[i] = got_opentemp();
+ if (tempfiles[i] == NULL) {
+ err = got_error_from_errno("got_opentemp");
goto done;
}
}
@@ -343,74 +369,23 @@ got_repo_index_pack(FILE *packfile, struct got_object_
goto done;
}
- if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
- err = got_error_from_errno("socketpair");
- goto done;
- }
- idxpid = fork();
- if (idxpid == -1) {
- err= got_error_from_errno("fork");
- goto done;
- } else if (idxpid == 0)
- got_privsep_exec_child(imsg_idxfds,
- GOT_PATH_PROG_INDEX_PACK, packfile_path);
- if (close(imsg_idxfds[1]) == -1) {
- err = got_error_from_errno("close");
- goto done;
- }
- imsg_init(&idxibuf, imsg_idxfds[0]);
+ arg.packfile_size = sb.st_size;
+ arg.progress_cb = progress_cb;
+ arg.progress_arg = progress_arg;
+ arg.cancel_cb = cancel_cb;
+ arg.cancel_arg = cancel_arg;
- npackfd = dup(fileno(packfile));
- if (npackfd == -1) {
- err = got_error_from_errno("dup");
+ pack.fd = fileno(packfile);
+ pack.filesize = sb.st_size;
+ err = got_delta_cache_alloc(&pack.delta_cache);
+ if (err)
goto done;
- }
- err = got_privsep_send_index_pack_req(&idxibuf, pack_hash->sha1,
- npackfd);
- if (err != NULL)
- goto done;
- npackfd = -1;
- err = got_privsep_send_index_pack_outfd(&idxibuf, nidxfd);
- if (err != NULL)
- goto done;
- nidxfd = -1;
- for (i = 0; i < nitems(tmpfds); i++) {
- err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
- if (err != NULL)
- goto done;
- tmpfds[i] = -1;
- }
- done = 0;
- while (!done) {
- int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
- if (cancel_cb) {
- err = cancel_cb(cancel_arg);
- if (err)
- goto done;
- }
-
- err = got_privsep_recv_index_progress(&done, &nobj_total,
- &nobj_indexed, &nobj_loose, &nobj_resolved,
- &idxibuf);
- if (err != NULL)
- goto done;
- if (nobj_indexed != 0) {
- err = progress_cb(progress_arg, sb.st_size,
- nobj_total, nobj_indexed, nobj_loose,
- nobj_resolved);
- if (err)
- break;
- }
- }
- if (close(imsg_idxfds[0]) == -1) {
- err = got_error_from_errno("close");
+ err = got_pack_index(&pack, idxfd,
+ tempfiles[0], tempfiles[1], tempfiles[2], pack_hash->sha1,
+ pack_index_progress, &arg, &rl);
+ if (err)
goto done;
- }
- if (waitpid(idxpid, &idxstatus, 0) == -1) {
- err = got_error_from_errno("waitpid");
- goto done;
- }
if (rename(tmpidxpath, idxpath) == -1) {
err = got_error_from_errno3("rename", tmpidxpath, idxpath);
@@ -420,15 +395,16 @@ got_repo_index_pack(FILE *packfile, struct got_object_
tmpidxpath = NULL;
done:
+ close_err = got_pack_close(&pack);
+ if (close_err && err == NULL)
+ err = close_err;
if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
err = got_error_from_errno2("unlink", tmpidxpath);
- if (npackfd != -1 && close(npackfd) == -1 && err == NULL)
- err = got_error_from_errno("close");
if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
err = got_error_from_errno("close");
- for (i = 0; i < nitems(tmpfds); i++) {
- if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
- err = got_error_from_errno("close");
+ for (i = 0; i < nitems(tempfiles); i++) {
+ if (tempfiles[i] && fclose(tempfiles[i]) == EOF && err == NULL)
+ err = got_error_from_errno("fclose");
}
free(tmpidxpath);
free(idxpath);
@@ -725,8 +701,7 @@ get_loose_object_ids(struct got_object_idset **loose_i
err = got_error_from_errno("fstat");
goto done;
}
- err = got_object_read_header_privsep(&obj, &id, repo,
- fd);
+ err = got_object_read_header(&obj, fd);
if (err)
goto done;
fd = -1; /* already closed */
run gotadmin without libexec helpers