From: Christian Weisgerber Subject: Re: gotadmin cleanup To: gameoftrees@openbsd.org Date: Sun, 4 Jul 2021 19:14:43 +0200 Stefan Sperling: > I consider the speed acceptable only because I expect that this command > will not be used very often. There may be ways to speed it up but for > the moment I am out of ideas. I guess in modern environments it's not a bottleneck per se, but should the progress output be moderated? Updating the count for each commit produces a lot of spew: $ script -c 'gotadmin cleanup -r ports.git' Script started, output file is typescript 103 loose objects; 158146 commits scanned; 20 objects purged loose total size before: 455K loose total size after: 219K disk space freed: 237K loose objects also found in pack files: 20 Script done, output file is typescript $ ls -lh typescript -rw-r--r-- 1 naddy naddy 6.2M Jul 4 14:40 typescript Unfortunately, I'm not sure how to do this. Any sort of skipping updates requires the addition of a flush operation to make sure the last update actually gets printed. I came up with the patch below, but I don't like it at all. Also, there are already a number of progress callbacks implemented, so maybe a generic solution is needed? ---------------- diff 2252c019b5ec7ee342e8ebf2854ec8d83182b3ee /home/naddy/got blob - 3a8fe2211096a986460483094705912532891e8e file + gotadmin/gotadmin.c --- gotadmin/gotadmin.c +++ gotadmin/gotadmin.c @@ -15,6 +15,7 @@ */ #include +#include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include @@ -909,14 +911,24 @@ struct got_cleanup_progress_arg { int verbosity; int printed_something; int dry_run; + struct timespec last_ts; }; static const struct got_error * -cleanup_progress(void *arg, int nloose, int ncommits, int npurged) +cleanup_progress(void *arg, int nloose, int ncommits, int npurged, int flush) { struct got_cleanup_progress_arg *a = arg; + struct timespec ts, elapsed; + const struct timespec interval = { 0, 500000000 }; int print_loose = 0, print_commits = 0, print_purged = 0; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) + return got_error_from_errno("clock_gettime"); + timespecsub(&ts, &a->last_ts, &elapsed); + if (timespeccmp(&elapsed, &interval, <) && !flush) + return NULL; + a->last_ts = ts; + if (a->last_nloose != nloose) { print_loose = 1; a->last_nloose = nloose; blob - 99fa6426ad538680697e303f69032b8c33f9266a file + include/got_repository_admin.h --- include/got_repository_admin.h +++ include/got_repository_admin.h @@ -70,7 +70,7 @@ got_repo_list_pack(FILE *packfile, struct got_object_i /* A callback function which gets invoked with cleanup information to print. */ typedef const struct got_error *(*got_cleanup_progress_cb)(void *arg, - int nloose, int ncommits, int npurged); + int nloose, int ncommits, int npurged, int flush); /* * Walk objects reachable via references to determine whether any loose blob - 7461d26208ce44417f438e825f9198e87bf5f41d file + lib/repository_admin.c --- lib/repository_admin.c +++ lib/repository_admin.c @@ -625,7 +625,7 @@ get_loose_object_ids(struct got_object_idset **loose_i if (asprintf(&path, "%s/%.2x", path_objects, i) == -1) { err = got_error_from_errno("asprintf"); - break; + goto done; } dir = opendir(path); @@ -635,7 +635,7 @@ get_loose_object_ids(struct got_object_idset **loose_i continue; } err = got_error_from_errno2("opendir", path); - break; + goto done; } while ((dent = readdir(dir)) != NULL) { @@ -689,7 +689,7 @@ get_loose_object_ids(struct got_object_idset **loose_i if (progress_cb) { err = progress_cb(progress_arg, got_object_idset_num_elements(*loose_ids), - -1, -1); + -1, -1, 0); if (err) goto done; } @@ -704,6 +704,9 @@ get_loose_object_ids(struct got_object_idset **loose_i free(path); path = NULL; } + if (progress_cb) + err = progress_cb(progress_arg, + got_object_idset_num_elements(*loose_ids), -1, -1, 1); done: if (dir && closedir(dir) != 0 && err == NULL) err = got_error_from_errno("closedir"); @@ -922,7 +925,7 @@ load_commit_or_tag(struct got_object_idset *loose_ids, if (cancel_cb) { err = (*cancel_cb)(cancel_arg); if (err) - break; + goto done; } qid = STAILQ_FIRST(&ids); @@ -936,16 +939,16 @@ load_commit_or_tag(struct got_object_idset *loose_ids, err = got_object_idset_add(traversed_ids, qid->id, NULL); if (err) - break; + goto done; /* This commit or tag is referenced. */ err = preserve_loose_object(loose_ids, qid->id, repo, npacked); if (err) - break; + goto done; err = got_object_get_type(&obj_type, repo, qid->id); if (err) - break; + goto done; switch (obj_type) { case GOT_OBJ_TYPE_COMMIT: err = got_object_open_as_commit(&commit, repo, qid->id); @@ -1002,16 +1005,17 @@ load_commit_or_tag(struct got_object_idset *loose_ids, err = load_tree(loose_ids, traversed_ids, tree_id, "", repo, npacked, cancel_cb, cancel_arg); if (err) - break; + goto done; } if (commit || tag) (*ncommits)++; /* scanned tags are counted as commits */ if (progress_cb) { - err = progress_cb(progress_arg, nloose, *ncommits, -1); + err = progress_cb(progress_arg, nloose, + *ncommits, -1, 0); if (err) - break; + goto done; } if (commit) { @@ -1020,7 +1024,7 @@ load_commit_or_tag(struct got_object_idset *loose_ids, parent_ids = got_object_commit_get_parent_ids(commit); err = got_object_id_queue_copy(parent_ids, &ids); if (err) - break; + goto done; got_object_commit_close(commit); commit = NULL; } @@ -1031,6 +1035,8 @@ load_commit_or_tag(struct got_object_idset *loose_ids, got_object_qid_free(qid); qid = NULL; } + if (progress_cb) + err = progress_cb(progress_arg, nloose, *ncommits, -1, 1); done: if (qid) got_object_qid_free(qid); @@ -1090,7 +1096,7 @@ purge_loose_object(struct got_object_id *id, void *dat a->size_purged += sb.st_size; if (a->progress_cb) { err = a->progress_cb(a->progress_arg, a->nloose, - a->ncommits, a->npurged); + a->ncommits, a->npurged, 0); } done: if (fd != -1 && close(fd) == -1 && err == NULL) @@ -1158,7 +1164,7 @@ got_repo_purge_unreferenced_loose_objects(struct got_r /* Produce a final progress report in case no objects can be purged. */ if (got_object_idset_num_elements(loose_ids) == 0 && progress_cb) { - err = progress_cb(progress_arg, nloose, ncommits, 0); + err = progress_cb(progress_arg, nloose, ncommits, 0, 0); if (err) goto done; } @@ -1176,6 +1182,9 @@ got_repo_purge_unreferenced_loose_objects(struct got_r if (err) goto done; *size_after = *size_before - arg.size_purged; + if (progress_cb) + err = progress_cb(progress_arg, nloose, ncommits, + arg.npurged, 1); done: got_object_idset_free(loose_ids); got_object_idset_free(traversed_ids); -- Christian "naddy" Weisgerber naddy@mips.inka.de