"GOT", but the "O" is a cute, smiling pufferfish. Index | Thread | Search

From:
Stefan Sperling <stsp@stsp.name>
Subject:
improve pack progress output
To:
gameoftrees@openbsd.org
Date:
Sun, 13 Mar 2022 15:13:28 +0100

Download raw body.

Thread
The initial steps of packing will usually complete relatively quickly,
which is why no progress output was implemented for them.
However, when creating huge packs in large repositories (such as running
pack -a in openbsd src.git) the initial steps can take a couple of minutes
of uncomfortable silence to get through.

This patch implements progress output during these steps, which appears
on a seperate line (the first line printed in the example below):

$ gotadmin pack main
4066 commits colored; 25233 objects found; 4485 trees scanned
packing 1 reference; 606 objects; deltify: 100%; writing pack:  710K 100%
Wrote 0e79c0a9ea1bdde529b6ed96b37f534e45bda446.pack
 710K packed; indexing 100%; resolving deltas 100%
Indexed 0e79c0a9ea1bdde529b6ed96b37f534e45bda446.pack
$

One test was trying to check progress output against known content.
Because the exact output depends on timing, just pipe it to /dev/null.
Only check stderr which has everything the test wants to know.

Maybe it would have been smart to put all progress-related parameters into
a struct instead of passing individual arguments around. But that can be
fixed later. I have kept the current idiom for consistency.

ok?

 
diff c0c2b3d1e2c80fe9ee36670cff033f948cca9fd6 7f6c6d86bb14667458ebd9f9272d6efd7b2b73b9
blob - 09c52c1e9bd95ceb4f5fa1f135b3fb570b7285b0
blob + 3a6211928791cdfffe4e708f6f6145429f476822
--- got/got.c
+++ got/got.c
@@ -7852,9 +7852,32 @@ usage_send(void)
 	exit(1);
 }
 
+static void
+print_load_info(int print_colored, int print_found, int print_trees,
+    int ncolored, int nfound, int ntrees)
+{
+	if (print_colored) {
+		printf("%d commit%s colored", ncolored,
+		    ncolored == 1 ? "" : "s");
+	}
+	if (print_found) {
+		printf("%s%d object%s found",
+		    ncolored > 0 ? "; " : "",
+		    nfound, nfound == 1 ? "" : "s");
+	}
+	if (print_trees) {
+		printf("; %d tree%s scanned", ntrees,
+		    ntrees == 1 ? "" : "s");
+	}
+}
+
 struct got_send_progress_arg {
 	char last_scaled_packsize[FMT_SCALED_STRSIZE];
 	int verbosity;
+	int last_ncolored;
+	int last_nfound;
+	int last_ntrees;
+	int loading_done;
 	int last_ncommits;
 	int last_nobj_total;
 	int last_p_deltify;
@@ -7866,14 +7889,15 @@ struct got_send_progress_arg {
 };
 
 static const struct got_error *
-send_progress(void *arg, off_t packfile_size, int ncommits, int nobj_total,
-    int nobj_deltify, int nobj_written, off_t bytes_sent, const char *refname,
-    int success)
+send_progress(void *arg, int ncolored, int nfound, int ntrees,
+    off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify,
+    int nobj_written, off_t bytes_sent, const char *refname, int success)
 {
 	struct got_send_progress_arg *a = arg;
 	char scaled_packsize[FMT_SCALED_STRSIZE];
 	char scaled_sent[FMT_SCALED_STRSIZE];
 	int p_deltify = 0, p_written = 0, p_sent = 0;
+	int print_colored = 0, print_found = 0, print_trees = 0;
 	int print_searching = 0, print_total = 0;
 	int print_deltify = 0, print_written = 0, print_sent = 0;
 
@@ -7903,6 +7927,39 @@ send_progress(void *arg, off_t packfile_size, int ncom
 		return NULL;
 	}
 
+	if (a->last_ncolored != ncolored) {
+		print_colored = 1;
+		a->last_ncolored = ncolored;
+	}
+
+	if (a->last_nfound != nfound) {
+		print_colored = 1;
+		print_found = 1;
+		a->last_nfound = nfound;
+	}
+
+	if (a->last_ntrees != ntrees) {
+		print_colored = 1;
+		print_found = 1;
+		print_trees = 1;
+		a->last_ntrees = ntrees;
+	}
+
+	if ((print_colored || print_found || print_trees) &&
+	    !a->loading_done) {
+		printf("\r");
+		print_load_info(print_colored, print_found, print_trees,
+		    ncolored, nfound, ntrees);
+		a->printed_something = 1;
+		fflush(stdout);
+		return NULL;
+	} else if (!a->loading_done) {
+		printf("\r");
+		print_load_info(1, 1, 1, ncolored, nfound, ntrees);
+		printf("\n");
+		a->loading_done = 1;
+	}
+
 	if (fmt_scaled(packfile_size, scaled_packsize) == -1)
 		return got_error_from_errno("fmt_scaled");
 	if (fmt_scaled(bytes_sent, scaled_sent) == -1)
blob - bd7343dcca754f398aaba30df74ecd27b90d2ee7
blob + 09d1ca1db98e99e774fa86e99f0212c6871c7fcb
--- gotadmin/gotadmin.c
+++ gotadmin/gotadmin.c
@@ -381,6 +381,10 @@ usage_pack(void)
 
 struct got_pack_progress_arg {
 	char last_scaled_size[FMT_SCALED_STRSIZE];
+	int last_ncolored;
+	int last_nfound;
+	int last_ntrees;
+	int loading_done;
 	int last_ncommits;
 	int last_nobj_total;
 	int last_p_deltify;
@@ -391,19 +395,73 @@ struct got_pack_progress_arg {
 	int printed_something;
 };
 
+static void
+print_load_info(int print_colored, int print_found, int print_trees,
+    int ncolored, int nfound, int ntrees)
+{
+	if (print_colored) {
+		printf("%d commit%s colored", ncolored,
+		    ncolored == 1 ? "" : "s");
+	}
+	if (print_found) {
+		printf("%s%d object%s found",
+		    ncolored > 0 ? "; " : "",
+		    nfound, nfound == 1 ? "" : "s");
+	}
+	if (print_trees) {
+		printf("; %d tree%s scanned", ntrees,
+		    ntrees == 1 ? "" : "s");
+	}
+}
+
 static const struct got_error *
-pack_progress(void *arg, off_t packfile_size, int ncommits,
-    int nobj_total, int nobj_deltify, int nobj_written)
+pack_progress(void *arg, int ncolored, int nfound, int ntrees,
+    off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify,
+    int nobj_written)
 {
 	struct got_pack_progress_arg *a = arg;
 	char scaled_size[FMT_SCALED_STRSIZE];
 	int p_deltify, p_written;
+	int print_colored = 0, print_found = 0, print_trees = 0;
 	int print_searching = 0, print_total = 0;
 	int print_deltify = 0, print_written = 0;
 
 	if (a->verbosity < 0)
 		return NULL;
 
+	if (a->last_ncolored != ncolored) {
+		print_colored = 1;
+		a->last_ncolored = ncolored;
+	}
+
+	if (a->last_nfound != nfound) {
+		print_colored = 1;
+		print_found = 1;
+		a->last_nfound = nfound;
+	}
+
+	if (a->last_ntrees != ntrees) {
+		print_colored = 1;
+		print_found = 1;
+		print_trees = 1;
+		a->last_ntrees = ntrees;
+	}
+
+	if ((print_colored || print_found || print_trees) &&
+	    !a->loading_done) {
+		printf("\r");
+		print_load_info(print_colored, print_found, print_trees,
+		    ncolored, nfound, ntrees);
+		a->printed_something = 1;
+		fflush(stdout);
+		return NULL;
+	} else if (!a->loading_done) {
+		printf("\r");
+		print_load_info(1, 1, 1, ncolored, nfound, ntrees);
+		printf("\n");
+		a->loading_done = 1;
+	}
+
 	if (fmt_scaled(packfile_size, scaled_size) == -1)
 		return got_error_from_errno("fmt_scaled");
 
blob - 8bc7ea591673f80b94917ac36991936b0fc46038
blob + 2caf2a94d4ed262a215863f07c9e967a8f052cc3
--- include/got_repository_admin.h
+++ include/got_repository_admin.h
@@ -16,8 +16,8 @@
 
 /* A callback function which gets invoked with progress information to print. */
 typedef const struct got_error *(*got_pack_progress_cb)(void *arg,
-    off_t packfile_size, int ncommits, int nobj_total, int obj_deltify,
-    int nobj_written);
+    int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits,
+    int nobj_total, int obj_deltify, int nobj_written);
 
 /*
  * Attempt to pack objects reachable via 'include_refs' into a new packfile.
blob - 225bc42faab244492a77e967c829b9cf47301009
blob + 19e9e4f116e70c354ec4bbff8ce7e5a1615acd1f
--- include/got_send.h
+++ include/got_send.h
@@ -36,8 +36,8 @@ const struct got_error *got_send_connect(pid_t *, int 
 
 /* A callback function which gets invoked with progress information to print. */
 typedef const struct got_error *(*got_send_progress_cb)(void *,
-    off_t packfile_size, int ncommits, int nobj_total,
-    int nobj_deltify, int nobj_written, off_t bytes_sent,
+    int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits,
+    int nobj_total, int nobj_deltify, int nobj_written, off_t bytes_sent,
     const char *refname, int success);
 
 /*
blob - 3b6af380cd3ae1ac103c402a5365107c232529dd
blob + 598a24d3ed3f054d8c65d0ab83d32f2db020856c
--- lib/pack_create.c
+++ lib/pack_create.c
@@ -408,8 +408,9 @@ encode_delta(struct got_pack_meta *m, struct got_raw_o
 
 static const struct got_error *
 report_progress(got_pack_progress_cb progress_cb, void *progress_arg,
-    struct got_ratelimit *rl, off_t packfile_size, int ncommits,
-    int nobj_total, int obj_deltify, int nobj_written)
+    struct got_ratelimit *rl, int ncolored, int nfound, int ntrees,
+    off_t packfile_size, int ncommits, int nobj_total, int obj_deltify,
+    int nobj_written)
 {
 	const struct got_error *err;
 	int elapsed;
@@ -421,8 +422,8 @@ report_progress(got_pack_progress_cb progress_cb, void
 	if (err || !elapsed)
 		return err;
 
-	return progress_cb(progress_arg, packfile_size, ncommits,
-	    nobj_total, obj_deltify, nobj_written);
+	return progress_cb(progress_arg, ncolored, nfound, ntrees,
+	    packfile_size, ncommits, nobj_total, obj_deltify, nobj_written);
 }
 
 static const struct got_error *
@@ -540,6 +541,9 @@ struct search_deltas_arg {
 	struct got_ratelimit *rl;
 	got_cancel_cb cancel_cb;
 	void *cancel_arg;
+	int ncolored;
+	int nfound;
+	int ntrees;
 	int ncommits;
 };
 
@@ -585,8 +589,8 @@ search_delta_for_object(struct got_object_id *id, void
 		if (err)
 			goto done;
 		err = report_progress(a->progress_cb, a->progress_arg, a->rl,
-		    0L, a->ncommits, got_object_idset_num_elements(a->idset),
-		    a->v->nmeta, 0);
+		    a->ncolored, a->nfound, a->ntrees, 0L, a->ncommits,
+		    got_object_idset_num_elements(a->idset), a->v->nmeta, 0);
 	}
 done:
 	got_object_close(obj);
@@ -595,7 +599,8 @@ done:
 
 static const struct got_error *
 search_deltas(struct got_pack_metavec *v, struct got_object_idset *idset,
-    int delta_cache_fd, int ncommits, struct got_repository *repo,
+    int delta_cache_fd, int ncolored, int nfound, int ntrees, int ncommits,
+    struct got_repository *repo,
     got_pack_progress_cb progress_cb, void *progress_arg,
     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 {
@@ -635,6 +640,9 @@ search_deltas(struct got_pack_metavec *v, struct got_o
 	sda.rl = rl;
 	sda.cancel_cb = cancel_cb;
 	sda.cancel_arg = cancel_arg;
+	sda.ncolored = ncolored;
+	sda.nfound = nfound;
+	sda.ntrees = ntrees;
 	sda.ncommits = ncommits;
 	err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
 done:
@@ -643,8 +651,9 @@ done:
 }
 
 static const struct got_error *
-pick_deltas(struct got_pack_meta **meta, int nmeta, int ncommits,
-    int nreused, FILE *delta_cache, struct got_repository *repo,
+pick_deltas(struct got_pack_meta **meta, int nmeta, int ncolored,
+    int nfound, int ntrees, int ncommits, int nreused, FILE *delta_cache,
+    struct got_repository *repo,
     got_pack_progress_cb progress_cb, void *progress_arg,
     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 {
@@ -667,7 +676,8 @@ pick_deltas(struct got_pack_meta **meta, int nmeta, in
 				break;
 		}
 		err = report_progress(progress_cb, progress_arg, rl,
-		    0L, ncommits, nreused + nmeta, nreused + i, 0);
+		    ncolored, nfound, ntrees, 0L, ncommits, nreused + nmeta,
+		    nreused + i, 0);
 		if (err)
 			goto done;
 		m = meta[i];
@@ -877,7 +887,9 @@ static const struct got_error *
 load_tree_entries(struct got_object_id_queue *ids, int want_meta,
     struct got_object_idset *idset, struct got_object_id *tree_id,
     const char *dpath, time_t mtime, struct got_repository *repo,
-    int loose_obj_only, got_cancel_cb cancel_cb, void *cancel_arg)
+    int loose_obj_only, int *ncolored, int *nfound, int *ntrees,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 {
 	const struct got_error *err;
 	struct got_tree_object *tree;
@@ -888,6 +900,12 @@ load_tree_entries(struct got_object_id_queue *ids, int
 	if (err)
 		return err;
 
+	(*ntrees)++;
+	err = report_progress(progress_cb, progress_arg, rl,
+	    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
+	if (err)
+		return err;
+
 	for (i = 0; i < got_object_tree_get_nentries(tree); i++) {
 		struct got_tree_entry *e = got_object_tree_get_entry(tree, i);
 		struct got_object_id *id = got_tree_entry_get_id(e);
@@ -920,6 +938,11 @@ load_tree_entries(struct got_object_id_queue *ids, int
 			    GOT_OBJ_TYPE_BLOB, mtime, loose_obj_only, repo);
 			if (err)
 				break;
+			(*nfound)++;
+			err = report_progress(progress_cb, progress_arg, rl,
+			    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
+			if (err)
+				break;
 		}
 		free(p);
 		p = NULL;
@@ -933,8 +956,10 @@ load_tree_entries(struct got_object_id_queue *ids, int
 static const struct got_error *
 load_tree(int want_meta, struct got_object_idset *idset,
     struct got_object_id *tree_id, const char *dpath, time_t mtime,
-    int loose_obj_only, struct got_repository *repo,
-    got_cancel_cb cancel_cb, void *cancel_arg)
+    struct got_repository *repo, int loose_obj_only,
+    int *ncolored, int *nfound, int *ntrees,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 {
 	const struct got_error *err = NULL;
 	struct got_object_id_queue tree_ids;
@@ -972,8 +997,16 @@ load_tree(int want_meta, struct got_object_idset *idse
 			break;
 		}
 
+		(*nfound)++;
+		err = report_progress(progress_cb, progress_arg, rl,
+		    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
+		if (err)
+			break;
+
 		err = load_tree_entries(&tree_ids, want_meta, idset, qid->id,
-		    dpath, mtime, repo, loose_obj_only, cancel_cb, cancel_arg);
+		    dpath, mtime, repo, loose_obj_only, ncolored, nfound,
+		    ntrees, progress_cb, progress_arg, rl,
+		    cancel_cb, cancel_arg);
 		got_object_qid_free(qid);
 		if (err)
 			break;
@@ -986,7 +1019,9 @@ load_tree(int want_meta, struct got_object_idset *idse
 static const struct got_error *
 load_commit(int want_meta, struct got_object_idset *idset,
     struct got_object_id *id, struct got_repository *repo, int loose_obj_only,
-    got_cancel_cb cancel_cb, void *cancel_arg)
+    int *ncolored, int *nfound, int *ntrees,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 {
 	const struct got_error *err;
 	struct got_commit_object *commit;
@@ -1013,9 +1048,16 @@ load_commit(int want_meta, struct got_object_idset *id
 	if (err)
 		goto done;
 
+	(*nfound)++;
+	err = report_progress(progress_cb, progress_arg, rl,
+	    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
+	if (err)
+		goto done;
+
 	err = load_tree(want_meta, idset, got_object_commit_get_tree_id(commit),
 	    "", got_object_commit_get_committer_time(commit),
-	    loose_obj_only, repo, cancel_cb, cancel_arg);
+	    repo, loose_obj_only, ncolored, nfound, ntrees,
+	    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
 done:
 	got_object_commit_close(commit);
 	return err;
@@ -1024,7 +1066,9 @@ done:
 static const struct got_error *
 load_tag(int want_meta, struct got_object_idset *idset,
     struct got_object_id *id, struct got_repository *repo, int loose_obj_only,
-    got_cancel_cb cancel_cb, void *cancel_arg)
+    int *ncolored, int *nfound, int *ntrees,
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 {
 	const struct got_error *err;
 	struct got_tag_object *tag = NULL;
@@ -1051,17 +1095,25 @@ load_tag(int want_meta, struct got_object_idset *idset
 	if (err)
 		goto done;
 
+	(*nfound)++;
+	err = report_progress(progress_cb, progress_arg, rl,
+	    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
+	if (err)
+		goto done;
+
 	switch (got_object_tag_get_object_type(tag)) {
 	case GOT_OBJ_TYPE_COMMIT:
 		err = load_commit(want_meta, idset,
-		    got_object_tag_get_object_id(tag), repo,
-		    loose_obj_only, cancel_cb, cancel_arg);
+		    got_object_tag_get_object_id(tag), repo, loose_obj_only,
+		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
+		    cancel_cb, cancel_arg);
 		break;
 	case GOT_OBJ_TYPE_TREE:
 		err = load_tree(want_meta, idset,
 		    got_object_tag_get_object_id(tag), "",
-		    got_object_tag_get_tagger_time(tag),
-		    loose_obj_only, repo, cancel_cb, cancel_arg);
+		    got_object_tag_get_tagger_time(tag), repo, loose_obj_only,
+		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
+		    cancel_cb, cancel_arg);
 		break;
 	default:
 		break;
@@ -1183,11 +1235,12 @@ append_id(struct got_object_id *id, void *data, void *
 }
 
 static const struct got_error *
-findtwixt(struct got_object_id ***res, int *nres,
+findtwixt(struct got_object_id ***res, int *nres, int *ncolored,
     struct got_object_id **head, int nhead,
     struct got_object_id **tail, int ntail,
     struct got_repository *repo,
-    got_cancel_cb cancel_cb, void *cancel_arg)
+    got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 {
 	const struct got_error *err = NULL;
 	struct got_object_id_queue ids;
@@ -1198,6 +1251,7 @@ findtwixt(struct got_object_id ***res, int *nres,
 	STAILQ_INIT(&ids);
 	*res = NULL;
 	*nres = 0;
+	*ncolored = 0;
 
 	keep = got_object_idset_alloc();
 	if (keep == NULL)
@@ -1248,6 +1302,12 @@ findtwixt(struct got_object_id ***res, int *nres,
 		else
 			ncolor = COLOR_BLANK;
 
+		(*ncolored)++;
+		err = report_progress(progress_cb, progress_arg, rl,
+		    *ncolored, 0, 0, 0L, 0, 0, 0, 0);
+		if (err)
+			goto done;
+
 		if (ncolor == COLOR_DROP || (ncolor == COLOR_KEEP &&
 		    qcolor == COLOR_KEEP)) {
 			STAILQ_REMOVE_HEAD(&ids, entry);
@@ -1336,17 +1396,22 @@ done:
 }
 
 static const struct got_error *
-load_object_ids(struct got_object_idset *idset,
-    struct got_object_id **theirs, int ntheirs,
+load_object_ids(int *ncolored, int *nfound, int *ntrees,
+    struct got_object_idset *idset, struct got_object_id **theirs, int ntheirs,
     struct got_object_id **ours, int nours, struct got_repository *repo,
-    int loose_obj_only, got_cancel_cb cancel_cb, void *cancel_arg)
+    int loose_obj_only, got_pack_progress_cb progress_cb, void *progress_arg,
+    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 {
 	const struct got_error *err = NULL;
 	struct got_object_id **ids = NULL;
 	int i, nobj = 0, obj_type;
 
-	err = findtwixt(&ids, &nobj, ours, nours, theirs, ntheirs, repo,
-	    cancel_cb, cancel_arg);
+	*ncolored = 0;
+	*nfound = 0;
+	*ntrees = 0;
+
+	err = findtwixt(&ids, &nobj, ncolored, ours, nours, theirs, ntheirs,
+	    repo, progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
 	if (err || nobj == 0)
 		goto done;
 
@@ -1359,8 +1424,9 @@ load_object_ids(struct got_object_idset *idset,
 			return err;
 		if (obj_type != GOT_OBJ_TYPE_COMMIT)
 			continue;
-		err = load_commit(0, idset, id, repo,
-		    loose_obj_only, cancel_cb, cancel_arg);
+		err = load_commit(0, idset, id, repo, loose_obj_only,
+		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
+		    cancel_cb, cancel_arg);
 		if (err)
 			goto done;
 	}
@@ -1379,15 +1445,17 @@ load_object_ids(struct got_object_idset *idset,
 			obj_type = m->obj_type;
 		if (obj_type != GOT_OBJ_TYPE_TAG)
 			continue;
-		err = load_tag(0, idset, id, repo,
-		    loose_obj_only, cancel_cb, cancel_arg);
+		err = load_tag(0, idset, id, repo, loose_obj_only,
+		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
+		    cancel_cb, cancel_arg);
 		if (err)
 			goto done;
 	}
 
 	for (i = 0; i < nobj; i++) {
-		err = load_commit(1, idset, ids[i], repo,
-		    loose_obj_only, cancel_cb, cancel_arg);
+		err = load_commit(1, idset, ids[i], repo, loose_obj_only,
+		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
+		    cancel_cb, cancel_arg);
 		if (err)
 			goto done;
 	}
@@ -1406,8 +1474,9 @@ load_object_ids(struct got_object_idset *idset,
 			obj_type = m->obj_type;
 		if (obj_type != GOT_OBJ_TYPE_TAG)
 			continue;
-		err = load_tag(1, idset, id, repo,
-		    loose_obj_only, cancel_cb, cancel_arg);
+		err = load_tag(1, idset, id, repo, loose_obj_only,
+		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
+		    cancel_cb, cancel_arg);
 		if (err)
 			goto done;
 	}
@@ -1634,7 +1703,8 @@ static const struct got_error *
 genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delta_cache,
     struct got_pack_meta **deltify, int ndeltify,
     struct got_pack_meta **reuse, int nreuse,
-    int nours, struct got_repository *repo,
+    int ncolored, int nfound, int ntrees, int nours,
+    struct got_repository *repo,
     got_pack_progress_cb progress_cb, void *progress_arg,
     struct got_ratelimit *rl,
     got_cancel_cb cancel_cb, void *cancel_arg)
@@ -1666,8 +1736,8 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 	    write_order_cmp);
 	for (i = 0; i < ndeltify; i++) {
 		err = report_progress(progress_cb, progress_arg, rl,
-		    packfile_size, nours, ndeltify + nreuse,
-		    ndeltify + nreuse, i);
+		    ncolored, nfound, ntrees, packfile_size, nours,
+		    ndeltify + nreuse, ndeltify + nreuse, i);
 		if (err)
 			goto done;
 		m = deltify[i];
@@ -1681,8 +1751,8 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 	    reuse_write_order_cmp);
 	for (i = 0; i < nreuse; i++) {
 		err = report_progress(progress_cb, progress_arg, rl,
-		    packfile_size, nours, ndeltify + nreuse,
-		    ndeltify + nreuse, ndeltify + i);
+		    ncolored, nfound, ntrees, packfile_size, nours,
+		    ndeltify + nreuse, ndeltify + nreuse, ndeltify + i);
 		if (err)
 			goto done;
 		m = reuse[i];
@@ -1699,9 +1769,9 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
 	packfile_size += SHA1_DIGEST_LENGTH;
 	packfile_size += sizeof(struct got_packfile_hdr);
 	if (progress_cb) {
-		err = progress_cb(progress_arg, packfile_size, nours,
-		    ndeltify + nreuse, ndeltify + nreuse,
-		    ndeltify + nreuse);
+		err = progress_cb(progress_arg, ncolored, nfound, ntrees,
+		    packfile_size, nours, ndeltify + nreuse,
+		    ndeltify + nreuse, ndeltify + nreuse);
 		if (err)
 			goto done;
 	}
@@ -1758,6 +1828,7 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
 	struct got_object_idset *idset;
 	struct got_ratelimit rl;
 	struct got_pack_metavec deltify, reuse;
+	int ncolored = 0, nfound = 0, ntrees = 0;
 
 	memset(&deltify, 0, sizeof(deltify));
 	memset(&reuse, 0, sizeof(reuse));
@@ -1768,8 +1839,9 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
 	if (idset == NULL)
 		return got_error_from_errno("got_object_idset_alloc");
 
-	err = load_object_ids(idset, theirs, ntheirs, ours, nours,
-	    repo, loose_obj_only, cancel_cb, cancel_arg);
+	err = load_object_ids(&ncolored, &nfound, &ntrees, idset, theirs,
+	    ntheirs, ours, nours, repo, loose_obj_only,
+	    progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
 	if (err)
 		return err;
 
@@ -1779,8 +1851,8 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
 		goto done;
 
 	if (progress_cb) {
-		err = progress_cb(progress_arg, 0L, nours,
-		    got_object_idset_num_elements(idset), 0, 0);
+		err = progress_cb(progress_arg, ncolored, nfound, ntrees,
+		    0L, nours, got_object_idset_num_elements(idset), 0, 0);
 		if (err)
 			goto done;
 	}
@@ -1804,8 +1876,9 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
 		goto done;
 	}
 
-	err = search_deltas(&reuse, idset, delta_cache_fd, nours, repo,
-	    progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
+	err = search_deltas(&reuse, idset, delta_cache_fd, ncolored, nfound,
+	    ntrees, nours, repo, progress_cb, progress_arg, &rl,
+	    cancel_cb, cancel_arg);
 	if (err)
 		goto done;
 	if (reuse.nmeta > 0) {
@@ -1839,8 +1912,8 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
 	if (err)
 		goto done;
 	if (deltify.nmeta > 0) {
-		err = pick_deltas(deltify.meta, deltify.nmeta, nours,
-		    reuse.nmeta, delta_cache, repo,
+		err = pick_deltas(deltify.meta, deltify.nmeta, ncolored,
+		    nfound, ntrees, nours, reuse.nmeta, delta_cache, repo,
 		    progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
 		if (err)
 			goto done;
@@ -1851,8 +1924,9 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
 	}
 
 	err = genpack(packsha1, packfile, delta_cache, deltify.meta,
-	    deltify.nmeta, reuse.meta, reuse.nmeta, nours, repo,
-	    progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
+	    deltify.nmeta, reuse.meta, reuse.nmeta, ncolored, nfound, ntrees,
+	    nours, repo, progress_cb, progress_arg, &rl,
+	    cancel_cb, cancel_arg);
 	if (err)
 		goto done;
 done:
blob - 0576ad42d0f1ba289507bd309c4b24c1bb0f2dbb
blob + b8290d0d6137059bea61a547bbbf82d00672571d
--- lib/send.c
+++ lib/send.c
@@ -105,6 +105,9 @@ struct pack_progress_arg {
     got_send_progress_cb progress_cb;
     void *progress_arg;
 
+    int ncolored;
+    int nfound;
+    int ntrees;
     off_t packfile_size;
     int ncommits;
     int nobj_total;
@@ -113,17 +116,22 @@ struct pack_progress_arg {
 };
 
 static const struct got_error *
-pack_progress(void *arg, off_t packfile_size, int ncommits,
-    int nobj_total, int nobj_deltify, int nobj_written)
+pack_progress(void *arg, int ncolored, int nfound, int ntrees,
+    off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify,
+    int nobj_written)
 {
 	const struct got_error *err;
 	struct pack_progress_arg *a = arg;
 
-	err = a->progress_cb(a->progress_arg, packfile_size, ncommits,
-	    nobj_total, nobj_deltify, nobj_written, 0, NULL, 0);
+	err = a->progress_cb(a->progress_arg, ncolored, nfound, ntrees,
+	    packfile_size, ncommits, nobj_total, nobj_deltify,
+	    nobj_written, 0, NULL, 0);
 	if (err)
 		return err;
 
+	a->ncolored= ncolored;
+	a->nfound = nfound;
+	a->ntrees = ntrees;
 	a->packfile_size = packfile_size;
 	a->ncommits = ncommits;
 	a->nobj_total = nobj_total;
@@ -684,7 +692,8 @@ got_send_pack(const char *remote_name, struct got_path
 		}
 		if (refname != NULL ||
 		    bytes_sent_cur != bytes_sent) {
-			err = progress_cb(progress_arg, ppa.packfile_size,
+			err = progress_cb(progress_arg, ppa.ncolored,
+			    ppa.nfound, ppa.ntrees, ppa.packfile_size,
 			    ppa.ncommits, ppa.nobj_total, ppa.nobj_deltify,
 			    ppa.nobj_written, bytes_sent,
 			    refname, success);
blob - 597e601e0fefb3fbe64e77e4c05e3ae7103dd8b3
blob + 2ac3207e191fd1b940a418ffd18b2c0b82a89dff
--- regress/cmdline/pack.sh
+++ regress/cmdline/pack.sh
@@ -304,7 +304,7 @@ test_pack_ambiguous_arg() {
 	local commit1=`git_show_branch_head $testroot/repo mybranch`
 
 	gotadmin pack -r $testroot/repo -x master master \
-		> $testroot/stdout 2> $testroot/stderr
+		> /dev/null 2> $testroot/stderr
 	ret="$?"
 	if [ "$ret" = "0" ]; then
 		echo "gotadmin pack succeeded unexpectedly" >&2
@@ -312,15 +312,6 @@ test_pack_ambiguous_arg() {
 		return 1
 	fi
 
-	printf "\rpacking 1 reference\n" > $testroot/stdout.expected
-	cmp -s $testroot/stdout.expected $testroot/stdout
-	ret="$?"
-	if [ "$ret" != "0" ]; then
-		diff -u $testroot/stdout.expected $testroot/stdout
-		test_done "$testroot" "$ret"
-		return 1
-	fi
-
 	echo "gotadmin: not enough objects to pack" > $testroot/stderr.expected
 	cmp -s $testroot/stderr.expected $testroot/stderr
 	ret="$?"