Stefan Sperling <stsp@stsp.name> wrote:
> The initial steps of packing will usually complete relatively quickly,
> which is why no progress output was implemented for them.
> However, when creating huge packs in large repositories (such as running
> pack -a in openbsd src.git) the initial steps can take a couple of minutes
> of uncomfortable silence to get through.
> 
> This patch implements progress output during these steps, which appears
> on a seperate line (the first line printed in the example below):
> 
> $ gotadmin pack main
> 4066 commits colored; 25233 objects found; 4485 trees scanned
> packing 1 reference; 606 objects; deltify: 100%; writing pack:  710K 100%
> Wrote 0e79c0a9ea1bdde529b6ed96b37f534e45bda446.pack
>  710K packed; indexing 100%; resolving deltas 100%
> Indexed 0e79c0a9ea1bdde529b6ed96b37f534e45bda446.pack
> $
> 
> One test was trying to check progress output against known content.
> Because the exact output depends on timing, just pipe it to /dev/null.
> Only check stderr which has everything the test wants to know.
> 
> Maybe it would have been smart to put all progress-related parameters into
> a struct instead of passing individual arguments around. But that can be
> fixed later. I have kept the current idiom for consistency.
> 
> ok?

sure, it's nice to have some progress output, especially when it can
take a while on bigger repositories.

> diff c0c2b3d1e2c80fe9ee36670cff033f948cca9fd6 7f6c6d86bb14667458ebd9f9272d6efd7b2b73b9
> blob - 09c52c1e9bd95ceb4f5fa1f135b3fb570b7285b0
> blob + 3a6211928791cdfffe4e708f6f6145429f476822
> --- got/got.c
> +++ got/got.c
> @@ -7852,9 +7852,32 @@ usage_send(void)
>  	exit(1);
>  }
>  
> +static void
> +print_load_info(int print_colored, int print_found, int print_trees,
> +    int ncolored, int nfound, int ntrees)
> +{
> +	if (print_colored) {
> +		printf("%d commit%s colored", ncolored,
> +		    ncolored == 1 ? "" : "s");
> +	}
> +	if (print_found) {
> +		printf("%s%d object%s found",
> +		    ncolored > 0 ? "; " : "",
> +		    nfound, nfound == 1 ? "" : "s");
> +	}
> +	if (print_trees) {
> +		printf("; %d tree%s scanned", ntrees,
> +		    ntrees == 1 ? "" : "s");
> +	}
> +}
> +
>  struct got_send_progress_arg {
>  	char last_scaled_packsize[FMT_SCALED_STRSIZE];
>  	int verbosity;
> +	int last_ncolored;
> +	int last_nfound;
> +	int last_ntrees;
> +	int loading_done;
>  	int last_ncommits;
>  	int last_nobj_total;
>  	int last_p_deltify;
> @@ -7866,14 +7889,15 @@ struct got_send_progress_arg {
>  };
>  
>  static const struct got_error *
> -send_progress(void *arg, off_t packfile_size, int ncommits, int nobj_total,
> -    int nobj_deltify, int nobj_written, off_t bytes_sent, const char *refname,
> -    int success)
> +send_progress(void *arg, int ncolored, int nfound, int ntrees,
> +    off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify,
> +    int nobj_written, off_t bytes_sent, const char *refname, int success)
>  {
>  	struct got_send_progress_arg *a = arg;
>  	char scaled_packsize[FMT_SCALED_STRSIZE];
>  	char scaled_sent[FMT_SCALED_STRSIZE];
>  	int p_deltify = 0, p_written = 0, p_sent = 0;
> +	int print_colored = 0, print_found = 0, print_trees = 0;
>  	int print_searching = 0, print_total = 0;
>  	int print_deltify = 0, print_written = 0, print_sent = 0;
>  
> @@ -7903,6 +7927,39 @@ send_progress(void *arg, off_t packfile_size, int ncom
>  		return NULL;
>  	}
>  
> +	if (a->last_ncolored != ncolored) {
> +		print_colored = 1;
> +		a->last_ncolored = ncolored;
> +	}
> +
> +	if (a->last_nfound != nfound) {
> +		print_colored = 1;
> +		print_found = 1;
> +		a->last_nfound = nfound;
> +	}
> +
> +	if (a->last_ntrees != ntrees) {
> +		print_colored = 1;
> +		print_found = 1;
> +		print_trees = 1;
> +		a->last_ntrees = ntrees;
> +	}
> +
> +	if ((print_colored || print_found || print_trees) &&
> +	    !a->loading_done) {
> +		printf("\r");
> +		print_load_info(print_colored, print_found, print_trees,
> +		    ncolored, nfound, ntrees);
> +		a->printed_something = 1;
> +		fflush(stdout);
> +		return NULL;
> +	} else if (!a->loading_done) {
> +		printf("\r");
> +		print_load_info(1, 1, 1, ncolored, nfound, ntrees);
> +		printf("\n");
> +		a->loading_done = 1;
> +	}
> +
>  	if (fmt_scaled(packfile_size, scaled_packsize) == -1)
>  		return got_error_from_errno("fmt_scaled");
>  	if (fmt_scaled(bytes_sent, scaled_sent) == -1)
> blob - bd7343dcca754f398aaba30df74ecd27b90d2ee7
> blob + 09d1ca1db98e99e774fa86e99f0212c6871c7fcb
> --- gotadmin/gotadmin.c
> +++ gotadmin/gotadmin.c
> @@ -381,6 +381,10 @@ usage_pack(void)
>  
>  struct got_pack_progress_arg {
>  	char last_scaled_size[FMT_SCALED_STRSIZE];
> +	int last_ncolored;
> +	int last_nfound;
> +	int last_ntrees;
> +	int loading_done;
>  	int last_ncommits;
>  	int last_nobj_total;
>  	int last_p_deltify;
> @@ -391,19 +395,73 @@ struct got_pack_progress_arg {
>  	int printed_something;
>  };
>  
> +static void
> +print_load_info(int print_colored, int print_found, int print_trees,
> +    int ncolored, int nfound, int ntrees)
> +{
> +	if (print_colored) {
> +		printf("%d commit%s colored", ncolored,
> +		    ncolored == 1 ? "" : "s");
> +	}
> +	if (print_found) {
> +		printf("%s%d object%s found",
> +		    ncolored > 0 ? "; " : "",
> +		    nfound, nfound == 1 ? "" : "s");
> +	}
> +	if (print_trees) {
> +		printf("; %d tree%s scanned", ntrees,
> +		    ntrees == 1 ? "" : "s");
> +	}
> +}
> +
>  static const struct got_error *
> -pack_progress(void *arg, off_t packfile_size, int ncommits,
> -    int nobj_total, int nobj_deltify, int nobj_written)
> +pack_progress(void *arg, int ncolored, int nfound, int ntrees,
> +    off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify,
> +    int nobj_written)
>  {
>  	struct got_pack_progress_arg *a = arg;
>  	char scaled_size[FMT_SCALED_STRSIZE];
>  	int p_deltify, p_written;
> +	int print_colored = 0, print_found = 0, print_trees = 0;
>  	int print_searching = 0, print_total = 0;
>  	int print_deltify = 0, print_written = 0;
>  
>  	if (a->verbosity < 0)
>  		return NULL;
>  
> +	if (a->last_ncolored != ncolored) {
> +		print_colored = 1;
> +		a->last_ncolored = ncolored;
> +	}
> +
> +	if (a->last_nfound != nfound) {
> +		print_colored = 1;
> +		print_found = 1;
> +		a->last_nfound = nfound;
> +	}
> +
> +	if (a->last_ntrees != ntrees) {
> +		print_colored = 1;
> +		print_found = 1;
> +		print_trees = 1;
> +		a->last_ntrees = ntrees;
> +	}
> +
> +	if ((print_colored || print_found || print_trees) &&
> +	    !a->loading_done) {
> +		printf("\r");
> +		print_load_info(print_colored, print_found, print_trees,
> +		    ncolored, nfound, ntrees);
> +		a->printed_something = 1;
> +		fflush(stdout);
> +		return NULL;
> +	} else if (!a->loading_done) {
> +		printf("\r");
> +		print_load_info(1, 1, 1, ncolored, nfound, ntrees);
> +		printf("\n");
> +		a->loading_done = 1;
> +	}
> +
>  	if (fmt_scaled(packfile_size, scaled_size) == -1)
>  		return got_error_from_errno("fmt_scaled");
>  
> blob - 8bc7ea591673f80b94917ac36991936b0fc46038
> blob + 2caf2a94d4ed262a215863f07c9e967a8f052cc3
> --- include/got_repository_admin.h
> +++ include/got_repository_admin.h
> @@ -16,8 +16,8 @@
>  
>  /* A callback function which gets invoked with progress information to print. */
>  typedef const struct got_error *(*got_pack_progress_cb)(void *arg,
> -    off_t packfile_size, int ncommits, int nobj_total, int obj_deltify,
> -    int nobj_written);
> +    int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits,
> +    int nobj_total, int obj_deltify, int nobj_written);
>  
>  /*
>   * Attempt to pack objects reachable via 'include_refs' into a new packfile.
> blob - 225bc42faab244492a77e967c829b9cf47301009
> blob + 19e9e4f116e70c354ec4bbff8ce7e5a1615acd1f
> --- include/got_send.h
> +++ include/got_send.h
> @@ -36,8 +36,8 @@ const struct got_error *got_send_connect(pid_t *, int 
>  
>  /* A callback function which gets invoked with progress information to print. */
>  typedef const struct got_error *(*got_send_progress_cb)(void *,
> -    off_t packfile_size, int ncommits, int nobj_total,
> -    int nobj_deltify, int nobj_written, off_t bytes_sent,
> +    int ncolored, int nfound, int ntrees, off_t packfile_size, int ncommits,
> +    int nobj_total, int nobj_deltify, int nobj_written, off_t bytes_sent,
>      const char *refname, int success);
>  
>  /*
> blob - 3b6af380cd3ae1ac103c402a5365107c232529dd
> blob + 598a24d3ed3f054d8c65d0ab83d32f2db020856c
> --- lib/pack_create.c
> +++ lib/pack_create.c
> @@ -408,8 +408,9 @@ encode_delta(struct got_pack_meta *m, struct got_raw_o
>  
>  static const struct got_error *
>  report_progress(got_pack_progress_cb progress_cb, void *progress_arg,
> -    struct got_ratelimit *rl, off_t packfile_size, int ncommits,
> -    int nobj_total, int obj_deltify, int nobj_written)
> +    struct got_ratelimit *rl, int ncolored, int nfound, int ntrees,
> +    off_t packfile_size, int ncommits, int nobj_total, int obj_deltify,
> +    int nobj_written)
>  {
>  	const struct got_error *err;
>  	int elapsed;
> @@ -421,8 +422,8 @@ report_progress(got_pack_progress_cb progress_cb, void
>  	if (err || !elapsed)
>  		return err;
>  
> -	return progress_cb(progress_arg, packfile_size, ncommits,
> -	    nobj_total, obj_deltify, nobj_written);
> +	return progress_cb(progress_arg, ncolored, nfound, ntrees,
> +	    packfile_size, ncommits, nobj_total, obj_deltify, nobj_written);
>  }
>  
>  static const struct got_error *
> @@ -540,6 +541,9 @@ struct search_deltas_arg {
>  	struct got_ratelimit *rl;
>  	got_cancel_cb cancel_cb;
>  	void *cancel_arg;
> +	int ncolored;
> +	int nfound;
> +	int ntrees;
>  	int ncommits;
>  };
>  
> @@ -585,8 +589,8 @@ search_delta_for_object(struct got_object_id *id, void
>  		if (err)
>  			goto done;
>  		err = report_progress(a->progress_cb, a->progress_arg, a->rl,
> -		    0L, a->ncommits, got_object_idset_num_elements(a->idset),
> -		    a->v->nmeta, 0);
> +		    a->ncolored, a->nfound, a->ntrees, 0L, a->ncommits,
> +		    got_object_idset_num_elements(a->idset), a->v->nmeta, 0);
>  	}
>  done:
>  	got_object_close(obj);
> @@ -595,7 +599,8 @@ done:
>  
>  static const struct got_error *
>  search_deltas(struct got_pack_metavec *v, struct got_object_idset *idset,
> -    int delta_cache_fd, int ncommits, struct got_repository *repo,
> +    int delta_cache_fd, int ncolored, int nfound, int ntrees, int ncommits,
> +    struct got_repository *repo,
>      got_pack_progress_cb progress_cb, void *progress_arg,
>      struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
>  {
> @@ -635,6 +640,9 @@ search_deltas(struct got_pack_metavec *v, struct got_o
>  	sda.rl = rl;
>  	sda.cancel_cb = cancel_cb;
>  	sda.cancel_arg = cancel_arg;
> +	sda.ncolored = ncolored;
> +	sda.nfound = nfound;
> +	sda.ntrees = ntrees;
>  	sda.ncommits = ncommits;
>  	err = got_object_idset_for_each(idset, search_delta_for_object, &sda);
>  done:
> @@ -643,8 +651,9 @@ done:
>  }
>  
>  static const struct got_error *
> -pick_deltas(struct got_pack_meta **meta, int nmeta, int ncommits,
> -    int nreused, FILE *delta_cache, struct got_repository *repo,
> +pick_deltas(struct got_pack_meta **meta, int nmeta, int ncolored,
> +    int nfound, int ntrees, int ncommits, int nreused, FILE *delta_cache,
> +    struct got_repository *repo,
>      got_pack_progress_cb progress_cb, void *progress_arg,
>      struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
>  {
> @@ -667,7 +676,8 @@ pick_deltas(struct got_pack_meta **meta, int nmeta, in
>  				break;
>  		}
>  		err = report_progress(progress_cb, progress_arg, rl,
> -		    0L, ncommits, nreused + nmeta, nreused + i, 0);
> +		    ncolored, nfound, ntrees, 0L, ncommits, nreused + nmeta,
> +		    nreused + i, 0);
>  		if (err)
>  			goto done;
>  		m = meta[i];
> @@ -877,7 +887,9 @@ static const struct got_error *
>  load_tree_entries(struct got_object_id_queue *ids, int want_meta,
>      struct got_object_idset *idset, struct got_object_id *tree_id,
>      const char *dpath, time_t mtime, struct got_repository *repo,
> -    int loose_obj_only, got_cancel_cb cancel_cb, void *cancel_arg)
> +    int loose_obj_only, int *ncolored, int *nfound, int *ntrees,
> +    got_pack_progress_cb progress_cb, void *progress_arg,
> +    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
>  {
>  	const struct got_error *err;
>  	struct got_tree_object *tree;
> @@ -888,6 +900,12 @@ load_tree_entries(struct got_object_id_queue *ids, int
>  	if (err)
>  		return err;
>  
> +	(*ntrees)++;
> +	err = report_progress(progress_cb, progress_arg, rl,
> +	    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
> +	if (err)
> +		return err;
> +
>  	for (i = 0; i < got_object_tree_get_nentries(tree); i++) {
>  		struct got_tree_entry *e = got_object_tree_get_entry(tree, i);
>  		struct got_object_id *id = got_tree_entry_get_id(e);
> @@ -920,6 +938,11 @@ load_tree_entries(struct got_object_id_queue *ids, int
>  			    GOT_OBJ_TYPE_BLOB, mtime, loose_obj_only, repo);
>  			if (err)
>  				break;
> +			(*nfound)++;
> +			err = report_progress(progress_cb, progress_arg, rl,
> +			    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
> +			if (err)
> +				break;
>  		}
>  		free(p);
>  		p = NULL;
> @@ -933,8 +956,10 @@ load_tree_entries(struct got_object_id_queue *ids, int
>  static const struct got_error *
>  load_tree(int want_meta, struct got_object_idset *idset,
>      struct got_object_id *tree_id, const char *dpath, time_t mtime,
> -    int loose_obj_only, struct got_repository *repo,
> -    got_cancel_cb cancel_cb, void *cancel_arg)
> +    struct got_repository *repo, int loose_obj_only,
> +    int *ncolored, int *nfound, int *ntrees,
> +    got_pack_progress_cb progress_cb, void *progress_arg,
> +    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
>  {
>  	const struct got_error *err = NULL;
>  	struct got_object_id_queue tree_ids;
> @@ -972,8 +997,16 @@ load_tree(int want_meta, struct got_object_idset *idse
>  			break;
>  		}
>  
> +		(*nfound)++;
> +		err = report_progress(progress_cb, progress_arg, rl,
> +		    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
> +		if (err)
> +			break;
> +
>  		err = load_tree_entries(&tree_ids, want_meta, idset, qid->id,
> -		    dpath, mtime, repo, loose_obj_only, cancel_cb, cancel_arg);
> +		    dpath, mtime, repo, loose_obj_only, ncolored, nfound,
> +		    ntrees, progress_cb, progress_arg, rl,
> +		    cancel_cb, cancel_arg);
>  		got_object_qid_free(qid);
>  		if (err)
>  			break;
> @@ -986,7 +1019,9 @@ load_tree(int want_meta, struct got_object_idset *idse
>  static const struct got_error *
>  load_commit(int want_meta, struct got_object_idset *idset,
>      struct got_object_id *id, struct got_repository *repo, int loose_obj_only,
> -    got_cancel_cb cancel_cb, void *cancel_arg)
> +    int *ncolored, int *nfound, int *ntrees,
> +    got_pack_progress_cb progress_cb, void *progress_arg,
> +    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
>  {
>  	const struct got_error *err;
>  	struct got_commit_object *commit;
> @@ -1013,9 +1048,16 @@ load_commit(int want_meta, struct got_object_idset *id
>  	if (err)
>  		goto done;
>  
> +	(*nfound)++;
> +	err = report_progress(progress_cb, progress_arg, rl,
> +	    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
> +	if (err)
> +		goto done;
> +
>  	err = load_tree(want_meta, idset, got_object_commit_get_tree_id(commit),
>  	    "", got_object_commit_get_committer_time(commit),
> -	    loose_obj_only, repo, cancel_cb, cancel_arg);
> +	    repo, loose_obj_only, ncolored, nfound, ntrees,
> +	    progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
>  done:
>  	got_object_commit_close(commit);
>  	return err;
> @@ -1024,7 +1066,9 @@ done:
>  static const struct got_error *
>  load_tag(int want_meta, struct got_object_idset *idset,
>      struct got_object_id *id, struct got_repository *repo, int loose_obj_only,
> -    got_cancel_cb cancel_cb, void *cancel_arg)
> +    int *ncolored, int *nfound, int *ntrees,
> +    got_pack_progress_cb progress_cb, void *progress_arg,
> +    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
>  {
>  	const struct got_error *err;
>  	struct got_tag_object *tag = NULL;
> @@ -1051,17 +1095,25 @@ load_tag(int want_meta, struct got_object_idset *idset
>  	if (err)
>  		goto done;
>  
> +	(*nfound)++;
> +	err = report_progress(progress_cb, progress_arg, rl,
> +	    *ncolored, *nfound, *ntrees, 0L, 0, 0, 0, 0);
> +	if (err)
> +		goto done;
> +
>  	switch (got_object_tag_get_object_type(tag)) {
>  	case GOT_OBJ_TYPE_COMMIT:
>  		err = load_commit(want_meta, idset,
> -		    got_object_tag_get_object_id(tag), repo,
> -		    loose_obj_only, cancel_cb, cancel_arg);
> +		    got_object_tag_get_object_id(tag), repo, loose_obj_only,
> +		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
> +		    cancel_cb, cancel_arg);
>  		break;
>  	case GOT_OBJ_TYPE_TREE:
>  		err = load_tree(want_meta, idset,
>  		    got_object_tag_get_object_id(tag), "",
> -		    got_object_tag_get_tagger_time(tag),
> -		    loose_obj_only, repo, cancel_cb, cancel_arg);
> +		    got_object_tag_get_tagger_time(tag), repo, loose_obj_only,
> +		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
> +		    cancel_cb, cancel_arg);
>  		break;
>  	default:
>  		break;
> @@ -1183,11 +1235,12 @@ append_id(struct got_object_id *id, void *data, void *
>  }
>  
>  static const struct got_error *
> -findtwixt(struct got_object_id ***res, int *nres,
> +findtwixt(struct got_object_id ***res, int *nres, int *ncolored,
>      struct got_object_id **head, int nhead,
>      struct got_object_id **tail, int ntail,
>      struct got_repository *repo,
> -    got_cancel_cb cancel_cb, void *cancel_arg)
> +    got_pack_progress_cb progress_cb, void *progress_arg,
> +    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
>  {
>  	const struct got_error *err = NULL;
>  	struct got_object_id_queue ids;
> @@ -1198,6 +1251,7 @@ findtwixt(struct got_object_id ***res, int *nres,
>  	STAILQ_INIT(&ids);
>  	*res = NULL;
>  	*nres = 0;
> +	*ncolored = 0;
>  
>  	keep = got_object_idset_alloc();
>  	if (keep == NULL)
> @@ -1248,6 +1302,12 @@ findtwixt(struct got_object_id ***res, int *nres,
>  		else
>  			ncolor = COLOR_BLANK;
>  
> +		(*ncolored)++;
> +		err = report_progress(progress_cb, progress_arg, rl,
> +		    *ncolored, 0, 0, 0L, 0, 0, 0, 0);
> +		if (err)
> +			goto done;
> +
>  		if (ncolor == COLOR_DROP || (ncolor == COLOR_KEEP &&
>  		    qcolor == COLOR_KEEP)) {
>  			STAILQ_REMOVE_HEAD(&ids, entry);
> @@ -1336,17 +1396,22 @@ done:
>  }
>  
>  static const struct got_error *
> -load_object_ids(struct got_object_idset *idset,
> -    struct got_object_id **theirs, int ntheirs,
> +load_object_ids(int *ncolored, int *nfound, int *ntrees,
> +    struct got_object_idset *idset, struct got_object_id **theirs, int ntheirs,
>      struct got_object_id **ours, int nours, struct got_repository *repo,
> -    int loose_obj_only, got_cancel_cb cancel_cb, void *cancel_arg)
> +    int loose_obj_only, got_pack_progress_cb progress_cb, void *progress_arg,
> +    struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
>  {
>  	const struct got_error *err = NULL;
>  	struct got_object_id **ids = NULL;
>  	int i, nobj = 0, obj_type;
>  
> -	err = findtwixt(&ids, &nobj, ours, nours, theirs, ntheirs, repo,
> -	    cancel_cb, cancel_arg);
> +	*ncolored = 0;
> +	*nfound = 0;
> +	*ntrees = 0;
> +
> +	err = findtwixt(&ids, &nobj, ncolored, ours, nours, theirs, ntheirs,
> +	    repo, progress_cb, progress_arg, rl, cancel_cb, cancel_arg);
>  	if (err || nobj == 0)
>  		goto done;
>  
> @@ -1359,8 +1424,9 @@ load_object_ids(struct got_object_idset *idset,
>  			return err;
>  		if (obj_type != GOT_OBJ_TYPE_COMMIT)
>  			continue;
> -		err = load_commit(0, idset, id, repo,
> -		    loose_obj_only, cancel_cb, cancel_arg);
> +		err = load_commit(0, idset, id, repo, loose_obj_only,
> +		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
> +		    cancel_cb, cancel_arg);
>  		if (err)
>  			goto done;
>  	}
> @@ -1379,15 +1445,17 @@ load_object_ids(struct got_object_idset *idset,
>  			obj_type = m->obj_type;
>  		if (obj_type != GOT_OBJ_TYPE_TAG)
>  			continue;
> -		err = load_tag(0, idset, id, repo,
> -		    loose_obj_only, cancel_cb, cancel_arg);
> +		err = load_tag(0, idset, id, repo, loose_obj_only,
> +		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
> +		    cancel_cb, cancel_arg);
>  		if (err)
>  			goto done;
>  	}
>  
>  	for (i = 0; i < nobj; i++) {
> -		err = load_commit(1, idset, ids[i], repo,
> -		    loose_obj_only, cancel_cb, cancel_arg);
> +		err = load_commit(1, idset, ids[i], repo, loose_obj_only,
> +		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
> +		    cancel_cb, cancel_arg);
>  		if (err)
>  			goto done;
>  	}
> @@ -1406,8 +1474,9 @@ load_object_ids(struct got_object_idset *idset,
>  			obj_type = m->obj_type;
>  		if (obj_type != GOT_OBJ_TYPE_TAG)
>  			continue;
> -		err = load_tag(1, idset, id, repo,
> -		    loose_obj_only, cancel_cb, cancel_arg);
> +		err = load_tag(1, idset, id, repo, loose_obj_only,
> +		    ncolored, nfound, ntrees, progress_cb, progress_arg, rl,
> +		    cancel_cb, cancel_arg);
>  		if (err)
>  			goto done;
>  	}
> @@ -1634,7 +1703,8 @@ static const struct got_error *
>  genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delta_cache,
>      struct got_pack_meta **deltify, int ndeltify,
>      struct got_pack_meta **reuse, int nreuse,
> -    int nours, struct got_repository *repo,
> +    int ncolored, int nfound, int ntrees, int nours,
> +    struct got_repository *repo,
>      got_pack_progress_cb progress_cb, void *progress_arg,
>      struct got_ratelimit *rl,
>      got_cancel_cb cancel_cb, void *cancel_arg)
> @@ -1666,8 +1736,8 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
>  	    write_order_cmp);
>  	for (i = 0; i < ndeltify; i++) {
>  		err = report_progress(progress_cb, progress_arg, rl,
> -		    packfile_size, nours, ndeltify + nreuse,
> -		    ndeltify + nreuse, i);
> +		    ncolored, nfound, ntrees, packfile_size, nours,
> +		    ndeltify + nreuse, ndeltify + nreuse, i);
>  		if (err)
>  			goto done;
>  		m = deltify[i];
> @@ -1681,8 +1751,8 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
>  	    reuse_write_order_cmp);
>  	for (i = 0; i < nreuse; i++) {
>  		err = report_progress(progress_cb, progress_arg, rl,
> -		    packfile_size, nours, ndeltify + nreuse,
> -		    ndeltify + nreuse, ndeltify + i);
> +		    ncolored, nfound, ntrees, packfile_size, nours,
> +		    ndeltify + nreuse, ndeltify + nreuse, ndeltify + i);
>  		if (err)
>  			goto done;
>  		m = reuse[i];
> @@ -1699,9 +1769,9 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delt
>  	packfile_size += SHA1_DIGEST_LENGTH;
>  	packfile_size += sizeof(struct got_packfile_hdr);
>  	if (progress_cb) {
> -		err = progress_cb(progress_arg, packfile_size, nours,
> -		    ndeltify + nreuse, ndeltify + nreuse,
> -		    ndeltify + nreuse);
> +		err = progress_cb(progress_arg, ncolored, nfound, ntrees,
> +		    packfile_size, nours, ndeltify + nreuse,
> +		    ndeltify + nreuse, ndeltify + nreuse);
>  		if (err)
>  			goto done;
>  	}
> @@ -1758,6 +1828,7 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
>  	struct got_object_idset *idset;
>  	struct got_ratelimit rl;
>  	struct got_pack_metavec deltify, reuse;
> +	int ncolored = 0, nfound = 0, ntrees = 0;
>  
>  	memset(&deltify, 0, sizeof(deltify));
>  	memset(&reuse, 0, sizeof(reuse));
> @@ -1768,8 +1839,9 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
>  	if (idset == NULL)
>  		return got_error_from_errno("got_object_idset_alloc");
>  
> -	err = load_object_ids(idset, theirs, ntheirs, ours, nours,
> -	    repo, loose_obj_only, cancel_cb, cancel_arg);
> +	err = load_object_ids(&ncolored, &nfound, &ntrees, idset, theirs,
> +	    ntheirs, ours, nours, repo, loose_obj_only,
> +	    progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
>  	if (err)
>  		return err;
>  
> @@ -1779,8 +1851,8 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
>  		goto done;
>  
>  	if (progress_cb) {
> -		err = progress_cb(progress_arg, 0L, nours,
> -		    got_object_idset_num_elements(idset), 0, 0);
> +		err = progress_cb(progress_arg, ncolored, nfound, ntrees,
> +		    0L, nours, got_object_idset_num_elements(idset), 0, 0);
>  		if (err)
>  			goto done;
>  	}
> @@ -1804,8 +1876,9 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
>  		goto done;
>  	}
>  
> -	err = search_deltas(&reuse, idset, delta_cache_fd, nours, repo,
> -	    progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
> +	err = search_deltas(&reuse, idset, delta_cache_fd, ncolored, nfound,
> +	    ntrees, nours, repo, progress_cb, progress_arg, &rl,
> +	    cancel_cb, cancel_arg);
>  	if (err)
>  		goto done;
>  	if (reuse.nmeta > 0) {
> @@ -1839,8 +1912,8 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
>  	if (err)
>  		goto done;
>  	if (deltify.nmeta > 0) {
> -		err = pick_deltas(deltify.meta, deltify.nmeta, nours,
> -		    reuse.nmeta, delta_cache, repo,
> +		err = pick_deltas(deltify.meta, deltify.nmeta, ncolored,
> +		    nfound, ntrees, nours, reuse.nmeta, delta_cache, repo,
>  		    progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
>  		if (err)
>  			goto done;
> @@ -1851,8 +1924,9 @@ got_pack_create(uint8_t *packsha1, FILE *packfile,
>  	}
>  
>  	err = genpack(packsha1, packfile, delta_cache, deltify.meta,
> -	    deltify.nmeta, reuse.meta, reuse.nmeta, nours, repo,
> -	    progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
> +	    deltify.nmeta, reuse.meta, reuse.nmeta, ncolored, nfound, ntrees,
> +	    nours, repo, progress_cb, progress_arg, &rl,
> +	    cancel_cb, cancel_arg);
>  	if (err)
>  		goto done;
>  done:
> blob - 0576ad42d0f1ba289507bd309c4b24c1bb0f2dbb
> blob + b8290d0d6137059bea61a547bbbf82d00672571d
> --- lib/send.c
> +++ lib/send.c
> @@ -105,6 +105,9 @@ struct pack_progress_arg {
>      got_send_progress_cb progress_cb;
>      void *progress_arg;
>  
> +    int ncolored;
> +    int nfound;
> +    int ntrees;
>      off_t packfile_size;
>      int ncommits;
>      int nobj_total;
> @@ -113,17 +116,22 @@ struct pack_progress_arg {
>  };
>  
>  static const struct got_error *
> -pack_progress(void *arg, off_t packfile_size, int ncommits,
> -    int nobj_total, int nobj_deltify, int nobj_written)
> +pack_progress(void *arg, int ncolored, int nfound, int ntrees,
> +    off_t packfile_size, int ncommits, int nobj_total, int nobj_deltify,
> +    int nobj_written)
>  {
>  	const struct got_error *err;
>  	struct pack_progress_arg *a = arg;
>  
> -	err = a->progress_cb(a->progress_arg, packfile_size, ncommits,
> -	    nobj_total, nobj_deltify, nobj_written, 0, NULL, 0);
> +	err = a->progress_cb(a->progress_arg, ncolored, nfound, ntrees,
> +	    packfile_size, ncommits, nobj_total, nobj_deltify,
> +	    nobj_written, 0, NULL, 0);
>  	if (err)
>  		return err;
>  
> +	a->ncolored= ncolored;
> +	a->nfound = nfound;
> +	a->ntrees = ntrees;
>  	a->packfile_size = packfile_size;
>  	a->ncommits = ncommits;
>  	a->nobj_total = nobj_total;
> @@ -684,7 +692,8 @@ got_send_pack(const char *remote_name, struct got_path
>  		}
>  		if (refname != NULL ||
>  		    bytes_sent_cur != bytes_sent) {
> -			err = progress_cb(progress_arg, ppa.packfile_size,
> +			err = progress_cb(progress_arg, ppa.ncolored,
> +			    ppa.nfound, ppa.ntrees, ppa.packfile_size,
>  			    ppa.ncommits, ppa.nobj_total, ppa.nobj_deltify,
>  			    ppa.nobj_written, bytes_sent,
>  			    refname, success);
> blob - 597e601e0fefb3fbe64e77e4c05e3ae7103dd8b3
> blob + 2ac3207e191fd1b940a418ffd18b2c0b82a89dff
> --- regress/cmdline/pack.sh
> +++ regress/cmdline/pack.sh
> @@ -304,7 +304,7 @@ test_pack_ambiguous_arg() {
>  	local commit1=`git_show_branch_head $testroot/repo mybranch`
>  
>  	gotadmin pack -r $testroot/repo -x master master \
> -		> $testroot/stdout 2> $testroot/stderr
> +		> /dev/null 2> $testroot/stderr
>  	ret="$?"
>  	if [ "$ret" = "0" ]; then
>  		echo "gotadmin pack succeeded unexpectedly" >&2
> @@ -312,15 +312,6 @@ test_pack_ambiguous_arg() {
>  		return 1
>  	fi
>  
> -	printf "\rpacking 1 reference\n" > $testroot/stdout.expected
> -	cmp -s $testroot/stdout.expected $testroot/stdout
> -	ret="$?"
> -	if [ "$ret" != "0" ]; then
> -		diff -u $testroot/stdout.expected $testroot/stdout
> -		test_done "$testroot" "$ret"
> -		return 1
> -	fi
> -
>  	echo "gotadmin: not enough objects to pack" > $testroot/stderr.expected
>  	cmp -s $testroot/stderr.expected $testroot/stderr
>  	ret="$?"