"GOT", but the "O" is a cute, smiling pufferfish. Index | Thread | Search

From:
Stefan Sperling <stsp@stsp.name>
Subject:
quiet regress test output by default
To:
gameoftrees@openbsd.org
Date:
Tue, 4 Aug 2020 13:30:59 +0200

Download raw body.

Thread 
A lot of tests have accumulated over time and because each test name is
printed by 'make regress' it has become a bit hard to judge test results
at a glance.

The patch below adds a -q option to all test programs and scripts and
uses this to make the output of 'make regress' quiet by default.
When tests are run individually the current default output remains.

A full test run now looks as shown below:

$ make regress
===> regress
===> regress/cmdline
./checkout.sh -q
./update.sh -q
./status.sh -q
./log.sh -q
./add.sh -q
./rm.sh -q
./diff.sh -q
./blame.sh -q
./branch.sh -q
./tag.sh -q
./ref.sh -q
./commit.sh -q
./revert.sh -q
./cherrypick.sh -q
./backout.sh -q
./rebase.sh -q
./import.sh -q
./histedit.sh -q
./integrate.sh -q
./stage.sh -q
./unstage.sh -q
./cat.sh -q
./clone.sh -q
./fetch.sh -q
./tree.sh -q
===> regress/delta
/home/stsp/src/got/regress/delta/obj/delta_test -q
===> regress/idset
/home/stsp/src/got/regress/idset/obj/idset_test -q
===> regress/path
/home/stsp/src/got/regress/path/obj/path_test -q
===> regress/fetch
/home/stsp/src/got/regress/fetch/obj/fetch_test -q
$

A test run with an artificial test failure looks like this:

$ make regress
===> regress
===> regress/cmdline
./checkout.sh -q
./update.sh -q
./status.sh -q
./log.sh -q
./add.sh -q
./rm.sh -q
./diff.sh -q
./blame.sh -q
./branch.sh -q
./tag.sh -q
./ref.sh -q
./commit.sh -q
./revert.sh -q
./cherrypick.sh -q
./backout.sh -q
./rebase.sh -q
./import.sh -q
./histedit.sh -q
./integrate.sh -q
./stage.sh -q
./unstage.sh -q
./cat.sh -q
./clone.sh -q
./fetch.sh -q
./tree.sh -q
--- /tmp/got-test-tree_branch-7IqP6M02/stdout.expected  Tue Aug  4 13:27:18 2020
+++ /tmp/got-test-tree_branch-7IqP6M02/stdout   Tue Aug  4 13:27:18 2020
@@ -1,4 +1,4 @@
-alfa
+alpha
 beta
 epsilon/
 foo
test failed; leaving test data in /tmp/got-test-tree_branch-7IqP6M02
===> regress/delta
/home/stsp/src/got/regress/delta/obj/delta_test -q
===> regress/idset
/home/stsp/src/got/regress/idset/obj/idset_test -q
===> regress/path
/home/stsp/src/got/regress/path/obj/path_test -q
===> regress/fetch
/home/stsp/src/got/regress/fetch/obj/fetch_test -q
$

ok?

diff 6d7260fd5af3b577d020bdbcb038b6c245216d5e /home/stsp/src/got
blob - 0f29c0ec552874fbfbdf266c710410d5726598aa
file + regress/cmdline/Makefile
--- regress/cmdline/Makefile
+++ regress/cmdline/Makefile
@@ -4,78 +4,78 @@ REGRESS_TARGETS=checkout update status log add rm diff
 NOOBJ=Yes
 
 checkout:
-	./checkout.sh
+	./checkout.sh -q
 
 update:
-	./update.sh
+	./update.sh -q
 
 status:
-	./status.sh
+	./status.sh -q
 
 log:
-	./log.sh
+	./log.sh -q
 
 add:
-	./add.sh
+	./add.sh -q
 
 rm:
-	./rm.sh
+	./rm.sh -q
 
 diff:
-	./diff.sh
+	./diff.sh -q
 
 blame:
-	./blame.sh
+	./blame.sh -q
 
 branch:
-	./branch.sh
+	./branch.sh -q
 
 tag:
-	./tag.sh
+	./tag.sh -q
 
 ref:
-	./ref.sh
+	./ref.sh -q
 
 commit:
-	./commit.sh
+	./commit.sh -q
 
 revert:
-	./revert.sh
+	./revert.sh -q
 
 cherrypick:
-	./cherrypick.sh
+	./cherrypick.sh -q
 
 backout:
-	./backout.sh
+	./backout.sh -q
 
 rebase:
-	./rebase.sh
+	./rebase.sh -q
 
 import:
-	./import.sh
+	./import.sh -q
 
 histedit:
-	./histedit.sh
+	./histedit.sh -q
 
 integrate:
-	./integrate.sh
+	./integrate.sh -q
 
 stage:
-	./stage.sh
+	./stage.sh -q
 
 unstage:
-	./unstage.sh
+	./unstage.sh -q
 
 cat:
-	./cat.sh
+	./cat.sh -q
 
 clone:
-	./clone.sh
+	./clone.sh -q
 
 fetch:
-	./fetch.sh
+	./fetch.sh -q
 
 tree:
-	./tree.sh
+	./tree.sh -q
 
 .include <bsd.regress.mk>
blob - 789676b33649267773f4b3b3f09800cc68d7bb1b
file + regress/cmdline/add.sh
--- regress/cmdline/add.sh
+++ regress/cmdline/add.sh
@@ -361,6 +361,7 @@ function test_add_symlink {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_add_basic
 run_test test_double_add
 run_test test_add_multiple
blob - 9784f978ae60d478844d4cb437e1bb1a6113a07e
file + regress/cmdline/backout.sh
--- regress/cmdline/backout.sh
+++ regress/cmdline/backout.sh
@@ -202,7 +202,7 @@ function test_backout_next_commit {
 	test_done "$testroot" "$ret"
 }
 
-
+test_parseargs "$@"
 run_test test_backout_basic
 run_test test_backout_edits_for_file_since_deleted
 run_test test_backout_next_commit
blob - 171849c46606421d957f9db7406193faf834319f
file + regress/cmdline/blame.sh
--- regress/cmdline/blame.sh
+++ regress/cmdline/blame.sh
@@ -882,6 +882,7 @@ function test_blame_symlink {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_blame_basic
 run_test test_blame_tag
 run_test test_blame_file_single_line
blob - c0793257149f648178c2a6dba7b0784645c26091
file + regress/cmdline/branch.sh
--- regress/cmdline/branch.sh
+++ regress/cmdline/branch.sh
@@ -443,6 +443,7 @@ function test_branch_show {
 
 }
 
+test_parseargs "$@"
 run_test test_branch_create
 run_test test_branch_list
 run_test test_branch_delete
blob - 57a5971b240539d837c4ce91d72ff249c5e70bee
file + regress/cmdline/cat.sh
--- regress/cmdline/cat.sh
+++ regress/cmdline/cat.sh
@@ -335,6 +335,7 @@ function test_cat_symlink {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_cat_basic
 run_test test_cat_path
 run_test test_cat_submodule
blob - 2ca82b6e47ed2d8f5e0de764be0ee241eb36cba3
file + regress/cmdline/checkout.sh
--- regress/cmdline/checkout.sh
+++ regress/cmdline/checkout.sh
@@ -756,6 +756,7 @@ function test_checkout_symlink_relative_wtpath {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_checkout_basic
 run_test test_checkout_dir_exists
 run_test test_checkout_dir_not_empty
blob - 95862835bab2b3bd688acdbf129886876cf210ec
file + regress/cmdline/cherrypick.sh
--- regress/cmdline/cherrypick.sh
+++ regress/cmdline/cherrypick.sh
@@ -779,6 +779,7 @@ function test_cherrypick_with_path_prefix_and_empty_tr
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_cherrypick_basic
 run_test test_cherrypick_root_commit
 run_test test_cherrypick_into_work_tree_with_conflicts
blob - e0ab0090ca236ade184a5727ebfd9842876870a0
file + regress/cmdline/clone.sh
--- regress/cmdline/clone.sh
+++ regress/cmdline/clone.sh
@@ -373,6 +373,7 @@ function test_clone_reference_mirror {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_clone_basic
 run_test test_clone_list
 run_test test_clone_branch
blob - 5e1cf69b0f453b136cd17913c1199ce2641b47d8
file + regress/cmdline/commit.sh
--- regress/cmdline/commit.sh
+++ regress/cmdline/commit.sh
@@ -1280,6 +1280,7 @@ function test_commit_fix_bad_symlink {
 	test_done "$testroot" "0"
 }
 
+test_parseargs "$@"
 run_test test_commit_basic
 run_test test_commit_new_subdir
 run_test test_commit_subdir
blob - bc744fcb7fa87573ecc329d19d9b5140d79d7444
file + regress/cmdline/common.sh
--- regress/cmdline/common.sh
+++ regress/cmdline/common.sh
@@ -197,10 +197,32 @@ function test_cleanup
 	rm -rf "$testroot"
 }
 
+function test_parseargs
+{
+	args=`getopt q $*`
+	if [ $? -ne 0 ]; then
+		echo "Supported options:"
+		echo "  -q: quiet mode"
+		exit 2
+	fi
+	set -- $args
+	while [ $# -ne 0 ]; do
+		case "$1"
+		in
+			-q)
+			   export GOT_TEST_QUIET=1; shift;;
+			--)
+			   shift; break;;
+		esac
+	done
+}
+
 function run_test
 {
 	testfunc="$1"
-	echo -n "$testfunc "
+	if [ -z "$GOT_TEST_QUIET" ]; then
+		echo -n "$testfunc "
+	fi
 	$testfunc
 }
 
@@ -210,7 +232,9 @@ function test_done
 	local result="$2"
 	if [ "$result" == "0" ]; then
 		test_cleanup "$testroot" || return 1
-		echo "ok"
+		if [ -z "$GOT_TEST_QUIET" ]; then
+			echo "ok"
+		fi
 	elif echo "$result" | grep -q "^xfail"; then
 		# expected test failure; test reproduces an unfixed bug
 		echo "$result"
blob - dbc39580319a7e28e9adc23ded3e53d643d3d6e0
file + regress/cmdline/diff.sh
--- regress/cmdline/diff.sh
+++ regress/cmdline/diff.sh
@@ -583,6 +583,7 @@ function test_diff_symlinks_in_repo {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_diff_basic
 run_test test_diff_shows_conflict
 run_test test_diff_tag
blob - 8d18f4cc109ac898d975c83a67497a64dd743205
file + regress/cmdline/fetch.sh
--- regress/cmdline/fetch.sh
+++ regress/cmdline/fetch.sh
@@ -948,6 +948,7 @@ function test_fetch_headref_deleted_locally {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_fetch_basic
 run_test test_fetch_list
 run_test test_fetch_branch
blob - a2bae29b0a70794309a14b5440419d30c4ff9cc0
file + regress/cmdline/histedit.sh
--- regress/cmdline/histedit.sh
+++ regress/cmdline/histedit.sh
@@ -1340,6 +1340,7 @@ function test_histedit_duplicate_commit_in_script {
 
 }
 
+test_parseargs "$@"
 run_test test_histedit_no_op
 run_test test_histedit_swap
 run_test test_histedit_drop
blob - 82df005b32ffe2f974cea0f4b8d19ac3638ce010
file + regress/cmdline/import.sh
--- regress/cmdline/import.sh
+++ regress/cmdline/import.sh
@@ -291,6 +291,7 @@ function test_import_symlink {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_import_basic
 run_test test_import_requires_new_branch
 run_test test_import_ignores
blob - 5c925e2030f9d146a0442dce5449d33e751b11f6
file + regress/cmdline/integrate.sh
--- regress/cmdline/integrate.sh
+++ regress/cmdline/integrate.sh
@@ -385,6 +385,7 @@ function test_integrate_backwards_in_time {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_integrate_basic
 run_test test_integrate_requires_rebase_first
 run_test test_integrate_path_prefix
blob - 6f9463fae57dbf5ce1d3fc5565e0e61424794051
file + regress/cmdline/log.sh
--- regress/cmdline/log.sh
+++ regress/cmdline/log.sh
@@ -792,6 +792,7 @@ function test_log_submodule {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_log_in_repo
 run_test test_log_in_bare_repo
 run_test test_log_in_worktree
blob - e54de3f2dd29851f2675c480f96c1061d6e4df17
file + regress/cmdline/rebase.sh
--- regress/cmdline/rebase.sh
+++ regress/cmdline/rebase.sh
@@ -1270,6 +1270,7 @@ function test_rebase_rm_add_rm_file {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_rebase_basic
 run_test test_rebase_ancestry_check
 run_test test_rebase_continue
blob - 94ee8dae3a4f0d0e662bbb772531e3f027ee28ab
file + regress/cmdline/ref.sh
--- regress/cmdline/ref.sh
+++ regress/cmdline/ref.sh
@@ -384,6 +384,7 @@ function test_ref_list {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_ref_create
 run_test test_ref_delete
 run_test test_ref_list
blob - 0ee9bb50011d761f484a2b1996a32492bebab27e
file + regress/cmdline/revert.sh
--- regress/cmdline/revert.sh
+++ regress/cmdline/revert.sh
@@ -1483,6 +1483,7 @@ EOF
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_revert_basic
 run_test test_revert_rm
 run_test test_revert_add
blob - 2ecf3848716083ed82686978dfb7da32ea4ea299
file + regress/cmdline/rm.sh
--- regress/cmdline/rm.sh
+++ regress/cmdline/rm.sh
@@ -438,6 +438,7 @@ function test_rm_symlink {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_rm_basic
 run_test test_rm_with_local_mods
 run_test test_double_rm
blob - 21af58910aa46d20189bd009ad4c9a6bf02d4453
file + regress/cmdline/stage.sh
--- regress/cmdline/stage.sh
+++ regress/cmdline/stage.sh
@@ -2958,6 +2958,7 @@ EOF
 	test_done "$testroot" "0"
 }
 
+test_parseargs "$@"
 run_test test_stage_basic
 run_test test_stage_no_changes
 run_test test_stage_unversioned
blob - d510c15f15bfe869139ef654fac21eb2100b2a51
file + regress/cmdline/status.sh
--- regress/cmdline/status.sh
+++ regress/cmdline/status.sh
@@ -634,6 +634,7 @@ function test_status_gitignore {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_status_basic
 run_test test_status_subdir_no_mods
 run_test test_status_subdir_no_mods2
blob - ca67103ac0f21f2ca6b3ded997d5bb8d02df01d2
file + regress/cmdline/tag.sh
--- regress/cmdline/tag.sh
+++ regress/cmdline/tag.sh
@@ -217,6 +217,7 @@ function test_tag_list_lightweight {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_tag_create
 run_test test_tag_list
 run_test test_tag_list_lightweight
blob - 44be74f368edca7e5c51454b4911ab4fc3a71465
file + regress/cmdline/tree.sh
--- regress/cmdline/tree.sh
+++ regress/cmdline/tree.sh
@@ -135,6 +135,7 @@ function test_tree_submodule_of_same_repo {
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_tree_basic
 run_test test_tree_branch
 run_test test_tree_submodule
blob - 3cd20d37557eceb704929d98cb9c43eab91e88eb
file + regress/cmdline/unstage.sh
--- regress/cmdline/unstage.sh
+++ regress/cmdline/unstage.sh
@@ -1417,6 +1417,7 @@ EOF
 	test_done "$testroot" "$ret"
 }
 
+test_parseargs "$@"
 run_test test_unstage_basic
 run_test test_unstage_unversioned
 run_test test_unstage_nonexistent
blob - 29f55aa5a7d96f3b060398ff0db5ae16d8e4f3e1
file + regress/cmdline/update.sh
--- regress/cmdline/update.sh
+++ regress/cmdline/update.sh
@@ -2254,6 +2254,7 @@ function test_update_symlink_conflicts {
 
 }
 
+test_parseargs "$@"
 run_test test_update_basic
 run_test test_update_adds_file
 run_test test_update_deletes_file
blob - 6ac1f2f67f65445533f26f7c05bfc15857d311ec
file + regress/delta/Makefile
--- regress/delta/Makefile
+++ regress/delta/Makefile
@@ -8,4 +8,7 @@ LDADD = -lz
 
 NOMAN = yes
 
+run-regress-delta_test:
+	${.OBJDIR}/delta_test -q
+
 .include <bsd.regress.mk>
blob - 0ee8340a74100f52a328596febc5de2ef96554f4
file + regress/delta/delta_test.c
--- regress/delta/delta_test.c
+++ regress/delta/delta_test.c
@@ -21,6 +21,7 @@
 #include <string.h>
 #include <err.h>
 #include <unistd.h>
+#include <getopt.h>
 
 #include "got_error.h"
 #include "got_opentemp.h"
@@ -117,19 +118,42 @@ delta_apply(void)
 	return (err == NULL);
 }
 
+static int quiet;
+
 #define RUN_TEST(expr, name) \
 	{ test_ok = (expr);  \
-	printf("test_%s %s\n", (name), test_ok ? "ok" : "failed"); \
+	if (!quiet) printf("test_%s %s\n", (name), test_ok ? "ok" : "failed"); \
 	failure = (failure || !test_ok); }
 
+static void
+usage(void)
+{
+	fprintf(stderr, "usage: delta_test [-q]\n");
+}
+
 int
-main(int argc, const char *argv[])
+main(int argc, char *argv[])
 {
 	int test_ok;
 	int failure = 0;
+	int ch;
 
-	if (argc != 1) {
-		fprintf(stderr, "usage: delta_test [REPO_PATH]\n");
+	while ((ch = getopt(argc, argv, "q")) != -1) {
+		switch (ch) {
+		case 'q':
+			quiet = 1;
+			break;
+		default:
+			usage();
+			return 1;
+		}
+	}
+
+	argc -= optind;
+	argv += optind;
+
+	if (argc != 0) {
+		usage();
 		return 1;
 	}
 
blob - 94a84612acbddadb06374e6a4344fe42b4b414f2
file + regress/fetch/Makefile
--- regress/fetch/Makefile
+++ regress/fetch/Makefile
@@ -11,4 +11,7 @@ LDADD = -lutil -lz
 
 NOMAN = yes
 
+run-regress-fetch_test:
+	${.OBJDIR}/fetch_test -q
+
 .include <bsd.regress.mk>
blob - 1d4429c2303caf6a1a577319457466d945db73ed
file + regress/fetch/fetch_test.c
--- regress/fetch/fetch_test.c
+++ regress/fetch/fetch_test.c
@@ -42,6 +42,7 @@
 #endif
 
 static int verbose;
+static int quiet;
 
 void
 test_printf(char *fmt, ...)
@@ -237,13 +238,13 @@ fetch_parse_uri(void)
 
 #define RUN_TEST(expr, name) \
 	{ test_ok = (expr);  \
-	printf("test_%s %s\n", (name), test_ok ? "ok" : "failed"); \
+	if (!quiet) printf("test_%s %s\n", (name), test_ok ? "ok" : "failed"); \
 	failure = (failure || !test_ok); }
 
 void
 usage(void)
 {
-	fprintf(stderr, "usage: fetch_test [-v]\n");
+	fprintf(stderr, "usage: fetch_test [-v] [-q]\n");
 }
 
 int
@@ -257,10 +258,15 @@ main(int argc, char *argv[])
 		err(1, "pledge");
 #endif
 
-	while ((ch = getopt(argc, argv, "v")) != -1) {
+	while ((ch = getopt(argc, argv, "vq")) != -1) {
 		switch (ch) {
 		case 'v':
 			verbose = 1;
+			quiet = 0;
+			break;
+		case 'q':
+			quiet = 1;
+			verbose = 0;
 			break;
 		default:
 			usage();
blob - c668822fccf1c73ab7e8e47eec5a14f547680191
file + regress/idset/Makefile
--- regress/idset/Makefile
+++ regress/idset/Makefile
@@ -9,4 +9,7 @@ LDADD = -lutil -lz
 
 NOMAN = yes
 
+run-regress-idset_test:
+	${.OBJDIR}/idset_test -q
+
 .include <bsd.regress.mk>
blob - ba2b42bdfcb5a8abcf06df94f35d91f85ca419cb
file + regress/idset/idset_test.c
--- regress/idset/idset_test.c
+++ regress/idset/idset_test.c
@@ -36,6 +36,7 @@
 #include "got_lib_object.h"
 
 static int verbose;
+static int quiet;
 
 void
 test_printf(char *fmt, ...)
@@ -169,13 +170,13 @@ done:
 
 #define RUN_TEST(expr, name) \
 	{ test_ok = (expr);  \
-	printf("test_%s %s\n", (name), test_ok ? "ok" : "failed"); \
+	if (!quiet) printf("test_%s %s\n", (name), test_ok ? "ok" : "failed"); \
 	failure = (failure || !test_ok); }
 
 void
 usage(void)
 {
-	fprintf(stderr, "usage: id_test [-v]\n");
+	fprintf(stderr, "usage: id_test [-v] [-q]\n");
 }
 
 int
@@ -189,10 +190,15 @@ main(int argc, char *argv[])
 		err(1, "pledge");
 #endif
 
-	while ((ch = getopt(argc, argv, "v")) != -1) {
+	while ((ch = getopt(argc, argv, "vq")) != -1) {
 		switch (ch) {
 		case 'v':
 			verbose = 1;
+			quiet = 0;
+			break;
+		case 'q':
+			quiet = 1;
+			verbose = 0;
 			break;
 		default:
 			usage();
blob - ebb3acb5a444722ac27006125494c5877a9e4ae9
file + regress/path/Makefile
--- regress/path/Makefile
+++ regress/path/Makefile
@@ -8,4 +8,7 @@ LDADD = -lutil -lz
 
 NOMAN = yes
 
+run-regress-path_test:
+	${.OBJDIR}/path_test -q
+
 .include <bsd.regress.mk>
blob - 07e0baa80ab15cf50861022e1fc478bf14075a7a
file + regress/path/path_test.c
--- regress/path/path_test.c
+++ regress/path/path_test.c
@@ -31,6 +31,7 @@
 #endif
 
 static int verbose;
+static int quiet;
 
 void
 test_printf(char *fmt, ...)
@@ -207,13 +208,13 @@ path_list_reverse_input(void)
 
 #define RUN_TEST(expr, name) \
 	{ test_ok = (expr);  \
-	printf("test_%s %s\n", (name), test_ok ? "ok" : "failed"); \
+	if (!quiet) printf("test_%s %s\n", (name), test_ok ? "ok" : "failed"); \
 	failure = (failure || !test_ok); }
 
 void
 usage(void)
 {
-	fprintf(stderr, "usage: path_test [-v]\n");
+	fprintf(stderr, "usage: path_test [-v] [-q]\n");
 }
 
 int
@@ -227,10 +228,15 @@ main(int argc, char *argv[])
 		err(1, "pledge");
 #endif
 
-	while ((ch = getopt(argc, argv, "v")) != -1) {
+	while ((ch = getopt(argc, argv, "vq")) != -1) {
 		switch (ch) {
 		case 'v':
 			verbose = 1;
+			quiet = 0;
+			break;
+		case 'q':
+			quiet = 1;
+			verbose = 0;
 			break;
 		default:
 			usage();