From: Stefan Sperling <stsp@stsp.name>
Subject: speed up 'got status' a little bit
To: gameoftrees@openbsd.org
Date: Thu, 30 Sep 2021 17:03:57 +0200

The status walk currently always traverses ignored directories on disk.
This can be expensive when ignored directories contain a lot of stuff,
such as 'obj' directories, /usr/ports/pobj, /usr/ports/distfiles, etc.

With this patch we stop crawling such directories unless the -I option
was passed to 'got status'. On my system this makes 'got status' in
/usr/ports complete in about 4 seconds instead of 5. The effect could
be more pronounced for people building a lot of ports (naddy? :)

Existing regression tests keep passing. Because this is just about a
performance boost there is no need for a new test in the regress suite.

ok?

diff 78caff98cb8783be4d83da7279e407d4ccd1f187 /home/stsp/src/got
blob - aec06af5925a0cd63ffc6b150466bad036d5abd8
file + lib/fileindex.c
--- lib/fileindex.c
+++ lib/fileindex.c
@@ -991,7 +991,7 @@ static const struct got_error *
 walk_dir(struct got_pathlist_entry **next, struct got_fileindex *fileindex,
     struct got_fileindex_entry **ie, struct got_pathlist_entry *dle, int fd,
     const char *path, const char *rootpath, struct got_repository *repo,
-    struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
+    int ignore, struct got_fileindex_diff_dir_cb *cb, void *cb_arg)
 {
 	const struct got_error *err = NULL;
 	struct dirent *de = dle->data;
@@ -1013,7 +1013,7 @@ walk_dir(struct got_pathlist_entry **next, struct got_
 	} else
 		type = de->d_type;
 
-	if (type == DT_DIR) {
+	if (type == DT_DIR && !ignore) {
 		char *subpath;
 		char *subdirpath;
 		struct got_pathlist_head subdirlist;
@@ -1079,6 +1079,7 @@ diff_fileindex_dir(struct got_fileindex *fileindex,
 	struct dirent *de = NULL;
 	size_t path_len = strlen(path);
 	struct got_pathlist_entry *dle;
+	int ignore;
 
 	if (cb->diff_traverse) {
 		err = cb->diff_traverse(cb_arg, path, dirfd);
@@ -1108,18 +1109,19 @@ diff_fileindex_dir(struct got_fileindex *fileindex,
 					break;
 				*ie = walk_fileindex(fileindex, *ie);
 				err = walk_dir(&dle, fileindex, ie, dle, dirfd,
-				    path, rootpath, repo, cb, cb_arg);
+				    path, rootpath, repo, 0, cb, cb_arg);
 			} else if (cmp < 0 ) {
 				err = cb->diff_old(cb_arg, *ie, path);
 				if (err)
 					break;
 				*ie = walk_fileindex(fileindex, *ie);
 			} else {
-				err = cb->diff_new(cb_arg, de, path, dirfd);
+				err = cb->diff_new(&ignore, cb_arg, de, path,
+				    dirfd);
 				if (err)
 					break;
 				err = walk_dir(&dle, fileindex, ie, dle, dirfd,
-				    path, rootpath, repo, cb, cb_arg);
+				    path, rootpath, repo, ignore, cb, cb_arg);
 			}
 			if (err)
 				break;
@@ -1130,11 +1132,11 @@ diff_fileindex_dir(struct got_fileindex *fileindex,
 			*ie = walk_fileindex(fileindex, *ie);
 		} else if (dle) {
 			de = dle->data;
-			err = cb->diff_new(cb_arg, de, path, dirfd);
+			err = cb->diff_new(&ignore, cb_arg, de, path, dirfd);
 			if (err)
 				break;
 			err = walk_dir(&dle, fileindex, ie, dle, dirfd, path,
-			    rootpath, repo, cb, cb_arg);
+			    rootpath, repo, ignore, cb, cb_arg);
 			if (err)
 				break;
 		}
blob - d03058e83c5f5c0a69baf66795455eb3ab8b7af7
file + lib/got_lib_fileindex.h
--- lib/got_lib_fileindex.h
+++ lib/got_lib_fileindex.h
@@ -148,7 +148,7 @@ typedef const struct got_error *(*got_fileindex_diff_d
     struct got_fileindex_entry *, struct dirent *, const char *, int);
 typedef const struct got_error *(*got_fileindex_diff_dir_old_cb)(void *,
     struct got_fileindex_entry *, const char *);
-typedef const struct got_error *(*got_fileindex_diff_dir_new_cb)(void *,
+typedef const struct got_error *(*got_fileindex_diff_dir_new_cb)(int *, void *,
     struct dirent *, const char *, int);
 typedef const struct got_error *(*got_fileindex_diff_dir_traverse)(void *,
     const char *, int);
blob - 2fab18710a8a5331db77b19986086dd87bc4820d
file + lib/worktree.c
--- lib/worktree.c
+++ lib/worktree.c
@@ -3617,12 +3617,16 @@ add_ignores(struct got_pathlist_head *ignores, const c
 }
 
 static const struct got_error *
-status_new(void *arg, struct dirent *de, const char *parent_path, int dirfd)
+status_new(int *ignore, void *arg, struct dirent *de, const char *parent_path,
+    int dirfd)
 {
 	const struct got_error *err = NULL;
 	struct diff_dir_cb_arg *a = arg;
 	char *path = NULL;
 
+	if (ignore != NULL)
+		*ignore = 0;
+
 	if (a->cancel_cb && a->cancel_cb(a->cancel_arg))
 		return got_error(GOT_ERR_CANCELLED);
 
@@ -3633,9 +3637,12 @@ status_new(void *arg, struct dirent *de, const char *p
 		path = de->d_name;
 	}
 
-	if (de->d_type != DT_DIR &&
-	    got_path_is_child(path, a->status_path, a->status_path_len)
-	    && !match_ignores(a->ignores, path))
+	if (de->d_type == DT_DIR) {
+		if (!a->no_ignores && ignore != NULL &&
+		    match_ignores(a->ignores, path))
+			*ignore = 1;
+	} else if (!match_ignores(a->ignores, path) &&
+	    got_path_is_child(path, a->status_path, a->status_path_len))
 		err = (*a->status_cb)(a->status_arg, GOT_STATUS_UNVERSIONED,
 		    GOT_STATUS_NO_CHANGE, path, NULL, NULL, NULL, -1, NULL);
 	if (parent_path[0])