JSON strings are made of UNICODE codepoints, of which only the control
characters, \ and " need to be escaped.  Furthermore, per RFC8259:

: JSON text exchanged between systems that are not part of a closed
: ecosystem MUST be encoded using UTF-8.

so when POSTing the notifications the JSON text has to be encoded in
UTF-8.

The current code is wrong because it escapes with \uXXXX *byte* over
0x7F, and this causes mis-decodings issues.

isu8cont() as far as I can see will happily accept surrogate pairs and
overlong sequences (since it doesn't parse), which will cause an error
on the receiving side while decoding the JSON.

I don't think I can reasonably use mbtowc() either since it will use the
current locale which is problematic in -portable.

So, I'm bundling my favourite utf8 decoder (DFAs are lovely) and using
that to read the text.  Upon decoding error the replacement character
U+FFFD is emitted in the JSON string, all the bytes considered so far
discarded and the decoder restarted with the next byte.  (Not the only
technique, just the simpler to implement.)

I'm also adding a test case that uses an invalid UTF-8 byte sequence in
the commit message.

I think it's reasonable to expect the commit messages, branches and tag
names to be in UTF-8, as long as we correctly handle decoding errors
too.

Worst case, some fields of the notification object will be parseable
gargabe, but the commit ID is still readable since it's made of only
ASCII chars, so the receiving end could still access the commit data on
its own should they use a more esotic encoding.

thoughts?

diff /home/op/w/got
commit - 3b44bdbe821e92c08ae1074b204b273b661e1941
path + /home/op/w/got
blob - 80cfc6f803bbf046beb7285ddb64fd7ef3169391
file + gotd/libexec/got-notify-http/got-notify-http.c
--- gotd/libexec/got-notify-http/got-notify-http.c
+++ gotd/libexec/got-notify-http/got-notify-http.c
@@ -33,6 +33,7 @@
 #include "got_version.h"
 
 #include "bufio.h"
+#include "utf8d.h"
 
 #define USERAGENT	 "got-notify-http/" GOT_VERSION_STR
 
@@ -91,40 +92,51 @@ dial(const char *host, const char *port)
 static void
 escape(FILE *fp, const uint8_t *s)
 {
+	uint32_t codepoint, state;
+	const uint8_t *start = s;
+
+	state = 0;
 	for (; *s; ++s) {
-		/*
-		 * XXX: this is broken for UNICODE: we should leave
-		 * the multibyte characters as-is.
-		 */
+		switch (decode(&state, &codepoint, *s)) {
+		case UTF8_ACCEPT:
+			switch (codepoint) {
+			case '"':
+			case '\\':
+				fprintf(fp, "\\%c", *s);
+				break;
+			case '\b':
+				fprintf(fp, "\\b");
+				break;
+			case '\f':
+				fprintf(fp, "\\f");
+				break;
+			case '\n':
+				fprintf(fp, "\\n");
+				break;
+			case '\r':
+				fprintf(fp, "\\r");
+				break;
+			case '\t':
+				fprintf(fp, "\\t");
+				break;
+			default:
+				/* other control characters */
+				if (codepoint < ' ' || codepoint == 0x7F) {
+					fprintf(fp, "\\u%04x", codepoint);
+					break;
+				}
+				fwrite(start, 1, s - start + 1, fp);
+				break;
+			}
+			start = s + 1;
+			break;
 
-		if (*s >= ' ' && *s <= '~') {
-			fputc(*s, fp);
-			continue;
-		}
-
-		switch (*s) {
-		case '"':
-		case '\\':
-			fprintf(fp, "\\%c", *s);
+		case UTF8_REJECT:
+			/* bad UTF-8 sequence; try to recover */
+			fputs("\\uFFFD", fp);
+			state = UTF8_ACCEPT;
+			start = s + 1;
 			break;
-		case '\b':
-			fprintf(fp, "\\b");
-			break;
-		case '\f':
-			fprintf(fp, "\\f");
-			break;
-		case '\n':
-			fprintf(fp, "\\n");
-			break;
-		case '\r':
-			fprintf(fp, "\\r");
-			break;
-		case '\t':
-			fprintf(fp, "\\t");
-			break;
-		default:
-			fprintf(fp, "\\u%04X", *s);
-			break;
 		}
         }
 }
blob - /dev/null
file + gotd/libexec/got-notify-http/utf8d.h (mode 644)
--- /dev/null
+++ gotd/libexec/got-notify-http/utf8d.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
+
+#define UTF8_ACCEPT 0
+#define UTF8_REJECT 1
+
+static const uint8_t utf8d[] = {
+  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
+  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
+  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
+  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
+  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
+  8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
+  0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef
+  0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff
+  0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0
+  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2
+  1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4
+  1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6
+  1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8
+};
+
+static uint32_t inline
+decode(uint32_t* state, uint32_t* codep, uint32_t byte) {
+  uint32_t type = utf8d[byte];
+
+  *codep = (*state != UTF8_ACCEPT) ?
+    (byte & 0x3fu) | (*codep << 6) :
+    (0xff >> type) & (byte);
+
+  *state = utf8d[256 + *state*16 + type];
+  return *state;
+}
blob - 5bd5779ff7fd7e13571b71c414ee5f1bb1f6dc2e
file + regress/gotd/http_notification.sh
--- regress/gotd/http_notification.sh
+++ regress/gotd/http_notification.sh
@@ -78,6 +78,69 @@ test_file_changed() {
 	test_done "$testroot" "$ret"
 }
 
+test_bad_utf8() {
+	local testroot=`test_init bad_utf8 1`
+
+	got clone -a -q ${GOTD_TEST_REPO_URL} $testroot/repo-clone
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		echo "got clone failed unexpectedly" >&2
+		test_done "$testroot" 1
+		return 1
+	fi
+
+	got checkout -q $testroot/repo-clone $testroot/wt >/dev/null
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		echo "got checkout failed unexpectedly" >&2
+		test_done "$testroot" 1
+	fi
+
+	# invalid utf8 sequenc
+	commit_msg="make$(printf '\xED\xA0\x80')changes"
+
+	echo "changed" > $testroot/wt/alpha
+	(cd $testroot/wt && got commit -m "$commit_msg" > /dev/null)
+	local commit_id=`git_show_head $testroot/repo-clone`
+	local author_time=`git_show_author_time $testroot/repo-clone`
+
+	timeout 5 ./http-server -p $GOTD_TEST_HTTP_PORT \
+	    > $testroot/stdout &
+
+	got send -b main -q -r $testroot/repo-clone
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		echo "got send failed unexpectedly" >&2
+		test_done "$testroot" "1"
+		return 1
+	fi
+
+	wait %1 # wait for the http "server"
+
+	d=`date -u -r $author_time +"%a %b %e %X %Y UTC"`
+
+	cat <<-EOF > $testroot/stdout.expected
+	POST / HTTP/1.1
+	Host: localhost:${GOTD_TEST_HTTP_PORT}
+	Content-Type: application/json
+	Content-Length: 235
+	User-Agent: got-notify-http/${GOT_VERSION_STR}
+	Connection: close
+
+	{"notifications":[{"short":false,"id":"$commit_id","author":"$GOT_AUTHOR","date":"$d","message":"make\uFFFD\uFFFDchanges\n","diffstat":{},"changes":{}}]}
+	EOF
+
+	cmp -s $testroot/stdout.expected $testroot/stdout
+	ret=$?
+	if [ $ret -ne 0 ]; then
+		diff -u $testroot/stdout.expected $testroot/stdout
+		test_done "$testroot" "$ret"
+		return 1
+	fi
+
+	test_done "$testroot" "$ret"
+}
+
 test_many_commits_not_summarized() {
 	local testroot=`test_init many_commits_not_summarized 1`
 
@@ -230,5 +293,6 @@ test_many_commits_summarized() {
 
 test_parseargs "$@"
 run_test test_file_changed
+run_test test_bad_utf8
 run_test test_many_commits_not_summarized
 run_test test_many_commits_summarized