From: Tracey Emery <tracey@traceyemery.net>
Subject: New library config parse.y start
To: gameoftrees@openbsd.org
Date: Fri, 19 Jun 2020 10:34:39 -0600

Hello,

The following diff integrates config parsing to the library. This is the
first step to adding gotconfig to individual git repos for git push and
git fetch, as discussed with Stefan off list.

This also paves the way for centralized parsing for gotweb and gotd.
After this, I'll be working on moving gotweb grammar and getting that to
work.

Once in, we can start tightening up functions, as this is mostly stock
from newd.

Ok?

-- 

Tracey Emery

diff 2c2d5c5f4cce08cb69d03aea1a1c747fd27f9e39 /home/tracey/src/got
blob - /dev/null
file + include/got_config_parse.h
--- include/got_config_parse.h
+++ include/got_config_parse.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2020 Tracey Emery <tracey@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+struct got_config_list_entry {
+	TAILQ_ENTRY(got_config_list_entry) entry;
+	const char	*remote;
+	const char	*repository;
+	const char	*server;
+	const char	*protocol;
+	const char	*user;
+};
+TAILQ_HEAD(got_config_list, got_config_list_entry);
+
+/*
+ * Parse individual gotconfig repository files
+ * Load got_config_list_entry struct and insert to got_config_list TAILQ
+ */
+const struct got_error* got_config_parse_got_config(struct got_config_list **,
+    char *filename);
blob - /dev/null
file + lib/parse.y
--- lib/parse.y
+++ lib/parse.y
@@ -0,0 +1,655 @@
+/*
+ * Copyright (c) 2020, Tracey Emery <tracey@openbsd.org>
+ * Copyright (c) 2004, 2005 Esben Norby <norby@openbsd.org>
+ * Copyright (c) 2004 Ryan McBride <mcbride@openbsd.org>
+ * Copyright (c) 2002, 2003, 2004 Henning Brauer <henning@openbsd.org>
+ * Copyright (c) 2001 Markus Friedl.  All rights reserved.
+ * Copyright (c) 2001 Daniel Hartmeier.  All rights reserved.
+ * Copyright (c) 2001 Theo de Raadt.  All rights reserved.
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+%{
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+
+#include <netinet/in.h>
+
+#include <arpa/inet.h>
+
+#include <ctype.h>
+#include <err.h>
+#include <errno.h>
+#include <event.h>
+#include <ifaddrs.h>
+#include <imsg.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <syslog.h>
+#include <unistd.h>
+
+#include "got_error.h"
+#include "got_config_parse.h"
+
+TAILQ_HEAD(files, file)		 files = TAILQ_HEAD_INITIALIZER(files);
+static struct file {
+	TAILQ_ENTRY(file)	 entry;
+	FILE			*stream;
+	char			*name;
+	size_t	 		 ungetpos;
+	size_t			 ungetsize;
+	u_char			*ungetbuf;
+	int			 eof_reached;
+	int			 lineno;
+} *file, *topfile;
+static const struct got_error*	pushfile(struct file**, const char *);
+int		 popfile(void);
+int		 yyparse(void);
+int		 yylex(void);
+int		 yyerror(const char *, ...)
+    __attribute__((__format__ (printf, 1, 2)))
+    __attribute__((__nonnull__ (1)));
+int		 kw_cmp(const void *, const void *);
+int		 lookup(char *);
+int		 igetc(void);
+int		 lgetc(int);
+void		 lungetc(int);
+int		 findeol(void);
+
+TAILQ_HEAD(symhead, sym)	 symhead = TAILQ_HEAD_INITIALIZER(symhead);
+struct sym {
+	TAILQ_ENTRY(sym)	 entry;
+	int			 used;
+	int			 persist;
+	char			*nam;
+	char			*val;
+};
+
+int	 symset(const char *, const char *, int);
+char	*symget(const char *);
+
+const struct got_error* gerror = NULL;
+struct got_config_list_entry *gotconfig;
+struct got_config_list got_config_list;
+static const struct got_error*	new_remote(struct got_config_list_entry **);
+
+typedef struct {
+	union {
+		int64_t		 number;
+		char		*string;
+	} v;
+	int lineno;
+} YYSTYPE;
+
+%}
+
+%token	ERROR
+%token	REMOTE REPOSITORY SERVER PROTOCOL USER
+%token	<v.string>	STRING
+%token	<v.number>	NUMBER
+
+%%
+
+grammar		: /* empty */
+		| grammar '\n'
+		| grammar remote '\n'
+		;
+remoteopts2	: remoteopts2 remoteopts1 nl
+	    	| remoteopts1 optnl
+		;
+remoteopts1	: REPOSITORY STRING {
+	    		gotconfig->repository = strdup($2);
+			if (gotconfig->repository == NULL) {
+				free($2);
+				yyerror("strdup");
+				YYERROR;
+			}
+			free($2);
+	    	}
+	    	| SERVER STRING {
+	    		gotconfig->server = strdup($2);
+			if (gotconfig->server == NULL) {
+				free($2);
+				yyerror("strdup");
+				YYERROR;
+			}
+			free($2);
+		}
+		| PROTOCOL STRING {
+	    		gotconfig->protocol = strdup($2);
+			if (gotconfig->protocol == NULL) {
+				free($2);
+				yyerror("strdup");
+				YYERROR;
+			}
+			free($2);
+		}
+		| USER STRING {
+	    		gotconfig->user = strdup($2);
+			if (gotconfig->user == NULL) {
+				free($2);
+				yyerror("strdup");
+				YYERROR;
+			}
+			free($2);
+		}
+	    	;
+remote		: REMOTE STRING {
+			static const struct got_error* error;
+
+			error = new_remote(&gotconfig);
+			if (error) {
+				free($2);
+				yyerror("%s", error->msg);
+				YYERROR;
+			}
+			gotconfig->remote = strdup($2);
+			if (gotconfig->remote == NULL) {
+				free($2);
+				yyerror("strdup");
+				YYERROR;
+			}
+			free($2);
+		} '{' optnl remoteopts2 '}' {
+			TAILQ_INSERT_TAIL(&got_config_list, gotconfig, entry);
+		}
+		;
+optnl		: '\n' optnl
+		| /* empty */
+		;
+nl		: '\n' optnl
+		;
+%%
+
+struct keywords {
+	const char	*k_name;
+	int		 k_val;
+};
+
+int
+yyerror(const char *fmt, ...)
+{
+	va_list		 ap;
+	char		*msg;
+	char		*err = NULL;
+
+	va_start(ap, fmt);
+	if (vasprintf(&msg, fmt, ap) == -1) {
+		gerror =  got_error_from_errno("vasprintf");
+		return 0;
+	}
+	va_end(ap);
+	if (asprintf(&err, "%s:%d: %s", file->name, yylval.lineno, msg) == -1) {
+		gerror = got_error_from_errno("asprintf");
+		return(0);
+	}
+	gerror = got_error_msg(GOT_ERR_PARSE_Y_YY, strdup(err));
+	free(msg);
+	free(err);
+	return(0);
+}
+
+int
+kw_cmp(const void *k, const void *e)
+{
+	return (strcmp(k, ((const struct keywords *)e)->k_name));
+}
+
+int
+lookup(char *s)
+{
+	/* This has to be sorted always. */
+	static const struct keywords keywords[] = {
+		{"protocol",		PROTOCOL},
+		{"remote",		REMOTE},
+		{"repository",		REPOSITORY},
+		{"server",		SERVER},
+		{"user",		USER},
+	};
+	const struct keywords	*p;
+
+	p = bsearch(s, keywords, sizeof(keywords)/sizeof(keywords[0]),
+	    sizeof(keywords[0]), kw_cmp);
+
+	if (p)
+		return (p->k_val);
+	else
+		return (STRING);
+}
+
+#define START_EXPAND	1
+#define DONE_EXPAND	2
+
+static int	expanding;
+
+int
+igetc(void)
+{
+	int	c;
+
+	while (1) {
+		if (file->ungetpos > 0)
+			c = file->ungetbuf[--file->ungetpos];
+		else
+			c = getc(file->stream);
+
+		if (c == START_EXPAND)
+			expanding = 1;
+		else if (c == DONE_EXPAND)
+			expanding = 0;
+		else
+			break;
+	}
+	return (c);
+}
+
+int
+lgetc(int quotec)
+{
+	int		c, next;
+
+	if (quotec) {
+		if ((c = igetc()) == EOF) {
+			yyerror("reached end of file while parsing "
+			    "quoted string");
+			if (file == topfile || popfile() == EOF)
+				return (EOF);
+			return (quotec);
+		}
+		return (c);
+	}
+
+	while ((c = igetc()) == '\\') {
+		next = igetc();
+		if (next != '\n') {
+			c = next;
+			break;
+		}
+		yylval.lineno = file->lineno;
+		file->lineno++;
+	}
+
+	if (c == EOF) {
+		/*
+		 * Fake EOL when hit EOF for the first time. This gets line
+		 * count right if last line in included file is syntactically
+		 * invalid and has no newline.
+		 */
+		if (file->eof_reached == 0) {
+			file->eof_reached = 1;
+			return ('\n');
+		}
+		while (c == EOF) {
+			if (file == topfile || popfile() == EOF)
+				return (EOF);
+			c = igetc();
+		}
+	}
+	return (c);
+}
+
+void
+lungetc(int c)
+{
+	if (c == EOF)
+		return;
+
+	if (file->ungetpos >= file->ungetsize) {
+		void *p = reallocarray(file->ungetbuf, file->ungetsize, 2);
+		if (p == NULL)
+			err(1, "%s", __func__);
+		file->ungetbuf = p;
+		file->ungetsize *= 2;
+	}
+	file->ungetbuf[file->ungetpos++] = c;
+}
+
+int
+findeol(void)
+{
+	int	c;
+
+	/* Skip to either EOF or the first real EOL. */
+	while (1) {
+		c = lgetc(0);
+		if (c == '\n') {
+			file->lineno++;
+			break;
+		}
+		if (c == EOF)
+			break;
+	}
+	return (ERROR);
+}
+
+int
+yylex(void)
+{
+	unsigned char	 buf[8096];
+	unsigned char	*p, *val;
+	int		 quotec, next, c;
+	int		 token;
+
+top:
+	p = buf;
+	while ((c = lgetc(0)) == ' ' || c == '\t')
+		; /* nothing */
+
+	yylval.lineno = file->lineno;
+	if (c == '#')
+		while ((c = lgetc(0)) != '\n' && c != EOF)
+			; /* nothing */
+	if (c == '$' && !expanding) {
+		while (1) {
+			if ((c = lgetc(0)) == EOF)
+				return (0);
+
+			if (p + 1 >= buf + sizeof(buf) - 1) {
+				yyerror("string too long");
+				return (findeol());
+			}
+			if (isalnum(c) || c == '_') {
+				*p++ = c;
+				continue;
+			}
+			*p = '\0';
+			lungetc(c);
+			break;
+		}
+		val = symget(buf);
+		if (val == NULL) {
+			yyerror("macro '%s' not defined", buf);
+			return (findeol());
+		}
+		p = val + strlen(val) - 1;
+		lungetc(DONE_EXPAND);
+		while (p >= val) {
+			lungetc(*p);
+			p--;
+		}
+		lungetc(START_EXPAND);
+		goto top;
+	}
+
+	switch (c) {
+	case '\'':
+	case '"':
+		quotec = c;
+		while (1) {
+			if ((c = lgetc(quotec)) == EOF)
+				return (0);
+			if (c == '\n') {
+				file->lineno++;
+				continue;
+			} else if (c == '\\') {
+				if ((next = lgetc(quotec)) == EOF)
+					return (0);
+				if (next == quotec || c == ' ' || c == '\t')
+					c = next;
+				else if (next == '\n') {
+					file->lineno++;
+					continue;
+				} else
+					lungetc(next);
+			} else if (c == quotec) {
+				*p = '\0';
+				break;
+			} else if (c == '\0') {
+				yyerror("syntax error");
+				return (findeol());
+			}
+			if (p + 1 >= buf + sizeof(buf) - 1) {
+				yyerror("string too long");
+				return (findeol());
+			}
+			*p++ = c;
+		}
+		yylval.v.string = strdup(buf);
+		if (yylval.v.string == NULL)
+			err(1, "%s", __func__);
+		return (STRING);
+	}
+
+#define allowed_to_end_number(x) \
+	(isspace(x) || x == ')' || x ==',' || x == '/' || x == '}' || x == '=')
+
+	if (c == '-' || isdigit(c)) {
+		do {
+			*p++ = c;
+			if ((unsigned)(p-buf) >= sizeof(buf)) {
+				yyerror("string too long");
+				return (findeol());
+			}
+		} while ((c = lgetc(0)) != EOF && isdigit(c));
+		lungetc(c);
+		if (p == buf + 1 && buf[0] == '-')
+			goto nodigits;
+		if (c == EOF || allowed_to_end_number(c)) {
+			const char *errstr = NULL;
+
+			*p = '\0';
+			yylval.v.number = strtonum(buf, LLONG_MIN,
+			    LLONG_MAX, &errstr);
+			if (errstr) {
+				yyerror("\"%s\" invalid number: %s",
+				    buf, errstr);
+				return (findeol());
+			}
+			return (NUMBER);
+		} else {
+nodigits:
+			while (p > buf + 1)
+				lungetc(*--p);
+			c = *--p;
+			if (c == '-')
+				return (c);
+		}
+	}
+
+#define allowed_in_string(x) \
+	(isalnum(x) || (ispunct(x) && x != '(' && x != ')' && \
+	x != '{' && x != '}' && \
+	x != '!' && x != '=' && x != '#' && \
+	x != ','))
+
+	if (isalnum(c) || c == ':' || c == '_') {
+		do {
+			*p++ = c;
+			if ((unsigned)(p-buf) >= sizeof(buf)) {
+				yyerror("string too long");
+				return (findeol());
+			}
+		} while ((c = lgetc(0)) != EOF && (allowed_in_string(c)));
+		lungetc(c);
+		*p = '\0';
+		if ((token = lookup(buf)) == STRING)
+			if ((yylval.v.string = strdup(buf)) == NULL)
+				err(1, "%s", __func__);
+		return (token);
+	}
+	if (c == '\n') {
+		yylval.lineno = file->lineno;
+		file->lineno++;
+	}
+	if (c == EOF)
+		return (0);
+	return (c);
+}
+
+static const struct got_error*
+pushfile(struct file **nfile, const char *name)
+{
+	const struct got_error* error = NULL;
+
+	if (((*nfile) = calloc(1, sizeof(struct file))) == NULL)
+		return got_error_from_errno2(__func__, "calloc");
+	if (((*nfile)->name = strdup(name)) == NULL) {
+		free(nfile);
+		return got_error_from_errno2(__func__, "strdup");
+	}
+	if (((*nfile)->stream = fopen((*nfile)->name, "r")) == NULL) {
+		char *msg = NULL;
+		if (asprintf(&msg, "%s", (*nfile)->name) == -1)
+			return got_error_from_errno("asprintf");
+		free((*nfile)->name);
+		free((*nfile));
+		error = got_error_from_errno(msg);
+		free(msg);
+		return error;
+	}
+	(*nfile)->lineno = TAILQ_EMPTY(&files) ? 1 : 0;
+	(*nfile)->ungetsize = 16;
+	(*nfile)->ungetbuf = malloc((*nfile)->ungetsize);
+	if ((*nfile)->ungetbuf == NULL) {
+		fclose((*nfile)->stream);
+		free((*nfile)->name);
+		free((*nfile));
+		return got_error_from_errno2(__func__, "malloc");
+	}
+	TAILQ_INSERT_TAIL(&files, (*nfile), entry);
+	return error;
+}
+
+static const struct got_error*
+new_remote(struct got_config_list_entry **gotconfig) {
+	const struct got_error* error = NULL;
+
+	if (((*gotconfig) = calloc(1, sizeof(struct got_config_list_entry))) ==
+	    NULL)
+	    error = got_error_from_errno("calloc");
+	return error;
+}
+
+int
+popfile(void)
+{
+	struct file	*prev = NULL;
+
+	TAILQ_REMOVE(&files, file, entry);
+	fclose(file->stream);
+	free(file->name);
+	free(file->ungetbuf);
+	free(file);
+	file = prev;
+	return (file ? 0 : EOF);
+}
+
+const struct got_error*
+got_config_parse_got_config(struct got_config_list **conf_list, char *filename)
+{
+	struct sym	*sym, *next;
+
+	*conf_list = NULL;
+	TAILQ_INIT(&got_config_list);
+	gerror = pushfile(&file, filename);
+	if (gerror)
+		return gerror;
+	topfile = file;
+
+	yyparse();
+	popfile();
+
+	/* Free macros and check which have not been used. */
+	TAILQ_FOREACH_SAFE(sym, &symhead, entry, next) {
+		if (!sym->persist) {
+			free(sym->nam);
+			free(sym->val);
+			TAILQ_REMOVE(&symhead, sym, entry);
+			free(sym);
+		}
+	}
+
+	*conf_list = &got_config_list;
+	return gerror;
+}
+
+int
+symset(const char *nam, const char *val, int persist)
+{
+	struct sym	*sym;
+
+	TAILQ_FOREACH(sym, &symhead, entry) {
+		if (strcmp(nam, sym->nam) == 0)
+			break;
+	}
+
+	if (sym != NULL) {
+		if (sym->persist == 1)
+			return (0);
+		else {
+			free(sym->nam);
+			free(sym->val);
+			TAILQ_REMOVE(&symhead, sym, entry);
+			free(sym);
+		}
+	}
+	if ((sym = calloc(1, sizeof(*sym))) == NULL)
+		return (-1);
+
+	sym->nam = strdup(nam);
+	if (sym->nam == NULL) {
+		free(sym);
+		return (-1);
+	}
+	sym->val = strdup(val);
+	if (sym->val == NULL) {
+		free(sym->nam);
+		free(sym);
+		return (-1);
+	}
+	sym->used = 0;
+	sym->persist = persist;
+	TAILQ_INSERT_TAIL(&symhead, sym, entry);
+	return (0);
+}
+
+int
+cmdline_symset(char *s)
+{
+	char	*sym, *val;
+	int	ret;
+	size_t	len;
+
+	if ((val = strrchr(s, '=')) == NULL)
+		return (-1);
+
+	len = strlen(s) - strlen(val) + 1;
+	if ((sym = malloc(len)) == NULL)
+		errx(1, "cmdline_symset: malloc");
+
+	strlcpy(sym, s, len);
+
+	ret = symset(sym, val + 1, 1);
+	free(sym);
+
+	return (ret);
+}
+
+char *
+symget(const char *nam)
+{
+	struct sym	*sym;
+
+	TAILQ_FOREACH(sym, &symhead, entry) {
+		if (strcmp(nam, sym->nam) == 0) {
+			sym->used = 1;
+			return (sym->val);
+		}
+	}
+	return (NULL);
+}