Implement new lexer that is not generated by flex to reduce code size

6 files changed, 505 insertions, 252 deletions

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 10e5656..1fe0873 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -11,7 +11,6 @@ endif(WITH_METHOD_AES128_GCM)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${FASTD_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${CAP_INCLUDE_DIR} ${UECC_INCLUDE_DIRS} ${NACL_INCLUDE_DIR})
 link_directories(${UECC_LIBRARY_DIRS})
 
-FLEX_TARGET(fastd_config_lex config.l ${CMAKE_CURRENT_BINARY_DIR}/config.ll.c)
 BISON_TARGET(fastd_config_parse config.y ${CMAKE_CURRENT_BINARY_DIR}/config.yy.c)
 
 add_executable(fastd
@@ -21,6 +20,7 @@ add_executable(fastd
   crypto.c
   crypto_linux.c
   handshake.c
+  lex.c
   options.c
   peer.c
   printf.c
@@ -34,7 +34,6 @@ add_executable(fastd
   task.c
   tuntap.c
   protocol_ec25519_fhmqvc.c
-  ${FLEX_fastd_config_lex_OUTPUTS}
   ${BISON_fastd_config_parse_OUTPUTS}
   ${METHODS}
 )
diff --git a/src/config.c b/src/config.c
index 128b114..8a417e9 100644
--- a/src/config.c
+++ b/src/config.c
@@ -25,8 +25,8 @@
 
 
 #include "fastd.h"
+#include "lex.h"
 #include "peer.h"
-#include <config.ll.h>
 #include <config.yy.h>
 
 #include <dirent.h>
@@ -414,11 +414,10 @@ bool fastd_read_config(fastd_context_t *ctx, fastd_config_t *conf, const char *f
 	char *filename2 = NULL;
 	char *dir = NULL;
 	FILE *file;
-	yyscan_t scanner;
+	fastd_lex_t *lex = NULL;
 	fastd_config_pstate *ps;
 	fastd_string_stack_t *strings = NULL;
 
-	fastd_config_yylex_init(&scanner);
 	ps = fastd_config_pstate_new();
 
 	if (!filename) {
@@ -433,7 +432,7 @@ bool fastd_read_config(fastd_context_t *ctx, fastd_config_t *conf, const char *f
 		}
 	}
 
-	fastd_config_yyset_in(file, scanner);
+	lex = fastd_lex_init(file);
 
 	if (filename) {
 		filename2 = strdup(filename);
@@ -458,7 +457,7 @@ bool fastd_read_config(fastd_context_t *ctx, fastd_config_t *conf, const char *f
 	int parse_ret = fastd_config_push_parse(ps, token, &token_val, &loc, ctx, conf, filename, depth+1);
 
 	while(parse_ret == YYPUSH_MORE) {
-		token = fastd_config_yylex(&token_val, &loc, scanner);
+		token = fastd_lex(&token_val, &loc, lex);
 
 		if (token < 0) {
 			pr_error(ctx, "config error: %s at %s:%i:%i", token_val.error, filename, loc.first_line, loc.first_column);
@@ -480,8 +479,8 @@ bool fastd_read_config(fastd_context_t *ctx, fastd_config_t *conf, const char *f
  end_free:
 	fastd_string_stack_free(strings);
 
+	fastd_lex_destroy(lex);
 	fastd_config_pstate_delete(ps);
-	fastd_config_yylex_destroy(scanner);
 
 	if(chdir(oldcwd))
 		pr_error(ctx, "can't chdir to `%s': %s", oldcwd, strerror(errno));
diff --git a/src/config.l b/src/config.l
deleted file mode 100644
index c460871..0000000
--- a/src/config.l
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
-  Copyright (c) 2012-2013, Matthias Schiffer <mschiffer@universe-factory.net>
-  All rights reserved.
-
-  Redistribution and use in source and binary forms, with or without
-  modification, are permitted provided that the following conditions are met:
-
-    1. Redistributions of source code must retain the above copyright notice,
-       this list of conditions and the following disclaimer.
-    2. Redistributions in binary form must reproduce the above copyright notice,
-       this list of conditions and the following disclaimer in the documentation
-       and/or other materials provided with the distribution.
-
-  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-
-%option prefix="fastd_config_yy"
-%option noyywrap
-%option nounput
-%option noinput
-%option bison-bridge
-%option bison-locations
-%option reentrant
-%option warn
-
-
-%s NEEDSPACE
-%s STRING
-%s COMMENT
-
-
-%top {
-#include <config.yy.h>
-}
-
-%{
-typedef struct keyword {
-	const char *keyword;
-	int token;
-} keyword_t;
-
-/* the keyword list must be sorted */
-static const keyword_t keywords[] = {
-	{ "addresses", TOK_ADDRESSES },
-	{ "any", TOK_ANY },
-	{ "as", TOK_AS },
-	{ "auto", TOK_AUTO },
-	{ "bind", TOK_BIND },
-	{ "capabilities", TOK_CAPABILITIES },
-	{ "crypto", TOK_CRYPTO },
-	{ "debug", TOK_DEBUG },
-	{ "default", TOK_DEFAULT },
-	{ "disestablish", TOK_DISESTABLISH },
-	{ "down", TOK_DOWN },
-	{ "drop", TOK_DROP },
-	{ "early", TOK_EARLY },
-	{ "error", TOK_ERROR },
-	{ "establish", TOK_ESTABLISH },
-	{ "fatal", TOK_FATAL },
-	{ "float", TOK_FLOAT },
-	{ "forward", TOK_FORWARD },
-	{ "from", TOK_FROM },
-	{ "group", TOK_GROUP },
-	{ "hide", TOK_HIDE },
-	{ "include", TOK_INCLUDE },
-	{ "info", TOK_INFO },
-	{ "interface", TOK_INTERFACE },
-	{ "ip", TOK_IP },
-	{ "ipv4", TOK_IPV4 },
-	{ "ipv6", TOK_IPV6 },
-	{ "key", TOK_KEY },
-	{ "level", TOK_LEVEL },
-	{ "limit", TOK_LIMIT },
-	{ "log", TOK_LOG },
-	{ "mac", TOK_MAC },
-	{ "method", TOK_METHOD },
-	{ "mode", TOK_MODE },
-	{ "mtu", TOK_MTU },
-	{ "no", TOK_NO },
-	{ "on", TOK_ON },
-	{ "peer", TOK_PEER },
-	{ "peers", TOK_PEERS },
-	{ "pmtu", TOK_PMTU },
-	{ "port", TOK_PORT },
-	{ "post-down", TOK_POST_DOWN },
-	{ "pre-up", TOK_PRE_UP },
-	{ "protocol", TOK_PROTOCOL },
-	{ "remote", TOK_REMOTE },
-	{ "secret", TOK_SECRET },
-	{ "stderr", TOK_STDERR },
-	{ "syslog", TOK_SYSLOG },
-	{ "tap", TOK_TAP },
-	{ "to", TOK_TO },
-	{ "tun", TOK_TUN },
-	{ "up", TOK_UP },
-	{ "use", TOK_USE },
-	{ "user", TOK_USER },
-	{ "verbose", TOK_VERBOSE },
-	{ "verify", TOK_VERIFY },
-	{ "warn", TOK_WARN },
-	{ "yes", TOK_YES },
-};
-
-static int compare_keywords(const void *v1, const void *v2) {
-	const keyword_t *k1 = v1, *k2 = v2;
-	return strcmp(k1->keyword, k2->keyword);
-}
-
-#define UPDATE_LOCATION do {					      \
-		yylloc->first_line = yylloc->last_line;		      \
-		yylloc->first_column = yylloc->last_column+1;	      \
-		yylloc->last_column += yyleng;			      \
-	} while (0)
-%}
-
-%%
-
-<INITIAL>{
-[a-z][-a-z0-9]* {
-	const keyword_t key = {yytext};
-	const keyword_t *ret = bsearch(&key, keywords, sizeof(keywords)/sizeof(keyword_t), sizeof(keyword_t), compare_keywords);
-
-	UPDATE_LOCATION;
-
-	if (!ret) {
-		yylval->error = "syntax error";
-		return -1;
-	}
-
-	BEGIN(NEEDSPACE);
-	return ret->token;
-}
-
-[0-9]+ {
-	char *endptr;
-
-	UPDATE_LOCATION;
-
-	yylval->uint64 = strtoull(yytext, &endptr, 10);
-	if (*endptr) {
-		yylval->error = "invalid integer constant";
-		return -1;
-	}
-
-
-	BEGIN(NEEDSPACE);
-	return TOK_UINT;
-}
-
-[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3} {
-			UPDATE_LOCATION;
-
-			if (!inet_pton(AF_INET, yytext, &yylval->addr4)) {
-				yylval->error = "invalid address";
-				return -1;
-			}
-
-			BEGIN(NEEDSPACE);
-			return TOK_ADDR4;
-		}
-
-\[[0-9a-fA-F:]+\] {
-			UPDATE_LOCATION;
-
-			yytext[yyleng-1] = 0;
-
-			if (!inet_pton(AF_INET6, yytext+1, &yylval->addr6)) {
-				yylval->error = "invalid address";
-				return -1;
-			}
-
-			BEGIN(NEEDSPACE);
-			return TOK_ADDR6;
-		}
-}
-
-<INITIAL,NEEDSPACE>{
-[;:\{\}]	{ UPDATE_LOCATION; BEGIN(INITIAL); return yytext[0]; }
-
-\n		{ yylloc->last_column = 0; yylloc->last_line++; BEGIN(INITIAL); }
-
-[ \t\r]		|
-#.*		|
-\/\/.*		{ UPDATE_LOCATION; BEGIN(INITIAL); }
-
-\/\*		{ UPDATE_LOCATION; BEGIN(COMMENT); }
-}
-
-<INITIAL>\"	{ UPDATE_LOCATION; BEGIN(STRING); }
-<STRING>[^"\\\n] { yylloc->last_column++; yymore(); }
-<STRING>\n	{ yylloc->last_line++; yylloc->last_column = 0; yymore(); }
-<STRING>\\.	{ yylloc->last_column+=2; yymore(); }
-<STRING>\\\n	{ yylloc->last_line++; yylloc->last_column = 0; yymore(); }
-<STRING>\" {
-			int i, esc = 0;
-
-			for (i = 0; i < yyleng; i++) {
-				if (yytext[i] == '\\') {
-					i++;
-					if (yytext[i] == '\n') {
-						esc+=2;
-					}
-					else {
-						yytext[i-esc-1] = yytext[i];
-						esc++;
-					}
-				}
-				else if(esc) {
-					yytext[i-esc] = yytext[i];
-				}
-			}
-			yytext[yyleng-esc-1] = 0;
-			yylval->str = fastd_string_stack_dup(yytext);
-			BEGIN(NEEDSPACE);
-			yylloc->last_column++;
-			return TOK_STRING;
-
-		}
-
-<COMMENT>\*\/	{ yylloc->last_column += yyleng; BEGIN(INITIAL); }
-<COMMENT>[^\n]  { yylloc->last_column++; }
-<COMMENT>\n	{ yylloc->last_line++; yylloc->last_column = 0; }
-
-.		{
-			yylloc->first_line = yylloc->last_line;
-			yylloc->first_column = yylloc->last_column+1;
-			yylval->error = "syntax error";
-			return -1;
-		}
-
-<INITIAL><<EOF>> { return 0; }
-<COMMENT><<EOF>> { yylval->error = "unterminated block comment"; return -1; }
-<STRING><<EOF>>	{ yylval->error = "unterminated string"; return -1; }
-%%
diff --git a/src/lex.c b/src/lex.c
new file mode 100644
index 0000000..575d83e
--- /dev/null
+++ b/src/lex.c
@@ -0,0 +1,457 @@
+/*
+  Copyright (c) 2012-2013, Matthias Schiffer <mschiffer@universe-factory.net>
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+    2. Redistributions in binary form must reproduce the above copyright notice,
+       this list of conditions and the following disclaimer in the documentation
+       and/or other materials provided with the distribution.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+
+#include "lex.h"
+
+#include <stdlib.h>
+
+
+struct fastd_lex {
+	FILE *file;
+
+	bool needspace;
+
+	size_t start;
+	size_t end;
+	size_t tok_len;
+	char buffer[1024];
+};
+
+
+typedef struct keyword {
+	const char *keyword;
+	int token;
+} keyword_t;
+
+/* the keyword list must be sorted */
+static const keyword_t keywords[] = {
+	{ "addresses", TOK_ADDRESSES },
+	{ "any", TOK_ANY },
+	{ "as", TOK_AS },
+	{ "auto", TOK_AUTO },
+	{ "bind", TOK_BIND },
+	{ "capabilities", TOK_CAPABILITIES },
+	{ "crypto", TOK_CRYPTO },
+	{ "debug", TOK_DEBUG },
+	{ "default", TOK_DEFAULT },
+	{ "disestablish", TOK_DISESTABLISH },
+	{ "down", TOK_DOWN },
+	{ "drop", TOK_DROP },
+	{ "early", TOK_EARLY },
+	{ "error", TOK_ERROR },
+	{ "establish", TOK_ESTABLISH },
+	{ "fatal", TOK_FATAL },
+	{ "float", TOK_FLOAT },
+	{ "forward", TOK_FORWARD },
+	{ "from", TOK_FROM },
+	{ "group", TOK_GROUP },
+	{ "hide", TOK_HIDE },
+	{ "include", TOK_INCLUDE },
+	{ "info", TOK_INFO },
+	{ "interface", TOK_INTERFACE },
+	{ "ip", TOK_IP },
+	{ "ipv4", TOK_IPV4 },
+	{ "ipv6", TOK_IPV6 },
+	{ "key", TOK_KEY },
+	{ "level", TOK_LEVEL },
+	{ "limit", TOK_LIMIT },
+	{ "log", TOK_LOG },
+	{ "mac", TOK_MAC },
+	{ "method", TOK_METHOD },
+	{ "mode", TOK_MODE },
+	{ "mtu", TOK_MTU },
+	{ "no", TOK_NO },
+	{ "on", TOK_ON },
+	{ "peer", TOK_PEER },
+	{ "peers", TOK_PEERS },
+	{ "pmtu", TOK_PMTU },
+	{ "port", TOK_PORT },
+	{ "post-down", TOK_POST_DOWN },
+	{ "pre-up", TOK_PRE_UP },
+	{ "protocol", TOK_PROTOCOL },
+	{ "remote", TOK_REMOTE },
+	{ "secret", TOK_SECRET },
+	{ "stderr", TOK_STDERR },
+	{ "syslog", TOK_SYSLOG },
+	{ "tap", TOK_TAP },
+	{ "to", TOK_TO },
+	{ "tun", TOK_TUN },
+	{ "up", TOK_UP },
+	{ "use", TOK_USE },
+	{ "user", TOK_USER },
+	{ "verbose", TOK_VERBOSE },
+	{ "verify", TOK_VERIFY },
+	{ "warn", TOK_WARN },
+	{ "yes", TOK_YES },
+};
+
+static int compare_keywords(const void *v1, const void *v2) {
+	const keyword_t *k1 = v1, *k2 = v2;
+	return strcmp(k1->keyword, k2->keyword);
+}
+
+
+static bool advance(fastd_lex_t *lex) {
+	if (lex->start > 0) {
+		memmove(lex->buffer, lex->buffer+lex->start, lex->end - lex->start);
+		lex->end -= lex->start;
+		lex->start = 0;
+	}
+
+	if (lex->end == sizeof(lex->buffer))
+		return false;
+
+	size_t l = fread(lex->buffer+lex->end, 1, sizeof(lex->buffer) - lex->end, lex->file);
+
+	lex->end += l;
+	return l;
+}
+
+static inline char current(fastd_lex_t *lex) {
+	return lex->buffer[lex->start + lex->tok_len];
+}
+
+static char* get_token(fastd_lex_t *lex) {
+	return strndup(lex->buffer+lex->start, lex->tok_len);
+}
+
+static bool next(YYLTYPE *yylloc, fastd_lex_t *lex, bool move) {
+	if (lex->start + lex->tok_len >= lex->end)
+		return false;
+
+	if (current(lex) == '\n') {
+		yylloc->last_column = 0;
+		yylloc->last_line++;
+	}
+	else {
+		yylloc->last_column++;
+	}
+
+	if (move)
+		lex->start++;
+	else
+		lex->tok_len++;
+
+
+	if (lex->start + lex->tok_len >= lex->end)
+		return advance(lex);
+
+	return true;
+}
+
+static void consume(fastd_lex_t *lex, bool needspace) {
+	lex->start += lex->tok_len;
+	lex->tok_len = 0;
+
+	lex->needspace = needspace;
+}
+
+static int syntax_error(YYSTYPE *yylval, fastd_lex_t *lex) {
+	yylval->error = "syntax error";
+	return -1;
+}
+
+static int io_error(YYSTYPE *yylval, fastd_lex_t *lex) {
+	yylval->error = "I/O error";
+	return -1;
+}
+
+static inline int end(YYSTYPE *yylval, fastd_lex_t *lex) {
+	if (ferror(lex->file))
+		return io_error(yylval, lex);
+
+	return 0;
+}
+
+static int consume_comment(YYSTYPE *yylval, YYLTYPE *yylloc, fastd_lex_t *lex) {
+	char prev = 0;
+
+	while (next(yylloc, lex, true)) {
+		if (prev == '*' && current(lex) == '/') {
+			next(yylloc, lex, true);
+			consume(lex, false);
+			return 0;
+		}
+
+		prev = current(lex);
+	}
+
+	if (ferror(lex->file))
+		return io_error(yylval, lex);
+
+	yylval->error = "unterminated block comment";
+	return -1;
+}
+
+static int unterminated_string(YYSTYPE *yylval, fastd_lex_t *lex) {
+	if (ferror(lex->file))
+		return io_error(yylval, lex);
+
+	yylval->error = "unterminated string";
+	return -1;
+}
+
+static int parse_string(YYSTYPE *yylval, YYLTYPE *yylloc, fastd_lex_t *lex) {
+	char *buf = NULL;
+	size_t len = 1024;
+	size_t pos = 0;
+
+	if (lex->needspace)
+		return syntax_error(yylval, lex);
+
+	buf = malloc(len);
+
+	while (true) {
+		if (!next(yylloc, lex, true)) {
+			free(buf);
+			return unterminated_string(yylval, lex);
+		}
+
+		char cur = current(lex);
+
+		if (cur == '"')
+			break;
+
+		if (cur == '\\') {
+			if (!next(yylloc, lex, true)) {
+				free(buf);
+				return unterminated_string(yylval, lex);
+			}
+
+			cur = current(lex);
+
+			if (cur == '\n')
+				continue;
+		}
+
+		if (pos >= len) {
+			len *= 2;
+			buf = realloc(buf, len);
+		}
+
+		buf[pos++] = cur;
+	}
+
+	yylval->str = fastd_string_stack_dupn(buf, pos);
+	free(buf);
+
+	next(yylloc, lex, true);
+	consume(lex, true);
+
+	return TOK_STRING;
+}
+
+static int parse_ipv6_address(YYSTYPE *yylval, YYLTYPE *yylloc, fastd_lex_t *lex) {
+	if (lex->needspace)
+		return syntax_error(yylval, lex);
+
+	while (next(yylloc, lex, false)) {
+		char cur = current(lex);
+
+		if (!((cur >= '0' && cur <= '9') || cur == ':'))
+			break;
+	}
+
+	bool ok = (current(lex) == ']');
+
+	if (ok) {
+		lex->buffer[lex->start + lex->tok_len] = 0;
+		ok = inet_pton(AF_INET6, lex->buffer+lex->start+1, &yylval->addr6);
+	}
+
+	if (!ok) {
+		yylval->error = "invalid address";
+		return -1;
+	}
+
+	next(yylloc, lex, true);
+	consume(lex, true);
+
+	return TOK_ADDR6;
+}
+
+static int parse_ipv4_address(YYSTYPE *yylval, YYLTYPE *yylloc, fastd_lex_t *lex) {
+	if (lex->needspace)
+		return syntax_error(yylval, lex);
+
+	while (next(yylloc, lex, false)) {
+		char cur = current(lex);
+
+		if (!((cur >= '0' && cur <= '9') || cur == '.'))
+			break;
+	}
+
+	char *token = get_token(lex);
+	bool ok = inet_pton(AF_INET, token, &yylval->addr4);
+
+	free(token);
+
+	if (!ok) {
+		yylval->error = "invalid address";
+		return -1;
+	}
+
+	return TOK_ADDR4;
+}
+
+static int parse_number(YYSTYPE *yylval, YYLTYPE *yylloc, fastd_lex_t *lex) {
+	if (lex->needspace)
+		return syntax_error(yylval, lex);
+
+	while (next(yylloc, lex, false)) {
+		char cur = current(lex);
+
+		if (cur == '.')
+			return parse_ipv4_address(yylval, yylloc, lex);
+
+		if (!(cur >= '0' && cur <= '9'))
+			break;
+	}
+
+	char *endptr, *token = get_token(lex);
+	yylval->uint64 = strtoull(token, &endptr, 10);
+
+	bool ok = !*endptr;
+	free(token);
+
+	if (!ok) {
+		yylval->error = "invalid integer constant";
+		return -1;
+	}
+
+	return TOK_UINT;
+}
+
+static int parse_keyword(YYSTYPE *yylval, YYLTYPE *yylloc, fastd_lex_t *lex) {
+	if (lex->needspace)
+		return syntax_error(yylval, lex);
+
+	while (next(yylloc, lex, false)) {
+		char cur = current(lex);
+
+		if (!((cur >= 'a' && cur <= 'z') || (cur >= '0' && cur <= '9') || cur == '-'))
+			break;
+	}
+
+	char *token = get_token(lex);
+	const keyword_t key = {token};
+	const keyword_t *ret = bsearch(&key, keywords, sizeof(keywords)/sizeof(keyword_t), sizeof(keyword_t), compare_keywords);
+	free(token);
+
+	if (!ret)
+		return syntax_error(yylval, lex);
+
+	consume(lex, true);
+
+	return ret->token;
+}
+
+fastd_lex_t* fastd_lex_init(FILE *file) {
+	fastd_lex_t *lex = calloc(1, sizeof(fastd_lex_t));
+	lex->file = file;
+
+	advance(lex);
+
+	return lex;
+}
+
+void fastd_lex_destroy(fastd_lex_t *lex) {
+	if (!lex)
+		return;
+
+	free(lex);
+}
+
+int fastd_lex(YYSTYPE *yylval, YYLTYPE *yylloc, fastd_lex_t *lex) {
+	int token;
+
+	while (lex->end > lex->start) {
+		yylloc->first_line = yylloc->last_line;
+		yylloc->first_column = yylloc->last_column+1;
+
+		switch (current(lex)) {
+		case ' ':
+		case '\n':
+		case '\t':
+		case '\r':
+			next(yylloc, lex, true);
+			consume(lex, false);
+			continue;
+
+		case ';':
+		case ':':
+		case '{':
+		case '}':
+			token = current(lex);
+			next(yylloc, lex, true);
+			consume(lex, false);
+			return token;
+
+		case '/':
+			if (!next(yylloc, lex, true))
+				return syntax_error(yylval, lex);
+
+			if (current(lex) == '*') {
+				token = consume_comment(yylval, yylloc, lex);
+				if (token)
+					return token;
+
+				continue;
+			}
+
+			if (current(lex) != '/')
+				return syntax_error(yylval, lex);
+
+			/* fall-through */
+		case '#':
+			while (next(yylloc, lex, true)) {
+				if (current(lex) == '\n')
+					break;
+			}
+
+			next(yylloc, lex, true);
+			consume(lex, false);
+			continue;
+
+		case '"':
+			return parse_string(yylval, yylloc, lex);
+
+		case '[':
+			return parse_ipv6_address(yylval, yylloc, lex);
+
+		case '0' ... '9':
+			return parse_number(yylval, yylloc, lex);
+
+		case 'a' ... 'z':
+			return parse_keyword(yylval, yylloc, lex);
+
+		default:
+			return syntax_error(yylval, lex);
+		}
+	}
+
+	return end(yylval, lex);
+}
diff --git a/src/lex.h b/src/lex.h
new file mode 100644
index 0000000..2a78421
--- /dev/null
+++ b/src/lex.h
@@ -0,0 +1,41 @@
+/*
+  Copyright (c) 2012-2013, Matthias Schiffer <mschiffer@universe-factory.net>
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+    2. Redistributions in binary form must reproduce the above copyright notice,
+       this list of conditions and the following disclaimer in the documentation
+       and/or other materials provided with the distribution.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+
+#ifndef _FASTD_LEX_H_
+#define _FASTD_LEX_H_
+
+#include "types.h"
+#include <config.yy.h>
+
+#include <stdio.h>
+
+
+fastd_lex_t* fastd_lex_init(FILE *file);
+void fastd_lex_destroy(fastd_lex_t *lex);
+
+int fastd_lex(YYSTYPE *yylval, YYLTYPE *yylloc, fastd_lex_t *lex);
+
+#endif /* _FASTD_LEX_H_ */
diff --git a/src/types.h b/src/types.h
index b1026b5..8eaa55d 100644
--- a/src/types.h
+++ b/src/types.h
@@ -87,6 +87,7 @@ typedef struct fastd_method fastd_method_t;
 
 typedef struct fastd_handshake fastd_handshake_t;
 
+typedef struct fastd_lex fastd_lex_t;
 typedef struct fastd_string_stack fastd_string_stack_t;
 
 typedef struct fastd_resolve_return fastd_resolve_return_t;